From fdbab27dd72378529e17cfd8a77811c90297fe9e Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 30 May 2026 12:02:13 +0300 Subject: [PATCH 01/27] Few implementations of RMQ --- .../benchmarks/scripts/plot_benchmarks.py | 159 +++++++++ agentic/cpp/skills/diagnose-segfault/SKILL.md | 195 +++++++++++ include/pixie/bits.h | 115 +++++++ include/pixie/rmq.h | 7 + include/pixie/rmq/bp_plus_minus_one_rmq.h | 258 +++++++++++++++ include/pixie/rmq/cartesian_tree_rmq.h | 223 +++++++++++++ include/pixie/rmq/rmq_base.h | 56 ++++ include/pixie/rmq/segment_tree.h | 116 +++++++ include/pixie/rmq/sparse_table.h | 104 ++++++ src/benchmarks/bench_rmq.cpp | 188 +++++++++++ src/tests/excess_positions_tests.cpp | 114 +++++++ src/tests/rmq_tests.cpp | 308 ++++++++++++++++++ 12 files changed, 1843 insertions(+) create mode 100644 agentic/cpp/skills/benchmarks/scripts/plot_benchmarks.py create mode 100644 agentic/cpp/skills/diagnose-segfault/SKILL.md create mode 100644 include/pixie/rmq.h create mode 100644 include/pixie/rmq/bp_plus_minus_one_rmq.h create mode 100644 include/pixie/rmq/cartesian_tree_rmq.h create mode 100644 include/pixie/rmq/rmq_base.h create mode 100644 include/pixie/rmq/segment_tree.h create mode 100644 include/pixie/rmq/sparse_table.h create mode 100644 src/benchmarks/bench_rmq.cpp create mode 100644 src/tests/rmq_tests.cpp diff --git a/agentic/cpp/skills/benchmarks/scripts/plot_benchmarks.py b/agentic/cpp/skills/benchmarks/scripts/plot_benchmarks.py new file mode 100644 index 0000000..abe5040 --- /dev/null +++ b/agentic/cpp/skills/benchmarks/scripts/plot_benchmarks.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Generic Google Benchmark JSON plotter. + +Plots timing and (optionally) hardware counter data from Google Benchmark +--benchmark_format=json output. Correctly handles repetition output by +averaging raw iterations and skipping aggregate entries. + +Usage: + python3 plot_benchmarks.py results.json [output_prefix] + python3 plot_benchmarks.py results.json report --min-n 16 --max-n 1048576 +""" +import json +import sys +import argparse +import matplotlib.pyplot as plt +import numpy as np + +def load_benchmark_json(path): + with open(path, 'r') as f: + return json.load(f) + +def extract_series(data, metric='time', min_n=None, max_n=None): + """Extract series from benchmark JSON. + + metric='time' -> real_time (ns) + metric='counter' -> first available hardware counter (e.g. CACHE-MISSES) + """ + # Determine counter key if requested + counter_key = None + if metric == 'counter': + # Find first counter key from the first iteration entry + for bench in data.get('benchmarks', []): + if bench.get('run_type', 'iteration') != 'iteration': + continue + for k in bench.keys(): + if k not in ('name', 'run_name', 'run_type', 'repetitions', + 'repetition_index', 'threads', 'iterations', + 'real_time', 'cpu_time', 'time_unit', + 'items_per_second', 'aggregate_name', + 'aggregate_unit', 'family_index', + 'per_family_instance_index'): + counter_key = k + break + if counter_key: + break + if not counter_key: + return {} + + raw = {} + for bench in data.get('benchmarks', []): + if bench.get('run_type', 'iteration') != 'iteration': + continue + + name = bench['name'] + parts = name.split('/') + if len(parts) < 2: + continue + + bench_name = parts[0] + try: + n = int(parts[1]) + except ValueError: + continue + + if min_n is not None and n < min_n: + continue + if max_n is not None and n > max_n: + continue + + if metric == 'time': + val = bench.get('real_time', bench.get('cpu_time', 0)) + else: + val = bench.get(counter_key) + + if val is None or val == 0: + continue + + key = (bench_name, n) + raw.setdefault(key, []).append(val) + + series = {} + for (bench_name, n), vals in raw.items(): + series.setdefault(bench_name, []).append((n, sum(vals) / len(vals))) + + for name in series: + series[name].sort(key=lambda x: x[0]) + + return series + +def plot_series(series, output_prefix, ylabel, title_suffix): + if not series: + print(f"No data to plot for {title_suffix}") + return + + fig, ax = plt.subplots(figsize=(12, 8)) + colors = plt.cm.tab10(np.linspace(0, 1, len(series))) + + for idx, (name, points) in enumerate(sorted(series.items())): + xs = [p[0] for p in points] + ys = [p[1] for p in points] + ax.plot(xs, ys, marker='o', markersize=3, label=name, color=colors[idx]) + + ax.set_xscale('log') + ax.set_xlabel('Benchmark parameter n') + ax.set_ylabel(ylabel) + ax.set_title(f'Benchmark Results - {title_suffix}') + ax.legend(loc='upper left', fontsize='small') + ax.grid(True, which='both', ls='--', alpha=0.5) + fig.tight_layout() + fig.savefig(f'{output_prefix}.png', dpi=150) + fig.savefig(f'{output_prefix}.svg') + print(f'Saved {output_prefix}.png and {output_prefix}.svg') + plt.close(fig) + +def main(): + parser = argparse.ArgumentParser( + description='Plot Google Benchmark JSON results.') + parser.add_argument('json_path', help='Path to benchmark JSON file') + parser.add_argument('output_prefix', nargs='?', default='report', + help='Output file prefix (default: report)') + parser.add_argument('--min-n', type=int, default=None, + help='Minimum parameter value to include (inclusive)') + parser.add_argument('--max-n', type=int, default=None, + help='Maximum parameter value to include (inclusive)') + args = parser.parse_args() + + data = load_benchmark_json(args.json_path) + + # Timing plot + time_series = extract_series(data, metric='time', + min_n=args.min_n, max_n=args.max_n) + if time_series: + plot_series(time_series, args.output_prefix, + 'Time per iteration (ns)', 'Time') + + # Counter plot + counter_series = extract_series(data, metric='counter', + min_n=args.min_n, max_n=args.max_n) + if counter_series: + # Determine counter name for labels + counter_name = 'Counter' + for bench in data.get('benchmarks', []): + if bench.get('run_type', 'iteration') != 'iteration': + continue + for k in bench.keys(): + if k not in ('name', 'run_name', 'run_type', 'repetitions', + 'repetition_index', 'threads', 'iterations', + 'real_time', 'cpu_time', 'time_unit', + 'items_per_second', 'aggregate_name', + 'aggregate_unit', 'family_index', + 'per_family_instance_index'): + counter_name = k + break + break + plot_series(counter_series, f'{args.output_prefix}_{counter_name.lower().replace("-", "_")}', + f'{counter_name} per iteration', counter_name) + +if __name__ == '__main__': + main() diff --git a/agentic/cpp/skills/diagnose-segfault/SKILL.md b/agentic/cpp/skills/diagnose-segfault/SKILL.md new file mode 100644 index 0000000..91c8950 --- /dev/null +++ b/agentic/cpp/skills/diagnose-segfault/SKILL.md @@ -0,0 +1,195 @@ +--- +name: diagnose-segfault +description: Diagnose C++ crashes and memory-safety errors with AddressSanitizer, GDB, and core dumps. Use when a C++ binary crashes with SIGSEGV, SIGABRT, heap-buffer-overflow, use-after-free, stack-buffer-overflow, double-free, suspected memory corruption, or an available core file. +--- + +# C++ Segfault and Memory Error Diagnosis + +Use this skill to find the first bad access or corrupting operation, not just +the frame where the process finally crashed. + +## When To Use + +- A C++ binary crashes with `Segmentation fault`, `SIGSEGV`, or `SIGABRT`. +- AddressSanitizer reports `ERROR: AddressSanitizer:`. +- A test reports heap-buffer-overflow, stack-buffer-overflow, use-after-free, + double-free, global-buffer-overflow, or similar memory-safety failures. +- A core dump exists and the user wants root-cause analysis. +- Memory corruption is suspected but the immediate failure is ambiguous. + +For repository-specific binary names, CMake options, or known reproducer +patterns, also read `agentic/local/cpp/skills/diagnose-segfault/EXAMPLES.md` +when present. + +## Workflow 1: ASan First + +Prefer AddressSanitizer when the issue is reproducible. It usually reports the +bad access with file and line information. + +### Build With ASan + +For CMake projects, first check whether the repository already has an ASan +preset or option. If not, configure a dedicated debug build: + +```bash +cmake -B build/asan -DCMAKE_BUILD_TYPE=Debug -DENABLE_ADDRESS_SANITIZER=ON +cmake --build build/asan -j +``` + +For non-CMake builds, compile and link with: + +```bash +-fsanitize=address -fno-omit-frame-pointer -g -O1 +``` + +Use `-O0` instead of `-O1` when you expect to inspect many local variables in +GDB. + +### Run The Minimal Reproducer + +Run the specific binary, test case, or input that triggers the crash. For Google +Test binaries, prefer a narrow filter: + +```bash +./build/asan/unittests --gtest_filter="SuiteName.TestName" +``` + +### Read The ASan Report + +Focus on: + +| Report section | Meaning | +|---|---| +| `ERROR: AddressSanitizer: ` | Error class | +| `READ/WRITE of size N` | Access direction and size | +| First user-code frame | Exact bad access | +| Allocation/deallocation stack | Object lifetime and ownership | +| Shadow-byte legend | Boundary or lifetime category | +| `SUMMARY:` | One-line location summary | + +Useful options: + +```bash +ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=1 +ASAN_OPTIONS=halt_on_error=0 +ASAN_OPTIONS=print_stats=1 +``` + +Disable leak detection while diagnosing a crash if leak noise hides the primary +failure: + +```bash +ASAN_OPTIONS=detect_leaks=0 ./build/asan/unittests +``` + +## Workflow 2: GDB Live Debugging + +Use GDB when ASan is unavailable, when the crash is not a direct memory-safety +violation, or when variable inspection is needed. + +Build with debug symbols: + +```bash +cmake -B build/debug -DCMAKE_BUILD_TYPE=Debug +cmake --build build/debug -j +``` + +Run under GDB: + +```bash +gdb --args [arguments...] +``` + +Core commands: + +```gdb +run +bt full +info registers +info locals +info args +frame N +list +print +thread apply all bt +``` + +Make C++ values easier to inspect: + +```gdb +set print pretty on +set print object on +set pagination off +``` + +## Workflow 3: ASan Under GDB + +Use this when ASan points at a bad access but the pointer or lifetime corruption +comes from an earlier frame. + +```bash +gdb --args [arguments...] +``` + +Break on ASan reporting or abort: + +```gdb +break __asan::ReportGenericError +catch signal SIGABRT +run +bt full +``` + +Then inspect the last user-code frames before ASan internals. + +## Workflow 4: Core Dump Analysis + +Use when the crash already happened or reproduction is expensive. + +Enable core dumps for future runs if needed: + +```bash +ulimit -c unlimited +``` + +Analyze: + +```bash +gdb +``` + +Useful commands: + +```gdb +bt full +info threads +thread apply all bt +frame N +info locals +info args +print +``` + +## Common ASan Errors + +| Error type | Typical cause | +|---|---| +| `heap-buffer-overflow` | Read or write past heap allocation bounds | +| `stack-buffer-overflow` | Read or write past a local stack object | +| `global-buffer-overflow` | Read or write past global/static storage | +| `heap-use-after-free` | Access after `delete`, `free`, or container invalidation | +| `stack-use-after-return` | Pointer/reference to a returned stack frame | +| `double-free` | Object released twice | +| `alloc-dealloc-mismatch` | Mixed allocation APIs, such as `new[]` with `free` | + +## Best Practices + +1. Build with `-g`; reports without symbols are often not actionable. +2. Prefer the smallest reproducer over full-suite runs. +3. Rebuild after toggling sanitizer or debug options. +4. Treat the first ASan error as primary; later errors are often fallout. +5. Check container iterator/reference invalidation around the reported object. +6. Validate the fix with the same reproducer under ASan before running broader + tests. +7. If ASan is too slow for a large input, use GDB on the same input or reduce + the input while preserving the crash. diff --git a/include/pixie/bits.h b/include/pixie/bits.h index 7d26597..f1250eb 100644 --- a/include/pixie/bits.h +++ b/include/pixie/bits.h @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -805,8 +806,46 @@ static inline const __m256i excess_lut_bit2 = _mm256_set1_epi8(4); static inline const __m256i excess_lut_bit3 = _mm256_set1_epi8(8); static inline const __m128i excess_lut_nibble_mask = _mm_set1_epi8(0x0F); // clang-format on + +static inline __m256i excess_bit_masks_16x_i16() noexcept { + return _mm256_setr_epi16(0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, + 0x0040, 0x0080, 0x0100, 0x0200, 0x0400, 0x0800, + 0x1000, 0x2000, 0x4000, + static_cast(0x8000)); +} + +static inline __m256i excess_prefix_sum_16x_i16(__m256i v) noexcept { + __m256i x = v; + __m256i t = _mm256_slli_si256(x, 2); + x = _mm256_add_epi16(x, t); + t = _mm256_slli_si256(x, 4); + x = _mm256_add_epi16(x, t); + t = _mm256_slli_si256(x, 8); + x = _mm256_add_epi16(x, t); + + __m128i lo = _mm256_extracti128_si256(x, 0); + __m128i hi = _mm256_extracti128_si256(x, 1); + const int16_t carry = static_cast(_mm_extract_epi16(lo, 7)); + hi = _mm_add_epi16(hi, _mm_set1_epi16(carry)); + + __m256i out = _mm256_castsi128_si256(lo); + return _mm256_inserti128_si256(out, hi, 1); +} #endif +/** + * @brief Minimum prefix excess in a 128-bit bitstring range. + * @details Prefix positions are offsets in `[0, 128]`; position 0 is the + * empty prefix and position `k` is the excess after consuming the first `k` + * bits. The query range `[left, right]` is inclusive. Ties return the first + * offset attaining the minimum. Invalid ranges return offset 128 as a + * sentinel. + */ +struct ExcessMin128Result { + int min_excess = 0; + size_t offset = 128; +}; + /** * @brief Find every prefix whose excess equals target_x in a 128-bit bitstring. * @@ -935,6 +974,82 @@ static inline int prefix_excess_128(const uint64_t* s, return 2 * ones - static_cast(end_offset); } +/** + * @brief Return the minimum prefix excess and first attaining offset. + * @param s 2 little-endian uint64_t words (bit 0 of s[0] is the first bit). + * @param left First prefix position to consider, inclusive. + * @param right Last prefix position to consider, inclusive. + */ +static inline ExcessMin128Result excess_min_128(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + +#ifdef PIXIE_AVX2_SUPPORT + static const __m256i masks = excess_bit_masks_16x_i16(); + static const __m256i zero = _mm256_setzero_si256(); + static const __m256i pos = _mm256_set1_epi16(1); + static const __m256i neg = _mm256_set1_epi16(-1); + + int carry = 0; + alignas(32) int16_t prefix_values[16]; + for (size_t chunk = 0; chunk < 8; ++chunk) { + const size_t chunk_bit = chunk * 16; + const uint16_t bits = + chunk < 4 + ? static_cast((s[0] >> (chunk * 16)) & 0xFFFFu) + : static_cast((s[1] >> ((chunk - 4) * 16)) & 0xFFFFu); + const int delta = 2 * static_cast(std::popcount(bits)) - 16; + + if (chunk_bit + 1 <= right && chunk_bit + 16 >= left) { + const __m256i selected = _mm256_and_si256( + _mm256_set1_epi16(static_cast(bits)), masks); + const __m256i is_zero = _mm256_cmpeq_epi16(selected, zero); + const __m256i steps = _mm256_blendv_epi8(pos, neg, is_zero); + const __m256i pref = + _mm256_add_epi16(excess_prefix_sum_16x_i16(steps), + _mm256_set1_epi16(static_cast(carry))); + _mm256_store_si256(reinterpret_cast<__m256i*>(prefix_values), pref); + + for (size_t lane = 0; lane < 16; ++lane) { + const size_t offset = chunk_bit + lane + 1; + if (offset < left || offset > right) { + continue; + } + const int value = prefix_values[lane]; + if (value < best) { + best = value; + best_offset = offset; + } + } + } + carry += delta; + } +#else + int current = 0; + for (size_t bit = 0; bit < right; ++bit) { + current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (offset >= left && current < best) { + best = current; + best_offset = offset; + } + } +#endif + + return {best, best_offset}; +} + /** * @brief Find the first prefix reaching target_x in a 128-bit bitstring. * diff --git a/include/pixie/rmq.h b/include/pixie/rmq.h new file mode 100644 index 0000000..452dfbe --- /dev/null +++ b/include/pixie/rmq.h @@ -0,0 +1,7 @@ +#pragma once + +#include +#include +#include +#include +#include diff --git a/include/pixie/rmq/bp_plus_minus_one_rmq.h b/include/pixie/rmq/bp_plus_minus_one_rmq.h new file mode 100644 index 0000000..57af0e1 --- /dev/null +++ b/include/pixie/rmq/bp_plus_minus_one_rmq.h @@ -0,0 +1,258 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief FCB-style RMQ backend for arrays with adjacent differences ±1. + * + * @details The indexed depth sequence is represented by BP deltas: bit 1 means + * the next depth is current + 1, and bit 0 means current - 1. A sequence with + * @p depth_count depth positions has @p depth_count - 1 delta bits. Blocks + * match the 128-bit excess primitives in `bits.h`; only the absolute minimum + * value of each block is stored, and positions are recovered by rescanning the + * selected block. + * + * @tparam Index Unsigned integer type used for stored positions. + * @tparam BlockSize Number of depth positions per microblock. + */ +template +class BpPlusMinusOneRmq { + public: + static_assert(BlockSize == 128); + + static constexpr std::size_t npos = std::numeric_limits::max(); + static constexpr Index invalid_index = std::numeric_limits::max(); + + BpPlusMinusOneRmq() = default; + + BpPlusMinusOneRmq(std::span bits, + std::size_t depth_count) + : input_bits_(bits), depth_count_(depth_count) { + build(); + } + + BpPlusMinusOneRmq(const BpPlusMinusOneRmq& other) + : input_bits_(other.input_bits_), + depth_count_(other.depth_count_), + block_min_values_(other.block_min_values_) { + reset_macro_rmq(); + } + + BpPlusMinusOneRmq& operator=(const BpPlusMinusOneRmq& other) { + if (this == &other) { + return *this; + } + input_bits_ = other.input_bits_; + depth_count_ = other.depth_count_; + block_min_values_ = other.block_min_values_; + reset_macro_rmq(); + return *this; + } + + BpPlusMinusOneRmq(BpPlusMinusOneRmq&& other) noexcept + : input_bits_(other.input_bits_), + depth_count_(other.depth_count_), + block_min_values_(std::move(other.block_min_values_)) { + reset_macro_rmq(); + } + + BpPlusMinusOneRmq& operator=(BpPlusMinusOneRmq&& other) noexcept { + if (this == &other) { + return *this; + } + input_bits_ = other.input_bits_; + depth_count_ = other.depth_count_; + block_min_values_ = std::move(other.block_min_values_); + reset_macro_rmq(); + return *this; + } + + std::size_t size() const { return depth_count_; } + + bool empty() const { return depth_count_ == 0; } + + std::size_t arg_min(std::size_t left, std::size_t right) const { + if (left > right || right >= depth_count_) { + return npos; + } + + const std::size_t left_block = left / BlockSize; + const std::size_t right_block = right / BlockSize; + if (left_block == right_block) { + return scan_block_range(left_block, left % BlockSize, right % BlockSize) + .position; + } + + Candidate answer = scan_block_range(left_block, left % BlockSize, + block_size(left_block) - 1); + answer = + better(answer, scan_block_range(right_block, 0, right % BlockSize)); + + if (left_block + 1 < right_block) { + const std::size_t block_position = + macro_rmq_.arg_min(left_block + 1, right_block - 1); + if (block_position != + SparseTable, Index>::npos) { + answer = better(answer, scan_full_block(block_position)); + } + } + + return answer.position; + } + + private: + struct Candidate { + std::size_t position = npos; + std::int64_t value = std::numeric_limits::max(); + }; + + std::size_t block_size(std::size_t block) const { + const std::size_t begin = block * BlockSize; + return std::min(BlockSize, depth_count_ - begin); + } + + Candidate better(Candidate left, Candidate right) const { + if (left.position == npos) { + return right; + } + if (right.position == npos) { + return left; + } + if (right.value < left.value) { + return right; + } + if (left.value < right.value) { + return left; + } + return right.position < left.position ? right : left; + } + + void build() { + block_min_values_.clear(); + macro_rmq_ = SparseTable, Index>(); + + if (depth_count_ == 0) { + return; + } + if (depth_count_ > static_cast(invalid_index)) { + throw std::length_error("RMQ ±1 index type is too small"); + } + if (input_bits_.size() < (depth_count_ - 1 + 63) / 64) { + throw std::invalid_argument("RMQ ±1 bit span is too small"); + } + + const std::size_t block_count = (depth_count_ + BlockSize - 1) / BlockSize; + block_min_values_.reserve(block_count); + + std::int64_t base_depth = 0; + for (std::size_t block = 0; block < block_count; ++block) { + const std::size_t begin = block * BlockSize; + const std::size_t size = std::min(BlockSize, depth_count_ - begin); + std::int64_t min_depth = base_depth; + std::int64_t current_depth = base_depth; + for (std::size_t offset = 1; offset < size; ++offset) { + const std::size_t delta_position = begin + offset - 1; + const bool up = bit(delta_position); + current_depth += up ? 1 : -1; + if (current_depth < min_depth) { + min_depth = current_depth; + } + } + + block_min_values_.push_back(min_depth); + if (block + 1 < block_count) { + base_depth += block_excess(begin, next_block_delta_count(begin)); + } + } + + reset_macro_rmq(); + } + + bool bit(std::size_t position) const { + return ((input_bits_[position >> 6] >> (position & 63)) & 1u) != 0; + } + + std::uint64_t word_or_zero(std::size_t word) const { + return word < input_bits_.size() ? input_bits_[word] : 0; + } + + std::array block_bits(std::size_t block) const { + const std::size_t first_word = block * (BlockSize / 64); + return {word_or_zero(first_word), word_or_zero(first_word + 1)}; + } + + std::size_t next_block_delta_count(std::size_t begin) const { + const std::size_t next_begin = begin + BlockSize; + return std::min(next_begin, depth_count_ - 1) - begin; + } + + std::int64_t block_excess(std::size_t begin, std::size_t delta_count) const { + std::int64_t excess = 0; + for (std::size_t i = 0; i < delta_count; ++i) { + excess += bit(begin + i) ? 1 : -1; + } + return excess; + } + + std::int64_t block_base_depth(std::size_t block, + const std::array& bits, + std::size_t size) const { + const ExcessMin128Result full_min = + excess_min_128(bits.data(), 0, size - 1); + return block_min_values_[block] - full_min.min_excess; + } + + Candidate scan_block_range(std::size_t block, + std::size_t left_offset, + std::size_t right_offset) const { + const std::size_t begin = block * BlockSize; + const std::size_t size = block_size(block); + right_offset = std::min(right_offset, size - 1); + const auto bits = block_bits(block); + const std::int64_t base_depth = block_base_depth(block, bits, size); + const ExcessMin128Result result = + excess_min_128(bits.data(), left_offset, right_offset); + if (result.offset == npos || result.offset >= size) { + return {}; + } + return {begin + result.offset, base_depth + result.min_excess}; + } + + Candidate scan_full_block(std::size_t block) const { + const std::size_t begin = block * BlockSize; + const std::size_t size = block_size(block); + const auto bits = block_bits(block); + const ExcessMin128Result result = excess_min_128(bits.data(), 0, size - 1); + if (result.offset == npos || result.offset >= size) { + return {}; + } + return {begin + result.offset, block_min_values_[block]}; + } + + void reset_macro_rmq() { + macro_rmq_ = SparseTable, Index>( + std::span(block_min_values_)); + } + + std::span input_bits_; + std::size_t depth_count_ = 0; + std::vector block_min_values_; + SparseTable, Index> macro_rmq_; +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/cartesian_tree_rmq.h b/include/pixie/rmq/cartesian_tree_rmq.h new file mode 100644 index 0000000..db7d315 --- /dev/null +++ b/include/pixie/rmq/cartesian_tree_rmq.h @@ -0,0 +1,223 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief General RMQ via Cartesian-tree reduction to ±1 RMQ. + * + * @details Builds a stable min Cartesian tree over the indexed values, takes + * its Euler tour, and answers original RMQ queries as LCA queries implemented + * by RMQ over the Euler depth sequence. The indexed values are not owned and + * must outlive this object. + * + * @tparam T Value type in the indexed array. + * @tparam Compare Strict weak ordering used to choose minima. + * @tparam Index Unsigned integer type used for stored positions. + */ +template , class Index = std::size_t> +class CartesianTreeRmq + : public RmqBase, T> { + public: + static constexpr std::size_t npos = + RmqBase, T>::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + + CartesianTreeRmq() = default; + + explicit CartesianTreeRmq(std::span values, + Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + CartesianTreeRmq(const CartesianTreeRmq& other) + : values_(other.values_), + compare_(other.compare_), + left_child_(other.left_child_), + right_child_(other.right_child_), + first_occurrence_(other.first_occurrence_), + euler_nodes_(other.euler_nodes_), + depths_(other.depths_), + euler_delta_bits_(other.euler_delta_bits_) { + reset_depth_rmq(); + } + + CartesianTreeRmq& operator=(const CartesianTreeRmq& other) { + if (this == &other) { + return *this; + } + values_ = other.values_; + compare_ = other.compare_; + left_child_ = other.left_child_; + right_child_ = other.right_child_; + first_occurrence_ = other.first_occurrence_; + euler_nodes_ = other.euler_nodes_; + depths_ = other.depths_; + euler_delta_bits_ = other.euler_delta_bits_; + reset_depth_rmq(); + return *this; + } + + CartesianTreeRmq(CartesianTreeRmq&& other) noexcept + : values_(other.values_), + compare_(std::move(other.compare_)), + left_child_(std::move(other.left_child_)), + right_child_(std::move(other.right_child_)), + first_occurrence_(std::move(other.first_occurrence_)), + euler_nodes_(std::move(other.euler_nodes_)), + depths_(std::move(other.depths_)), + euler_delta_bits_(std::move(other.euler_delta_bits_)) { + reset_depth_rmq(); + } + + CartesianTreeRmq& operator=(CartesianTreeRmq&& other) noexcept { + if (this == &other) { + return *this; + } + values_ = other.values_; + compare_ = std::move(other.compare_); + left_child_ = std::move(other.left_child_); + right_child_ = std::move(other.right_child_); + first_occurrence_ = std::move(other.first_occurrence_); + euler_nodes_ = std::move(other.euler_nodes_); + depths_ = std::move(other.depths_); + euler_delta_bits_ = std::move(other.euler_delta_bits_); + reset_depth_rmq(); + return *this; + } + + std::size_t size_impl() const { return values_.size(); } + + T value_at_impl(std::size_t position) const { return values_[position]; } + + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left > right || right >= values_.size()) { + return npos; + } + std::size_t first = first_occurrence_[left]; + std::size_t second = first_occurrence_[right]; + if (first > second) { + std::swap(first, second); + } + const std::size_t euler_position = depth_rmq_.arg_min(first, second); + if (euler_position == npos) { + return npos; + } + return euler_nodes_[euler_position]; + } + + std::span euler_nodes() const { return euler_nodes_; } + + std::span euler_depths() const { return depths_; } + + private: + void build() { + left_child_.clear(); + right_child_.clear(); + first_occurrence_.clear(); + euler_nodes_.clear(); + depths_.clear(); + euler_delta_bits_.clear(); + depth_rmq_ = BpPlusMinusOneRmq(); + + if (values_.empty()) { + return; + } + if (values_.size() > static_cast(invalid_index)) { + throw std::length_error("Cartesian RMQ index type is too small"); + } + + left_child_.assign(values_.size(), invalid_index); + right_child_.assign(values_.size(), invalid_index); + + const std::size_t root = build_cartesian_tree(); + first_occurrence_.assign(values_.size(), invalid_index); + euler_nodes_.reserve(2 * values_.size() - 1); + depths_.reserve(2 * values_.size() - 1); + euler_tour(root, 0); + build_euler_delta_bits(); + reset_depth_rmq(); + } + + std::size_t build_cartesian_tree() { + std::vector stack; + stack.reserve(values_.size()); + + for (std::size_t i = 0; i < values_.size(); ++i) { + Index last = invalid_index; + while (!stack.empty() && compare_(values_[i], values_[stack.back()])) { + last = stack.back(); + stack.pop_back(); + } + if (last != invalid_index) { + left_child_[i] = last; + } + if (!stack.empty()) { + right_child_[stack.back()] = static_cast(i); + } + stack.push_back(static_cast(i)); + } + + return stack.front(); + } + + void euler_tour(std::size_t node, std::int64_t depth) { + append_euler(node, depth); + if (left_child_[node] != invalid_index) { + euler_tour(left_child_[node], depth + 1); + append_euler(node, depth); + } + if (right_child_[node] != invalid_index) { + euler_tour(right_child_[node], depth + 1); + append_euler(node, depth); + } + } + + void append_euler(std::size_t node, std::int64_t depth) { + if (first_occurrence_[node] == invalid_index) { + first_occurrence_[node] = static_cast(euler_nodes_.size()); + } + euler_nodes_.push_back(static_cast(node)); + depths_.push_back(depth); + } + + void reset_depth_rmq() { + depth_rmq_ = BpPlusMinusOneRmq( + std::span(euler_delta_bits_), depths_.size()); + } + + void build_euler_delta_bits() { + euler_delta_bits_.assign((depths_.size() - 1 + 63) / 64, 0); + for (std::size_t i = 1; i < depths_.size(); ++i) { + const std::int64_t delta = depths_[i] - depths_[i - 1]; + if (delta == 1) { + euler_delta_bits_[(i - 1) >> 6] |= std::uint64_t{1} << ((i - 1) & 63); + } + } + } + + std::span values_; + Compare compare_; + std::vector left_child_; + std::vector right_child_; + std::vector first_occurrence_; + std::vector euler_nodes_; + std::vector depths_; + std::vector euler_delta_bits_; + BpPlusMinusOneRmq depth_rmq_; +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/rmq_base.h b/include/pixie/rmq/rmq_base.h new file mode 100644 index 0000000..cc919da --- /dev/null +++ b/include/pixie/rmq/rmq_base.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +namespace pixie::rmq { + +/** + * @brief CRTP facade for static range-minimum-query indexes. + * + * Implementations are non-owning indexes over an external random-access array. + * Queries use inclusive zero-based ranges and return the first position + * attaining the minimum. Invalid ranges return `npos`. + */ +template +class RmqBase { + public: + /** + * @brief Sentinel returned when no valid query answer exists. + */ + static constexpr std::size_t npos = std::numeric_limits::max(); + + /** + * @brief Number of indexed values. + */ + std::size_t size() const { return impl().size_impl(); } + + /** + * @brief Whether the indexed array is empty. + */ + bool empty() const { return size() == 0; } + + /** + * @brief Return the first minimum position in [@p left, @p right]. + */ + std::size_t arg_min(std::size_t left, std::size_t right) const { + return impl().arg_min_impl(left, right); + } + + /** + * @brief Return the minimum value in [@p left, @p right]. + * @details Invalid ranges return a default-constructed value. + */ + Value range_min(std::size_t left, std::size_t right) const { + const std::size_t position = arg_min(left, right); + if (position == npos) { + return Value{}; + } + return impl().value_at_impl(position); + } + + private: + const Impl& impl() const { return static_cast(*this); } +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/segment_tree.h b/include/pixie/rmq/segment_tree.h new file mode 100644 index 0000000..5e734a8 --- /dev/null +++ b/include/pixie/rmq/segment_tree.h @@ -0,0 +1,116 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief Static iterative segment-tree RMQ baseline. + * + * @details Stores the index of the first minimum for each segment in a flat + * binary tree. Query time is O(log n), build time is O(n), and storage is O(n) + * indices. The indexed values are not owned and must outlive this object. + * + * @tparam T Value type in the indexed array. + * @tparam Compare Strict weak ordering used to choose minima. + * @tparam Index Unsigned integer type used for stored positions. + */ +template , class Index = std::size_t> +class SegmentTree : public RmqBase, T> { + public: + static constexpr std::size_t npos = + RmqBase, T>::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + + SegmentTree() = default; + + explicit SegmentTree(std::span values, Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + std::size_t size_impl() const { return values_.size(); } + + T value_at_impl(std::size_t position) const { return values_[position]; } + + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left > right || right >= values_.size()) { + return npos; + } + + left += leaf_base_; + right += leaf_base_; + std::size_t answer = npos; + while (left <= right) { + if ((left & 1u) != 0) { + answer = better(answer, tree_[left]); + ++left; + } + if ((right & 1u) == 0) { + answer = better(answer, tree_[right]); + if (right == 0) { + break; + } + --right; + } + left >>= 1; + right >>= 1; + } + return answer; + } + + private: + std::size_t better(std::size_t left, std::size_t right) const { + if (left == npos || left == invalid_index) { + return right; + } + if (right == npos || right == invalid_index) { + return left; + } + if (compare_(values_[right], values_[left])) { + return right; + } + if (compare_(values_[left], values_[right])) { + return left; + } + return std::min(left, right); + } + + void build() { + tree_.clear(); + leaf_base_ = 0; + if (values_.empty()) { + return; + } + if (values_.size() > static_cast(invalid_index)) { + throw std::length_error("RMQ segment tree index type is too small"); + } + + leaf_base_ = std::bit_ceil(values_.size()); + tree_.assign(2 * leaf_base_, invalid_index); + for (std::size_t i = 0; i < values_.size(); ++i) { + tree_[leaf_base_ + i] = static_cast(i); + } + for (std::size_t node = leaf_base_; node > 1;) { + --node; + tree_[node] = + static_cast(better(tree_[node << 1], tree_[(node << 1) | 1])); + } + } + + std::span values_; + Compare compare_; + std::size_t leaf_base_ = 0; + std::vector tree_; +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/sparse_table.h b/include/pixie/rmq/sparse_table.h new file mode 100644 index 0000000..d2ea6ab --- /dev/null +++ b/include/pixie/rmq/sparse_table.h @@ -0,0 +1,104 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief Static sparse-table RMQ baseline. + * + * @details Stores the index of the first minimum for each power-of-two range. + * Query time is O(1), build time is O(n log n), and storage is O(n log n) + * indices. The indexed values are not owned and must outlive this object. + * + * @tparam T Value type in the indexed array. + * @tparam Compare Strict weak ordering used to choose minima. + * @tparam Index Unsigned integer type used for stored positions. + */ +template , class Index = std::size_t> +class SparseTable : public RmqBase, T> { + public: + static constexpr std::size_t npos = + RmqBase, T>::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + + SparseTable() = default; + + explicit SparseTable(std::span values, Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + std::size_t size_impl() const { return values_.size(); } + + T value_at_impl(std::size_t position) const { return values_[position]; } + + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left > right || right >= values_.size()) { + return npos; + } + const std::size_t length = right - left + 1; + const std::size_t level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << level; + const std::size_t first = table_[level][left]; + const std::size_t second = table_[level][right + 1 - span]; + return better(first, second); + } + + private: + std::size_t better(std::size_t left, std::size_t right) const { + if (left == npos) { + return right; + } + if (right == npos) { + return left; + } + if (compare_(values_[right], values_[left])) { + return right; + } + if (compare_(values_[left], values_[right])) { + return left; + } + return std::min(left, right); + } + + void build() { + table_.clear(); + if (values_.empty()) { + return; + } + if (values_.size() > static_cast(invalid_index)) { + throw std::length_error("RMQ sparse table index type is too small"); + } + + table_.emplace_back(values_.size()); + for (std::size_t i = 0; i < values_.size(); ++i) { + table_[0][i] = static_cast(i); + } + + for (std::size_t span = 2, half = 1; span <= values_.size(); + half = span, span <<= 1) { + const std::size_t level = table_.size(); + table_.emplace_back(values_.size() - span + 1); + for (std::size_t i = 0; i < table_[level].size(); ++i) { + table_[level][i] = static_cast( + better(table_[level - 1][i], table_[level - 1][i + half])); + } + } + } + + std::span values_; + Compare compare_; + std::vector> table_; +}; + +} // namespace pixie::rmq diff --git a/src/benchmarks/bench_rmq.cpp b/src/benchmarks/bench_rmq.cpp new file mode 100644 index 0000000..4067e7e --- /dev/null +++ b/src/benchmarks/bench_rmq.cpp @@ -0,0 +1,188 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr std::uint64_t kSeed = 42; +constexpr std::size_t kQueryCount = 32768; +using Index = std::size_t; + +struct Dataset { + std::size_t size = 0; + std::size_t max_width = 0; + std::vector values; + std::vector> ranges; +}; + +struct DepthDataset { + std::size_t size = 0; + std::size_t max_width = 0; + std::vector depths; + std::vector bits; + std::vector> ranges; +}; + +Dataset make_dataset(std::size_t size, std::size_t max_width) { + Dataset dataset; + dataset.size = size; + dataset.max_width = max_width; + dataset.values.resize(size); + dataset.ranges.resize(kQueryCount); + + std::mt19937_64 rng(kSeed ^ (size * 0x9E3779B185EBCA87ull) ^ + (max_width * 0xBF58476D1CE4E5B9ull)); + std::uniform_int_distribution value_dist(-1'000'000, 1'000'000); + std::generate(dataset.values.begin(), dataset.values.end(), + [&] { return value_dist(rng); }); + + std::uniform_int_distribution left_dist(0, size - 1); + for (auto& [left, right] : dataset.ranges) { + left = left_dist(rng); + const std::size_t available = size - left; + const std::size_t width_limit = std::min(max_width, available); + std::uniform_int_distribution width_dist(1, width_limit); + right = left + width_dist(rng) - 1; + } + return dataset; +} + +DepthDataset make_depth_dataset(std::size_t size, std::size_t max_width) { + DepthDataset dataset; + dataset.size = size; + dataset.max_width = max_width; + dataset.depths.resize(size); + dataset.ranges.resize(kQueryCount); + + std::mt19937_64 rng(kSeed ^ (size * 0xD6E8FEB86659FD93ull) ^ + (max_width * 0xA5A3564E27F88695ull)); + for (std::size_t i = 1; i < dataset.depths.size(); ++i) { + dataset.depths[i] = dataset.depths[i - 1] + ((rng() & 1u) ? 1 : -1); + } + dataset.bits.assign((size - 1 + 63) / 64, 0); + for (std::size_t i = 1; i < dataset.depths.size(); ++i) { + if (dataset.depths[i] - dataset.depths[i - 1] == 1) { + dataset.bits[(i - 1) >> 6] |= std::uint64_t{1} << ((i - 1) & 63); + } + } + + std::uniform_int_distribution left_dist(0, size - 1); + for (auto& [left, right] : dataset.ranges) { + left = left_dist(rng); + const std::size_t available = size - left; + const std::size_t width_limit = std::min(max_width, available); + std::uniform_int_distribution width_dist(1, width_limit); + right = left + width_dist(rng) - 1; + } + return dataset; +} + +template +void run_queries(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const std::size_t max_width = static_cast(state.range(1)); + const Dataset dataset = make_dataset(size, max_width); + const Rmq rmq(std::span(dataset.values)); + + std::size_t query_index = 0; + for (auto _ : state) { + const auto [left, right] = + dataset.ranges[query_index++ % dataset.ranges.size()]; + std::size_t result = rmq.arg_min(left, right); + benchmark::DoNotOptimize(result); + } + + state.counters["N"] = static_cast(size); + state.counters["max_width"] = static_cast(max_width); + state.counters["index_bytes"] = static_cast(sizeof(Index)); +} + +template +void run_depth_queries(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const std::size_t max_width = static_cast(state.range(1)); + const DepthDataset dataset = make_depth_dataset(size, max_width); + const Rmq rmq(std::span(dataset.bits), + dataset.depths.size()); + + std::size_t query_index = 0; + for (auto _ : state) { + const auto [left, right] = + dataset.ranges[query_index++ % dataset.ranges.size()]; + std::size_t result = rmq.arg_min(left, right); + benchmark::DoNotOptimize(result); + } + + state.counters["N"] = static_cast(size); + state.counters["max_width"] = static_cast(max_width); + state.counters["index_bytes"] = static_cast(sizeof(Index)); +} + +void register_benchmarks() { + const std::vector sizes = {1ull << 10, 1ull << 14, 1ull << 18, + 1ull << 22, 1ull << 26}; + const std::vector widths = {64, 4096, 1ull << 18, 1ull << 22, + 1ull << 26}; + + for (const std::size_t size : sizes) { + std::vector effective_widths; + for (const std::size_t width : widths) { + if (width > size) { + continue; + } + effective_widths.push_back(width); + } + effective_widths.push_back(size); + std::sort(effective_widths.begin(), effective_widths.end()); + effective_widths.erase( + std::unique(effective_widths.begin(), effective_widths.end()), + effective_widths.end()); + + for (const std::size_t width : effective_widths) { + benchmark::RegisterBenchmark( + "rmq_sparse_table", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_segment_tree", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_cartesian_tree", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_bp_plus_minus_one", + &run_depth_queries>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + } + } +} + +} // namespace + +int main(int argc, char** argv) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + benchmark::Initialize(&argc, argv); + register_benchmarks(); + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + return 0; +} diff --git a/src/tests/excess_positions_tests.cpp b/src/tests/excess_positions_tests.cpp index 57d69c8..d37a7d3 100644 --- a/src/tests/excess_positions_tests.cpp +++ b/src/tests/excess_positions_tests.cpp @@ -8,6 +8,7 @@ #include #include #include +#include using pixie::experimental::excess_positions_512_branching_lut; using pixie::experimental::excess_positions_512_byte_lut; @@ -68,6 +69,41 @@ static int naive_prefix_excess_128(const uint64_t* s, size_t end_offset) { return cur; } +static ExcessMin128Result naive_excess_min_128(const uint64_t* s, + size_t left, + size_t right) { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int cur = 0; + int best = 0; + size_t best_offset = 0; + bool found = false; + if (left == 0) { + found = true; + } + for (size_t bit = 0; bit < right; ++bit) { + cur += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (offset < left) { + continue; + } + if (!found || cur < best) { + best = cur; + best_offset = offset; + found = true; + } + } + if (!found) { + best = naive_prefix_excess_128(s, left); + best_offset = left; + } + return {best, best_offset}; +} + static size_t naive_forward_search_128(const uint64_t* s, int target_x, size_t start_offset) { @@ -170,6 +206,84 @@ TEST(ExcessPositions128, PrefixExcessMatchesNaive) { } } +TEST(ExcessPositions128, MinMatchesNaiveFixedCases) { + const std::array, 5> cases = {{ + {0, 0}, + {UINT64_MAX, UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull}, + {0x0000FFFF0000FFFFull, 0xFFFF0000FFFF0000ull}, + }}; + const std::array, 12> ranges = {{ + {0, 128}, + {0, 0}, + {1, 1}, + {63, 65}, + {64, 64}, + {64, 128}, + {3, 6}, + {5, 5}, + {127, 128}, + {128, 128}, + {120, 127}, + {129, 140}, + }}; + + for (const auto& s : cases) { + for (const auto [left, right] : ranges) { + const ExcessMin128Result result = excess_min_128(s.data(), left, right); + const ExcessMin128Result expected = + naive_excess_min_128(s.data(), left, right); + EXPECT_EQ(result.min_excess, expected.min_excess) + << "left=" << left << " right=" << right; + EXPECT_EQ(result.offset, expected.offset) + << "left=" << left << " right=" << right; + } + } +} + +TEST(ExcessPositions128, MinReturnsFirstTie) { + const std::array s = {0x5555555555555555ull, + 0x5555555555555555ull}; + const ExcessMin128Result result = excess_min_128(s.data(), 0, 128); + EXPECT_EQ(result.min_excess, 0); + EXPECT_EQ(result.offset, 0u); + + const ExcessMin128Result shifted = excess_min_128(s.data(), 1, 128); + EXPECT_EQ(shifted.min_excess, 0); + EXPECT_EQ(shifted.offset, 2u); +} + +TEST(ExcessPositions128, MinInvalidRangeUsesSentinel) { + const std::array s = {0, 0}; + const ExcessMin128Result result = excess_min_128(s.data(), 17, 16); + EXPECT_EQ(result.min_excess, 0); + EXPECT_EQ(result.offset, 128u); +} + +TEST(ExcessPositions128, MinMatchesNaiveRandom) { + std::mt19937_64 rng(43); + std::uniform_int_distribution offset_dist(0, 128); + + for (int t = 0; t < 1000; ++t) { + const std::array s = {rng(), rng()}; + for (int q = 0; q < 32; ++q) { + size_t left = offset_dist(rng); + size_t right = offset_dist(rng); + if (left > right) { + std::swap(left, right); + } + const ExcessMin128Result result = excess_min_128(s.data(), left, right); + const ExcessMin128Result expected = + naive_excess_min_128(s.data(), left, right); + ASSERT_EQ(result.min_excess, expected.min_excess) + << "case=" << t << " left=" << left << " right=" << right; + ASSERT_EQ(result.offset, expected.offset) + << "case=" << t << " left=" << left << " right=" << right; + } + } +} + TEST(ExcessPositions128, ForwardAndBackwardSearchMatchNaive) { std::mt19937_64 rng(42); const std::array offsets = {0, 1, 63, 64, 65, 126, 127, 128}; diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp new file mode 100644 index 0000000..e727353 --- /dev/null +++ b/src/tests/rmq_tests.cpp @@ -0,0 +1,308 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +template +std::size_t naive_arg_min(std::span values, + std::size_t left, + std::size_t right, + Compare compare) { + if (left > right || right >= values.size()) { + return pixie::rmq::SparseTable::npos; + } + std::size_t best = left; + for (std::size_t i = left + 1; i <= right; ++i) { + if (compare(values[i], values[best])) { + best = i; + } + } + return best; +} + +template +void check_all_ranges(const Rmq& rmq, + std::span values, + Compare compare) { + ASSERT_EQ(rmq.size(), values.size()); + for (std::size_t left = 0; left < values.size(); ++left) { + for (std::size_t right = left; right < values.size(); ++right) { + const std::size_t expected = naive_arg_min(values, left, right, compare); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << "]"; + EXPECT_EQ(rmq.range_min(left, right), values[expected]) + << "range=[" << left << "," << right << "]"; + } + } +} + +template +void check_all_arg_min_ranges(const Rmq& rmq, + std::span values, + Compare compare) { + ASSERT_EQ(rmq.size(), values.size()); + for (std::size_t left = 0; left < values.size(); ++left) { + for (std::size_t right = left; right < values.size(); ++right) { + const std::size_t expected = naive_arg_min(values, left, right, compare); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << "]"; + } + } +} + +std::vector pack_depth_deltas( + std::span depths) { + std::vector bits((depths.size() - 1 + 63) / 64, 0); + for (std::size_t i = 1; i < depths.size(); ++i) { + if (depths[i] - depths[i - 1] == 1) { + bits[(i - 1) >> 6] |= std::uint64_t{1} << ((i - 1) & 63); + } + } + return bits; +} + +} // namespace + +TEST(RmqSparseTable, ExhaustiveSmallArray) { + const std::vector values = {4, 1, 3, 1, 5, 0, 0, 2}; + const pixie::rmq::SparseTable rmq(values); + check_all_ranges(rmq, std::span(values), std::less()); +} + +TEST(RmqSegmentTree, ExhaustiveSmallArray) { + const std::vector values = {4, 1, 3, 1, 5, 0, 0, 2}; + const pixie::rmq::SegmentTree rmq(values); + check_all_ranges(rmq, std::span(values), std::less()); +} + +TEST(RmqBpPlusMinusOne, ExhaustiveSmallDepthArray) { + const std::vector depths = {0, 1, 0, 1, 2, 1, 2, 1, 0}; + const std::vector bits = pack_depth_deltas(depths); + const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); + check_all_arg_min_ranges(rmq, std::span(depths), + std::less()); +} + +TEST(RmqBpPlusMinusOne, CrossBlockRanges) { + std::vector depths(385); + for (std::size_t i = 1; i < depths.size(); ++i) { + const bool up = (i % 7 == 0) || (i % 7 == 1) || (i % 7 == 4); + depths[i] = depths[i - 1] + (up ? 1 : -1); + } + + const std::vector bits = pack_depth_deltas(depths); + const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); + check_all_arg_min_ranges(rmq, std::span(depths), + std::less()); +} + +TEST(RmqBpPlusMinusOne, BoundaryRangesAround128PositionBlocks) { + std::vector depths(260); + for (std::size_t i = 1; i < depths.size(); ++i) { + depths[i] = depths[i - 1] + ((i % 5 == 0 || i % 11 == 0) ? 1 : -1); + } + + const std::vector bits = pack_depth_deltas(depths); + const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); + const std::vector> ranges = { + {0, 0}, {126, 127}, {127, 128}, {128, 128}, + {128, 255}, {129, 255}, {255, 256}, {0, 259}, + }; + + for (const auto [left, right] : ranges) { + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(depths), left, right, + std::less())) + << "range=[" << left << "," << right << "]"; + } +} + +TEST(RmqBpPlusMinusOne, LongSequenceRangesNearEnd) { + std::vector depths(8193); + for (std::size_t i = 1; i < depths.size(); ++i) { + const bool up = (i % 13 == 0) || (i % 17 == 0) || (i % 19 == 0); + depths[i] = depths[i - 1] + (up ? 1 : -1); + } + + const std::vector bits = pack_depth_deltas(depths); + const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); + const std::vector> ranges = { + {depths.size() - 1, depths.size() - 1}, + {depths.size() - 128, depths.size() - 1}, + {depths.size() - 513, depths.size() - 3}, + {depths.size() - 4097, depths.size() - 7}, + }; + + for (const auto [left, right] : ranges) { + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(depths), left, right, + std::less())) + << "range=[" << left << "," << right << "]"; + } +} + +TEST(RmqBpPlusMinusOne, CrossBlockTieKeepsFirstPosition) { + std::vector depths(384); + for (std::size_t i = 1; i < depths.size(); ++i) { + depths[i] = (i % 2 == 0) ? 0 : 1; + } + + const std::vector bits = pack_depth_deltas(depths); + const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); + const std::size_t left = 120; + const std::size_t right = 260; + + EXPECT_EQ(rmq.arg_min(left, right), left); + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(depths), left, right, + std::less())); +} + +TEST(RmqBpPlusMinusOne, RejectsTooSmallBitSpan) { + const std::vector bits; + EXPECT_THROW((pixie::rmq::BpPlusMinusOneRmq<>(bits, 2)), + std::invalid_argument); +} + +TEST(RmqCartesianTree, ExhaustiveSmallArray) { + const std::vector values = {4, 1, 3, 1, 5, 0, 0, 2}; + const pixie::rmq::CartesianTreeRmq rmq(values); + check_all_ranges(rmq, std::span(values), std::less()); +} + +TEST(Rmq, FirstMinimumTieBreaking) { + const std::vector values = {7, 2, 2, 3, 2}; + const pixie::rmq::SparseTable sparse(values); + const pixie::rmq::SegmentTree segment(values); + const pixie::rmq::CartesianTreeRmq cartesian(values); + + EXPECT_EQ(sparse.arg_min(0, 4), 1u); + EXPECT_EQ(segment.arg_min(0, 4), 1u); + EXPECT_EQ(cartesian.arg_min(0, 4), 1u); + EXPECT_EQ(sparse.arg_min(2, 4), 2u); + EXPECT_EQ(segment.arg_min(2, 4), 2u); + EXPECT_EQ(cartesian.arg_min(2, 4), 2u); +} + +TEST(Rmq, InvalidAndEmptyRanges) { + const std::vector values = {3, 1, 2}; + const pixie::rmq::SparseTable sparse(values); + const pixie::rmq::SegmentTree segment(values); + const pixie::rmq::CartesianTreeRmq cartesian(values); + + EXPECT_EQ(sparse.arg_min(2, 1), pixie::rmq::SparseTable::npos); + EXPECT_EQ(segment.arg_min(2, 1), pixie::rmq::SegmentTree::npos); + EXPECT_EQ(cartesian.arg_min(2, 1), pixie::rmq::CartesianTreeRmq::npos); + EXPECT_EQ(sparse.arg_min(0, values.size()), + pixie::rmq::SparseTable::npos); + EXPECT_EQ(segment.arg_min(0, values.size()), + pixie::rmq::SegmentTree::npos); + EXPECT_EQ(cartesian.arg_min(0, values.size()), + pixie::rmq::CartesianTreeRmq::npos); + EXPECT_EQ(sparse.range_min(2, 1), 0); + EXPECT_EQ(segment.range_min(2, 1), 0); + EXPECT_EQ(cartesian.range_min(2, 1), 0); + + const std::vector empty; + const pixie::rmq::SparseTable empty_sparse(empty); + const pixie::rmq::SegmentTree empty_segment(empty); + const pixie::rmq::CartesianTreeRmq empty_cartesian(empty); + EXPECT_TRUE(empty_sparse.empty()); + EXPECT_TRUE(empty_segment.empty()); + EXPECT_TRUE(empty_cartesian.empty()); + EXPECT_EQ(empty_sparse.arg_min(0, 0), pixie::rmq::SparseTable::npos); + EXPECT_EQ(empty_segment.arg_min(0, 0), pixie::rmq::SegmentTree::npos); + EXPECT_EQ(empty_cartesian.arg_min(0, 0), + pixie::rmq::CartesianTreeRmq::npos); +} + +TEST(Rmq, ComparatorCanSelectMaximum) { + const std::vector values = {1, 8, 3, 8, 4}; + const pixie::rmq::SparseTable> sparse(values); + const pixie::rmq::SegmentTree> segment(values); + const pixie::rmq::CartesianTreeRmq> cartesian(values); + + check_all_ranges(sparse, std::span(values), std::greater()); + check_all_ranges(segment, std::span(values), std::greater()); + check_all_ranges(cartesian, std::span(values), + std::greater()); + EXPECT_EQ(sparse.arg_min(0, 4), 1u); + EXPECT_EQ(segment.arg_min(0, 4), 1u); + EXPECT_EQ(cartesian.arg_min(0, 4), 1u); +} + +TEST(RmqCartesianTree, MonotoneArrays) { + const std::vector increasing = {1, 2, 3, 4, 5, 6}; + const std::vector decreasing = {6, 5, 4, 3, 2, 1}; + const pixie::rmq::CartesianTreeRmq increasing_rmq(increasing); + const pixie::rmq::CartesianTreeRmq decreasing_rmq(decreasing); + + check_all_ranges(increasing_rmq, std::span(increasing), + std::less()); + check_all_ranges(decreasing_rmq, std::span(decreasing), + std::less()); +} + +TEST(RmqCartesianTree, CopyAndMoveRebuildInternalSpans) { + const std::vector values = {5, 4, 3, 2, 1, 2, 3}; + const pixie::rmq::CartesianTreeRmq original(values); + pixie::rmq::CartesianTreeRmq copied(original); + pixie::rmq::CartesianTreeRmq assigned; + assigned = copied; + pixie::rmq::CartesianTreeRmq moved(std::move(copied)); + + check_all_ranges(original, std::span(values), std::less()); + check_all_ranges(assigned, std::span(values), std::less()); + check_all_ranges(moved, std::span(values), std::less()); +} + +TEST(RmqCartesianTree, EulerDepthsArePlusMinusOne) { + const std::vector values = {4, 1, 3, 2, 5}; + const pixie::rmq::CartesianTreeRmq rmq(values); + const auto depths = rmq.euler_depths(); + + ASSERT_FALSE(depths.empty()); + for (std::size_t i = 1; i < depths.size(); ++i) { + EXPECT_EQ(std::abs(depths[i] - depths[i - 1]), 1); + } +} + +TEST(Rmq, DifferentialRandom) { + std::mt19937_64 rng(42); + std::uniform_int_distribution value_dist(-50, 50); + for (std::size_t size = 1; size <= 257; size += 17) { + std::vector values(size); + std::generate(values.begin(), values.end(), + [&] { return value_dist(rng); }); + + const pixie::rmq::SparseTable sparse(values); + const pixie::rmq::SegmentTree segment(values); + const pixie::rmq::CartesianTreeRmq cartesian(values); + check_all_ranges(sparse, std::span(values), std::less()); + check_all_ranges(segment, std::span(values), std::less()); + check_all_ranges(cartesian, std::span(values), std::less()); + } +} + +TEST(RmqBpPlusMinusOne, DifferentialRandomWalks) { + std::mt19937_64 rng(77); + for (std::size_t size = 1; size <= 257; size += 17) { + std::vector depths(size); + for (std::size_t i = 1; i < depths.size(); ++i) { + depths[i] = depths[i - 1] + ((rng() & 1u) != 0 ? 1 : -1); + } + + const std::vector bits = pack_depth_deltas(depths); + const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); + check_all_arg_min_ranges(rmq, std::span(depths), + std::less()); + } +} From 213f0f7217309c148c7e630f43fb9ef884266fa1 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 30 May 2026 20:37:55 +0300 Subject: [PATCH 02/27] optimization skill --- .../skills/optimization-experiment/SKILL.md | 193 ++++++++++++++++++ .../optimization-experiment/EXAMPLES.md | 86 ++++++++ 2 files changed, 279 insertions(+) create mode 100644 agentic/cpp/skills/optimization-experiment/SKILL.md create mode 100644 agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md diff --git a/agentic/cpp/skills/optimization-experiment/SKILL.md b/agentic/cpp/skills/optimization-experiment/SKILL.md new file mode 100644 index 0000000..75904f6 --- /dev/null +++ b/agentic/cpp/skills/optimization-experiment/SKILL.md @@ -0,0 +1,193 @@ +--- +name: optimization-experiment +description: Run iterative C++ optimization experiments for a target function or class by adding same-API experimental variants, validating correctness, benchmarking, comparing results, and deciding whether to promote a faster implementation. +--- + +# Optimization Experiment Skill + +Use this skill when a user wants to improve performance of a specific C++ +function, class, algorithm, or hot path through benchmark-driven experiments. + +This workflow depends on: + +1. `../benchmarks/SKILL.md` for Google Benchmark build/run commands, JSON output, + hardware counters, pinning, and perf profiling. +2. `../benchmarks-affected/SKILL.md` when changes need an affected benchmark + scope. +3. `../benchmarks-compare-revisions/SKILL.md` when comparing committed + revisions. + +## Goal + +Iterate from a production implementation to one or more experimental +implementations, prove semantic equivalence, measure the impact, and decide +whether a candidate is worth promoting. + +The standard loop is: + +```text +target -> benchmark baseline -> experimental same-API variant + -> correctness check -> benchmark compare -> keep / revise / discard +``` + +Stop when a candidate is clearly better on the intended workload without +correctness or maintenance regressions, or when the remaining ideas are too weak +to justify more iteration. + +## Step 1 - Identify the Target and Contract + +Start from the requested function/class and inspect the real implementation. +Record: + +- public signature/API and call sites that must not change +- input domains, invalid-input behavior, boundary conditions, and tie-breaking +- compile-time feature gates such as SIMD flags or platform-specific paths +- existing tests and reference implementations +- existing benchmarks that should move if the optimization succeeds + +Do not optimize before the contract is clear. If behavior is ambiguous, add or +find tests before changing implementation. + +## Step 2 - Establish Benchmark Coverage + +Find benchmark rows that directly exercise the target. Prefer narrow benchmark +filters over full-suite runs during iteration. + +If coverage is missing or too broad, add focused benchmark cases before adding +the optimized implementation. Include cases for: + +- the expected common path +- boundary and alignment-sensitive paths +- short, medium, and long ranges or sizes when width matters +- random or mixed workloads when real calls are not fixed-shape +- current production behavior and each experimental variant + +Capture a baseline JSON before implementation changes: + +```bash +BENCH_CPU=${BENCH_CPU:-0} +taskset -c "${BENCH_CPU}" \ + --benchmark_filter="${FILTER}" \ + --benchmark_report_aggregates_only=true \ + --benchmark_display_aggregates_only=true \ + --benchmark_out=/tmp/_baseline.json \ + --benchmark_out_format=json +``` + +Use `../benchmarks/SKILL.md` for exact build directories, Release versus +diagnostic builds, hardware-counter setup, and retry policy. + +## Step 3 - Add Experimental Same-API Variants + +Add candidate implementations beside production code in an experimental area, +namespace, header, or benchmark-local adapter that is already consistent with +the repository. + +Rules: + +- keep the callable signature/API identical to production where practical +- preserve public semantics exactly, including invalid inputs and tie-breaking +- keep production callers unchanged during experiments +- make variants benchmark-selectable by name +- avoid unrelated refactors while measuring +- keep losing variants only when they document useful evidence or support future + comparison + +For C++ libraries with feature-gated implementations, provide correct fallbacks +for unsupported targets or compile configurations. + +## Step 4 - Validate Correctness Before Timing + +Run relevant tests before trusting benchmark numbers. Add tests when the +experimental implementation introduces new risk. + +Prefer: + +- fixed edge cases for boundaries, empty/sentinel behavior, and exact ties +- randomized differential tests against a scalar or naive reference +- tests for feature-gated fallback builds when the code has SIMD or platform + branches +- targeted regression tests for any bug found during benchmarking + +Do not compare performance for a candidate that has not passed the correctness +checks for the same semantics as production. + +## Step 5 - Benchmark and Compare + +Run timing benchmarks from Release builds. Save JSON for every meaningful +baseline and candidate. + +Use diagnostic builds with hardware counters when timing changes need +explanation: + +- cycles and instructions for core execution cost +- cache counters for memory behavior +- branch counters when early exits or dispatch logic are involved + +Compare both absolute timings and relative deltas. Watch for cases where a +candidate wins the cherry-picked row but regresses neighboring or realistic +workloads. + +When results are noisy: + +- pin to a CPU with `taskset` when available +- increase repetitions or minimum benchmark time +- rerun the narrow benchmark filter once +- avoid changing benchmark scope between baseline and candidate + +## Step 6 - Iterate Deliberately + +For each candidate, decide one of: + +- **Promote**: repeatedly faster on intended rows, no important regressions, + correct and maintainable. +- **Keep experimental**: interesting or workload-specific, but not production + ready. +- **Discard**: slower, too complex, too narrow, or semantically risky. + +Use benchmark data to choose the next idea. Examples: + +- higher instruction count suggests fewer operations or simpler dispatch +- lower instructions but higher cycles suggests stalls, memory, or dependency + chains +- short-range regressions suggest a narrower dispatch condition +- alignment-sensitive rows suggest splitting aligned and unaligned paths + +When no idea wins convincingly, document the best result and stop rather than +overfitting. + +## Step 7 - Finalize the Result + +If promoting a candidate to production: + +- keep the public API unchanged unless the user explicitly requested otherwise +- keep or update tests that protect the optimized behavior +- remove accidental benchmark-only scaffolding from production code +- preserve experimental variants only when useful for future research + +If leaving work experimental: + +- add a short note near the experimental code with benchmark date, command, and + the relevant table or JSON artifact path +- clearly state that production callers do not use the experimental variant +- explain which workload the variant helps and where it loses + +The final response should include: + +- what changed +- correctness checks run +- benchmark command or JSON artifacts +- concise result table +- recommendation: promote, keep experimenting, or stop + +## Guardrails + +1. Benchmark before optimizing; otherwise there is no trustworthy baseline. +2. Never change semantics to win a benchmark. +3. Never compare Debug timings. +4. Keep production and experimental code paths distinguishable. +5. Prefer focused benchmark filters during iteration, then broaden before + promotion. +6. Treat hardware counters as explanatory data, not a replacement for timing. +7. Record enough benchmark context that future agents do not confuse + experimental wins with production behavior. diff --git a/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md b/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md new file mode 100644 index 0000000..91aa68f --- /dev/null +++ b/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md @@ -0,0 +1,86 @@ +# Pixie Optimization Experiment Examples + +Use these notes with `agentic/cpp/skills/optimization-experiment/SKILL.md` for +Pixie-specific optimization work. + +## Example: `excess_min_128` + +The `excess_min_128` experiment is a good template for future hot-path work: + +1. **Lock semantics first**: preserve inclusive prefix range `[left, right]`, + prefix offsets `[0, 128]`, first-min tie breaking, and invalid-range sentinel + behavior. +2. **Add focused benchmarks before trusting results**: include full-range, + RMQ-style `[0, 127]`, aligned short ranges, non-aligned short ranges, + cross-word ranges, point ranges, and a reproducible random-range benchmark. +3. **Keep candidates same-API**: experimental variants should accept the same + parameters and return the same result type as production, so benchmarks can + swap implementations without adapter logic. +4. **Use experimental space for ideas**: keep exploratory variants in + `include/pixie/experimental/` unless they are clearly production-ready. +5. **Promote narrowly**: when a variant wins only under a specific condition, + promote only that condition. For `excess_min_128`, byte-LUT was useful only + for byte-aligned short ranges, so the production fallback was narrowed instead + of applied to every short range. +6. **Mark unlikely cold dispatches**: if a promoted optimization is for a narrow + special case, mark the branch unlikely when that matches expected workload. +7. **Record benchmark evidence near experimental code**: add a top-of-header + benchmark note in `/** ... */` form when future agents might confuse + experimental winners with production behavior. + +## Benchmark Pattern + +Prefer a diagnostic run that can explain timing changes, not just report them: + +```bash +taskset -c 0 build/benchmarks-diagnostic_local/excess_positions_benchmarks \ + --benchmark_filter='BM_ExcessMin128(_|/|$)' \ + --benchmark_repetitions=3 \ + --benchmark_perf_counters=CYCLES,INSTRUCTIONS,CACHE-MISSES \ + --benchmark_counters_tabular=true \ + --benchmark_out=/tmp/pixie_excess_min_counters.json \ + --benchmark_out_format=json +``` + +For final comparison tables, include at least: + +- CPU time +- cycles +- instructions +- cache misses when collected +- a random or mixed workload row +- the rows that justified any narrowed production dispatch condition + +## Documentation Pattern + +For experimental algorithms, use Doxygen block comments: + +```cpp +/** + * @brief Short algorithm name and purpose. + * + * @details Workflow: + * + * input -> transform -> candidate result -> final reduction + * + * Explain the non-obvious tradeoff, such as avoiding lane crossing, reducing + * loop iterations, or paying scalar boundary work. + */ +``` + +Keep long benchmark tables as a top-level `/** ... */` note when they describe +the whole experimental header rather than one symbol. + +## Non-Obvious Lessons From This Session + +- A faster micro-kernel can regress neighboring ranges if dispatch is too broad; + benchmark aligned and non-aligned cases separately. +- Random-range benchmarks are useful as a sanity check against overfitting fixed + rows. +- Hardware counters helped distinguish lower-level cost: cycles and instructions + were more actionable than cache misses for the `excess_min_128` candidates. +- Production and experimental code must stay visually distinct. A benchmark win + in `pixie::experimental` is only a candidate, not a caller-visible change. +- When result names encode width unnecessarily, prefer the stable semantic name + (`ExcessResult`) over width-specific detail (`ExcessResult128`) unless the API + truly needs multiple widths. From f7083f17340fd85f7d5ded2a3abc74e2327c3f93 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 30 May 2026 22:49:26 +0300 Subject: [PATCH 03/27] Experiments on excess min --- CMakeLists.txt | 17 + include/pixie/bits.h | 266 +++++-- include/pixie/experimental/excess.h | 724 ++++++++++++++++++ include/pixie/rmq/bp_plus_minus_one_rmq.h | 7 +- .../excess_positions_benchmarks.cpp | 274 ++++++- .../excess_positions_benchmark_results.md | 9 + src/tests/excess_positions_tests.cpp | 159 +++- 7 files changed, 1378 insertions(+), 78 deletions(-) create mode 100644 src/docs/excess_positions_benchmark_results.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 87c59ad..4bc2e0d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,6 +183,15 @@ if(PIXIE_TESTS) gtest gtest_main ${PIXIE_DIAGNOSTICS_LIBS}) + + add_executable(rmq_tests + src/tests/rmq_tests.cpp) + target_include_directories(rmq_tests + PUBLIC include) + target_link_libraries(rmq_tests + gtest + gtest_main + ${PIXIE_DIAGNOSTICS_LIBS}) endif() # --------------------------------------------------------------------------- @@ -218,6 +227,14 @@ if(PIXIE_BENCHMARKS) benchmark ${PIXIE_DIAGNOSTICS_LIBS}) + add_executable(bench_rmq + src/benchmarks/bench_rmq.cpp) + target_include_directories(bench_rmq + PUBLIC include) + target_link_libraries(bench_rmq + benchmark + ${PIXIE_DIAGNOSTICS_LIBS}) + if(PIXIE_THIRD_PARTY_BACKENDS) add_executable(bench_rmm_sdsl src/benchmarks/bench_rmm_sdsl.cpp) diff --git a/include/pixie/bits.h b/include/pixie/bits.h index f1250eb..b103600 100644 --- a/include/pixie/bits.h +++ b/include/pixie/bits.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -798,15 +799,48 @@ static inline const __m256i excess_lut_pos2 = _mm256_setr_epi8( -1, 1, 1, 3, -3, -1, -1, 1, -1, 1, 1, 3); +static inline const __m256i excess_lut_min = _mm256_setr_epi8( + -4, -2, -2, 0, + -2, 0, -1, 1, + -3, -1, -1, 1, + -2, 0, -1, 1, + -4, -2, -2, 0, + -2, 0, -1, 1, + -3, -1, -1, 1, + -2, 0, -1, 1); +static inline constexpr int8_t excess_lut_min_offset[16] = { + 4, 4, 4, 4, 2, 2, 1, 1, 3, 3, 1, 1, 2, 2, 1, 1}; static inline const __m256i excess_lut_pack_multiplier = _mm256_set1_epi16(0x1001); static inline const __m256i excess_lut_bit0 = _mm256_set1_epi8(1); static inline const __m256i excess_lut_bit1 = _mm256_set1_epi8(2); static inline const __m256i excess_lut_bit2 = _mm256_set1_epi8(4); static inline const __m256i excess_lut_bit3 = _mm256_set1_epi8(8); +static inline const __m256i excess_lut_nibble_index = _mm256_setr_epi8( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, + 28, 29, 30, 31); static inline const __m128i excess_lut_nibble_mask = _mm_set1_epi8(0x0F); // clang-format on +static inline __m256i excess_nibbles_128_avx2(const uint64_t* s) noexcept { + __m128i word_vec = _mm_loadu_si128(reinterpret_cast(s)); + __m128i lo_nibbles = _mm_and_si128(word_vec, excess_lut_nibble_mask); + __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(word_vec, 4), excess_lut_nibble_mask); + + __m128i unpack_lo = _mm_unpacklo_epi8(lo_nibbles, hi_nibbles); + __m128i unpack_hi = _mm_unpackhi_epi8(lo_nibbles, hi_nibbles); + + return _mm256_inserti128_si256(_mm256_castsi128_si256(unpack_lo), unpack_hi, + 1); +} + static inline __m256i excess_bit_masks_16x_i16() noexcept { return _mm256_setr_epi16(0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, 0x0200, 0x0400, 0x0800, @@ -841,11 +875,59 @@ static inline __m256i excess_prefix_sum_16x_i16(__m256i v) noexcept { * offset attaining the minimum. Invalid ranges return offset 128 as a * sentinel. */ -struct ExcessMin128Result { +struct ExcessResult { int min_excess = 0; size_t offset = 128; }; +constexpr int8_t excess_byte_delta_value(uint8_t x) { + return static_cast(2 * std::popcount(x) - 8); +} + +constexpr int8_t excess_byte_min_prefix_value(uint8_t x) { + int cur = 0; + int best = 0; + for (int bit = 0; bit < 8; ++bit) { + cur += ((x >> bit) & 1u) != 0 ? 1 : -1; + if (bit == 0 || cur < best) { + best = cur; + } + } + return static_cast(best); +} + +constexpr int8_t excess_byte_min_prefix_offset_value(uint8_t x) { + int cur = 0; + int best = 0; + int best_offset = 1; + for (int bit = 0; bit < 8; ++bit) { + cur += ((x >> bit) & 1u) != 0 ? 1 : -1; + if (bit == 0 || cur < best) { + best = cur; + best_offset = bit + 1; + } + } + return static_cast(best_offset); +} + +template +constexpr std::array excess_make_byte_lut(Fn fn) { + std::array out{}; + for (size_t i = 0; i < out.size(); ++i) { + out[i] = fn(static_cast(i)); + } + return out; +} + +static inline constexpr std::array excess_byte_delta_lut = + excess_make_byte_lut([](uint8_t x) { return excess_byte_delta_value(x); }); +static inline constexpr std::array excess_byte_min_lut = + excess_make_byte_lut( + [](uint8_t x) { return excess_byte_min_prefix_value(x); }); +static inline constexpr std::array excess_byte_min_offset_lut = + excess_make_byte_lut( + [](uint8_t x) { return excess_byte_min_prefix_offset_value(x); }); + /** * @brief Find every prefix whose excess equals target_x in a 128-bit bitstring. * @@ -879,22 +961,13 @@ static inline int excess_positions_128(const uint64_t* s, const __m256i vbit1 = excess_lut_bit1; const __m256i vbit2 = excess_lut_bit2; const __m256i vbit3 = excess_lut_bit3; - const __m128i vnibble_mask = excess_lut_nibble_mask; const int d = 2 * target_x - block_delta; if (d < -128 || d > 128) { return block_delta; } - __m128i word_vec = _mm_loadu_si128((const __m128i*)s); - __m128i lo_nibbles = _mm_and_si128(word_vec, vnibble_mask); - __m128i hi_nibbles = _mm_and_si128(_mm_srli_epi16(word_vec, 4), vnibble_mask); - - __m128i unpack_lo = _mm_unpacklo_epi8(lo_nibbles, hi_nibbles); - __m128i unpack_hi = _mm_unpackhi_epi8(lo_nibbles, hi_nibbles); - - __m256i nibbles = - _mm256_inserti128_si256(_mm256_castsi128_si256(unpack_lo), unpack_hi, 1); + __m256i nibbles = excess_nibbles_128_avx2(s); __m256i ps = _mm256_shuffle_epi8(vdelta, nibbles); ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 1)); @@ -974,15 +1047,59 @@ static inline int prefix_excess_128(const uint64_t* s, return 2 * ones - static_cast(end_offset); } +static inline ExcessResult excess_min_128_byte_lut_short( + const uint64_t* s, + size_t left, + size_t right) noexcept { + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + size_t bit = left; + for (; bit < right && (bit & 7u) != 0; ++bit) { + current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } + + for (; bit + 8 <= right; bit += 8) { + const uint8_t byte = + static_cast((s[bit >> 6] >> (bit & 63)) & 0xFFu); + const int candidate = current + excess_byte_min_lut[byte]; + if (candidate < best) { + best = candidate; + best_offset = bit + static_cast(excess_byte_min_offset_lut[byte]); + } + current += excess_byte_delta_lut[byte]; + } + + for (; bit < right; ++bit) { + current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } + + return {best, best_offset}; +} + /** * @brief Return the minimum prefix excess and first attaining offset. * @param s 2 little-endian uint64_t words (bit 0 of s[0] is the first bit). * @param left First prefix position to consider, inclusive. * @param right Last prefix position to consider, inclusive. */ -static inline ExcessMin128Result excess_min_128(const uint64_t* s, - size_t left, - size_t right) noexcept { +static inline ExcessResult excess_min_128(const uint64_t* s, + size_t left, + size_t right) noexcept { if (left > right) { return {}; } @@ -994,46 +1111,93 @@ static inline ExcessMin128Result excess_min_128(const uint64_t* s, if (left == right) { return {best, best_offset}; } + if (right - left <= 32 && (left & 7u) == 0 && (right & 7u) == 0) + [[unlikely]] { + return excess_min_128_byte_lut_short(s, left, right); + } #ifdef PIXIE_AVX2_SUPPORT - static const __m256i masks = excess_bit_masks_16x_i16(); - static const __m256i zero = _mm256_setzero_si256(); - static const __m256i pos = _mm256_set1_epi16(1); - static const __m256i neg = _mm256_set1_epi16(-1); - - int carry = 0; - alignas(32) int16_t prefix_values[16]; - for (size_t chunk = 0; chunk < 8; ++chunk) { - const size_t chunk_bit = chunk * 16; - const uint16_t bits = - chunk < 4 - ? static_cast((s[0] >> (chunk * 16)) & 0xFFFFu) - : static_cast((s[1] >> ((chunk - 4) * 16)) & 0xFFFFu); - const int delta = 2 * static_cast(std::popcount(bits)) - 16; - - if (chunk_bit + 1 <= right && chunk_bit + 16 >= left) { - const __m256i selected = _mm256_and_si256( - _mm256_set1_epi16(static_cast(bits)), masks); - const __m256i is_zero = _mm256_cmpeq_epi16(selected, zero); - const __m256i steps = _mm256_blendv_epi8(pos, neg, is_zero); - const __m256i pref = - _mm256_add_epi16(excess_prefix_sum_16x_i16(steps), - _mm256_set1_epi16(static_cast(carry))); - _mm256_store_si256(reinterpret_cast<__m256i*>(prefix_values), pref); - - for (size_t lane = 0; lane < 16; ++lane) { - const size_t offset = chunk_bit + lane + 1; - if (offset < left || offset > right) { - continue; - } - const int value = prefix_values[lane]; - if (value < best) { - best = value; - best_offset = offset; - } - } + int current = best; + size_t bit = left; + for (; bit < right && (bit & 3u) != 0; ++bit) { + current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } + + const size_t first_full_nibble = bit >> 2; + const size_t last_full_nibble = right >> 2; + if (first_full_nibble < last_full_nibble) { + const __m256i nibbles = excess_nibbles_128_avx2(s); + + __m256i ps = _mm256_shuffle_epi8(excess_lut_delta, nibbles); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 1)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 2)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 4)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 8)); + + __m128i ps_lo = _mm256_castsi256_si128(ps); + __m128i ps_hi = _mm256_extracti128_si256(ps, 1); + __m128i carry = + _mm_set1_epi8(static_cast(_mm_extract_epi8(ps_lo, 15))); + ps_hi = _mm_add_epi8(ps_hi, carry); + ps = _mm256_inserti128_si256(_mm256_castsi128_si256(ps_lo), ps_hi, 1); + + __m256i b = _mm256_permute2x128_si256(ps, ps, 0x08); + const __m256i excl_ps = _mm256_alignr_epi8(ps, b, 15); + const __m256i candidates = + _mm256_add_epi8(excl_ps, _mm256_shuffle_epi8(excess_lut_min, nibbles)); + + const __m256i idx = excess_lut_nibble_index; + const int first_minus_one_value = static_cast(first_full_nibble) - 1; + const __m256i first_minus_one = + _mm256_set1_epi8(static_cast(first_minus_one_value)); + const __m256i last = + _mm256_set1_epi8(static_cast(last_full_nibble)); + const __m256i active = _mm256_and_si256( + _mm256_cmpgt_epi8(idx, first_minus_one), _mm256_cmpgt_epi8(last, idx)); + const __m256i masked_candidates = + _mm256_blendv_epi8(_mm256_set1_epi8(127), candidates, active); + + __m128i min128 = + _mm_min_epi8(_mm256_castsi256_si128(masked_candidates), + _mm256_extracti128_si256(masked_candidates, 1)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 8)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 4)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 2)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 1)); + + const int candidate_min = + static_cast(static_cast(_mm_extract_epi8(min128, 0))); + if (candidate_min < best) { + const __m256i equal_min = _mm256_cmpeq_epi8( + masked_candidates, + _mm256_set1_epi8(static_cast(candidate_min))); + const uint32_t equal_mask = + static_cast(_mm256_movemask_epi8(equal_min)); + const uint32_t nibble_index = std::countr_zero(equal_mask); + const uint64_t word = s[nibble_index >> 4]; + const uint8_t nibble = + static_cast((word >> ((nibble_index & 15u) * 4u)) & 0xFu); + best = candidate_min; + best_offset = static_cast(nibble_index) * 4u + + static_cast(excess_lut_min_offset[nibble]); + } + + bit = last_full_nibble * 4; + current = prefix_excess_128(s, bit); + } + + for (; bit < right; ++bit) { + current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; } - carry += delta; } #else int current = 0; diff --git a/include/pixie/experimental/excess.h b/include/pixie/experimental/excess.h index ef64dde..d5a0e66 100644 --- a/include/pixie/experimental/excess.h +++ b/include/pixie/experimental/excess.h @@ -2,13 +2,579 @@ #include +#include +#include #include #include #include +/** + * Benchmark note: + * + * This header keeps experimental and historical excess_min/excess_positions + * variants for comparison benchmarks. Production excess code lives in + * pixie/bits.h; a benchmark win here should be treated as a candidate to port, + * not evidence that callers use this variant already. + * + * Diagnostic run, 2026-05-30: + * taskset -c 0 build/benchmarks-diagnostic_local/excess_positions_benchmarks + * --benchmark_filter='BM_ExcessMin128(_|/|$)' + * --benchmark_repetitions=3 + * --benchmark_perf_counters=CYCLES,INSTRUCTIONS,CACHE-MISSES + * --benchmark_counters_tabular=true + * + * Production excess_min_128: + * range ns cycles instructions cache misses + * 0-128 8.1 35.8 117.9 0.001 + * 0-127 11.6 50.9 174.9 0.001 + * 0-16 3.7 16.4 88.5 0.000 + * 0-32 5.6 24.7 124.8 0.000 + * 0-48 9.1 40.5 122.1 0.000 + * 0-64 8.1 36.2 121.4 0.000 + * 0-31 12.1 54.2 177.6 0.000 + * 1-17 14.4 64.6 213.3 0.000 + * 3-35 13.5 60.1 212.4 0.000 + * 5-37 12.9 57.5 215.3 0.000 + * 32-64 7.4 33.1 153.0 0.000 + * 33-65 14.3 63.6 214.6 0.001 + * 64-96 7.8 34.2 148.9 0.000 + * 61-93 16.0 69.4 218.5 0.002 + * 96-128 6.5 28.1 151.9 0.001 + * 56-72 5.2 22.7 116.5 0.000 + * 60-68 8.5 38.1 148.7 0.000 + * 63-64 3.1 13.3 84.0 0.000 + * 17-17 2.1 9.1 47.0 0.000 + * Random 11.3 49.7 197.0 0.001 + * + * New non-aligned/random range timings, ns: + * range Prod Scalar Nibble Byte Hybrid Expand16 Lane64 Split64 Skip + * 1-17 14.4 14.6 10.1 12.5 11.9 30.3 13.3 15.9 16.3 + * 3-35 13.5 27.1 12.5 14.2 16.1 46.1 13.1 12.7 14.1 + * 5-37 12.9 27.0 14.6 15.2 18.5 44.7 13.5 13.0 14.7 + * 33-65 14.3 26.6 13.3 17.8 17.4 42.1 14.5 13.3 13.6 + * 61-93 16.0 26.6 15.4 14.8 17.7 43.8 14.0 13.6 11.4 + * Random 11.3 39.9 18.9 14.7 11.5 68.2 11.9 12.4 12.6 + * + * Random range hardware counters: + * variant ns cycles instructions + * Production 11.3 49.7 197.0 + * ScalarBits 39.9 175.9 721.9 + * NibbleLUT 18.9 83.2 291.2 + * ByteLUT 14.7 63.9 261.2 + * HybridLUT 11.5 50.8 209.8 + * Expand16 68.2 300.8 879.8 + * Lane64SSE 11.9 52.1 224.1 + * Split64SSE 12.4 54.1 236.9 + * ShortSkip 12.6 55.7 252.5 + * + * Diagnostic run, 2026-05-30: + * taskset -c 0 build/benchmarks-diagnostic_local/excess_positions_benchmarks + * --benchmark_filter='BM_ExcessPositions512' + * --benchmark_repetitions=3 + * --benchmark_perf_counters=CYCLES,INSTRUCTIONS,CACHE-MISSES + * --benchmark_counters_tabular=true + * + * excess_positions_512 random target in [-128, 128]: + * variant ns cycles instructions cache misses + * Production 11.4 50.8 188.9 0.001 + * LUTAVX512 12.8 56.8 195.5 0.002 + * BranchingLUT 16.7 73.4 261.4 0.003 + * ExpandAVX512 21.0 93.6 266.6 0.003 + * Expand8 24.6 109.9 449.7 0.002 + * Expand 46.8 207.7 784.8 0.006 + * ByteLUT 49.7 221.4 754.5 0.008 + * Scalar 374.2 1656.0 7716.6 0.041 + * + * excess_positions_512 fixed-target timings, ns: + * variant -64 -8 0 8 64 + * Production 11.6 18.0 18.3 19.1 12.3 + * LUTAVX512 13.4 17.9 19.2 21.3 13.2 + * BranchingLUT 19.1 28.9 28.7 28.3 16.8 + * ExpandAVX512 22.7 36.6 36.3 36.2 22.5 + * Expand8 17.6 52.7 47.5 46.9 17.7 + * Expand 51.0 85.9 86.1 85.5 53.9 + * ByteLUT 34.7 77.4 76.9 79.0 34.3 + * Scalar 367.2 433.5 466.3 428.1 364.6 + */ + namespace pixie::experimental { +namespace detail { + +constexpr int8_t nibble_delta(uint8_t x) { + return static_cast(2 * std::popcount(x) - 4); +} + +constexpr int8_t byte_delta(uint8_t x) { + return static_cast(2 * std::popcount(x) - 8); +} + +constexpr int8_t min_prefix(uint8_t x, int bits) { + int cur = 0; + int best = 0; + for (int bit = 0; bit < bits; ++bit) { + cur += ((x >> bit) & 1u) != 0 ? 1 : -1; + if (bit == 0 || cur < best) { + best = cur; + } + } + return static_cast(best); +} + +constexpr int8_t min_prefix_offset(uint8_t x, int bits) { + int cur = 0; + int best = 0; + int best_offset = 1; + for (int bit = 0; bit < bits; ++bit) { + cur += ((x >> bit) & 1u) != 0 ? 1 : -1; + if (bit == 0 || cur < best) { + best = cur; + best_offset = bit + 1; + } + } + return static_cast(best_offset); +} + +template +constexpr std::array make_lut(Fn fn) { + std::array out{}; + for (size_t i = 0; i < N; ++i) { + out[i] = fn(static_cast(i)); + } + return out; +} + +static inline constexpr std::array kNibbleDelta = + make_lut<16>([](uint8_t x) { return nibble_delta(x); }); +static inline constexpr std::array kNibbleMin = + make_lut<16>([](uint8_t x) { return min_prefix(x, 4); }); +static inline constexpr std::array kNibbleMinOffset = + make_lut<16>([](uint8_t x) { return min_prefix_offset(x, 4); }); +static inline constexpr std::array kByteDelta = + make_lut<256>([](uint8_t x) { return byte_delta(x); }); +static inline constexpr std::array kByteMin = + make_lut<256>([](uint8_t x) { return min_prefix(x, 8); }); +static inline constexpr std::array kByteMinOffset = + make_lut<256>([](uint8_t x) { return min_prefix_offset(x, 8); }); + +static inline void scan_bit(const uint64_t* s, + size_t bit, + int& current, + int& best, + size_t& best_offset) noexcept { + current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } +} + +} // namespace detail + +/** + * @brief Reference scalar excess_min_128 implementation. + * + * @details Workflow: + * + * prefix(left) -> scan bits left..right-1 -> first strict minimum + * + * The value at offset left is included before scanning any bits, matching the + * production inclusive prefix range [left, right]. Each scanned bit advances to + * the next prefix offset. Ties are intentionally ignored, so the first minimum + * offset is preserved. + */ +static inline ExcessResult excess_min_128_scalar_bits(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + for (size_t bit = left; bit < right; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + return {best, best_offset}; +} + +/** + * @brief Scalar 4-bit LUT excess_min_128 experiment. + * + * @details Workflow: + * + * unaligned bits -> full nibbles -> trailing bits + * | delta + * | local min + * ` first local min offset + * + * Full nibbles use lookup tables for the nibble delta, the minimum prefix value + * inside positions 1..4, and the first local bit offset that reaches that + * minimum. Boundary bits are scanned scalar so the LUT never observes bits + * outside the query range. + */ +static inline ExcessResult excess_min_128_nibble_lut(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + size_t bit = left; + for (; bit < right && (bit & 3u) != 0; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + + for (; bit + 4 <= right; bit += 4) { + const uint8_t nibble = + static_cast((s[bit >> 6] >> (bit & 63)) & 0xFu); + const int candidate = current + detail::kNibbleMin[nibble]; + if (candidate < best) { + best = candidate; + best_offset = bit + static_cast(detail::kNibbleMinOffset[nibble]); + } + current += detail::kNibbleDelta[nibble]; + } + + for (; bit < right; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + return {best, best_offset}; +} + +/** + * @brief Scalar 8-bit LUT excess_min_128 experiment. + * + * @details Workflow: + * + * unaligned bits -> full bytes -> trailing bits + * | delta + * | local min + * ` first local min offset + * + * Full bytes use lookup tables for byte delta, minimum prefix value inside + * positions 1..8, and the first local bit offset that reaches that minimum. + * This reduces loop iterations on byte-aligned ranges but pays scalar boundary + * work on unaligned ranges. + */ +static inline ExcessResult excess_min_128_byte_lut(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + size_t bit = left; + for (; bit < right && (bit & 7u) != 0; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + + for (; bit + 8 <= right; bit += 8) { + const uint8_t byte = + static_cast((s[bit >> 6] >> (bit & 63)) & 0xFFu); + const int candidate = current + detail::kByteMin[byte]; + if (candidate < best) { + best = candidate; + best_offset = bit + static_cast(detail::kByteMinOffset[byte]); + } + current += detail::kByteDelta[byte]; + } + + for (; bit < right; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + return {best, best_offset}; +} + +/** + * @brief Hybrid dispatch over scalar, byte-LUT, nibble-LUT, and production. + * + * @details Workflow: + * + * width <= 2 -> scalar bits + * width <= 64 and byte aligned -> byte LUT + * width <= 32 -> nibble LUT + * otherwise -> production excess_min_128 + * + * This variant probes whether the fastest implementation depends primarily on + * query width and boundary alignment. It keeps production behavior for wider + * ranges where the AVX2 production path usually wins. + */ +static inline ExcessResult excess_min_128_hybrid_lut(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + const size_t clamped_left = std::min(left, 128); + const size_t clamped_right = std::min(right, 128); + const size_t width = clamped_right - clamped_left; + + if (width <= 2) { + return excess_min_128_scalar_bits(s, left, right); + } + if (width <= 64 && (clamped_left & 7u) == 0 && (clamped_right & 7u) == 0) { + return excess_min_128_byte_lut(s, left, right); + } + if (width <= 32) { + return excess_min_128_nibble_lut(s, left, right); + } + return excess_min_128(s, left, right); +} + #ifdef PIXIE_AVX2_SUPPORT +// clang-format off +static inline const __m128i excess_lut_delta_128 = _mm_setr_epi8( + -4, -2, -2, 0, + -2, 0, 0, 2, + -2, 0, 0, 2, + 0, 2, 2, 4); +static inline const __m128i excess_lut_min_128 = _mm_setr_epi8( + -4, -2, -2, 0, + -2, 0, -1, 1, + -3, -1, -1, 1, + -2, 0, -1, 1); +static inline const __m128i excess_lut_nibble_index_128 = _mm_setr_epi8( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15); +static inline const __m128i excess_lut_nibble_mask_128 = _mm_set1_epi8(0x0F); +// clang-format on + +namespace detail { + +static inline __m128i excess_nibbles_64_sse(uint64_t word) noexcept { + const __m128i word_vec = _mm_cvtsi64_si128(static_cast(word)); + const __m128i lo_nibbles = + _mm_and_si128(word_vec, excess_lut_nibble_mask_128); + const __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(word_vec, 4), excess_lut_nibble_mask_128); + return _mm_unpacklo_epi8(lo_nibbles, hi_nibbles); +} + +static inline __m128i excess_prefix_sum_16x_i8(__m128i v) noexcept { + __m128i x = v; + __m128i t = _mm_slli_si128(x, 1); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 2); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 4); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 8); + return _mm_add_epi8(x, t); +} + +static inline void scan_full_nibbles_64_sse(uint64_t word, + int lane_base_excess, + size_t lane_base_offset, + size_t first_nibble, + size_t last_nibble, + int& best, + size_t& best_offset) noexcept { + if (first_nibble >= last_nibble) { + return; + } + + const __m128i nibbles = excess_nibbles_64_sse(word); + __m128i ps = + excess_prefix_sum_16x_i8(_mm_shuffle_epi8(excess_lut_delta_128, nibbles)); + const __m128i excl_ps = _mm_alignr_epi8(ps, _mm_setzero_si128(), 15); + const __m128i candidates = _mm_add_epi8( + _mm_add_epi8(_mm_set1_epi8(static_cast(lane_base_excess)), + excl_ps), + _mm_shuffle_epi8(excess_lut_min_128, nibbles)); + + const __m128i idx = excess_lut_nibble_index_128; + const __m128i first_minus_one = + _mm_set1_epi8(static_cast(static_cast(first_nibble) - 1)); + const __m128i last = _mm_set1_epi8(static_cast(last_nibble)); + const __m128i active = _mm_and_si128(_mm_cmpgt_epi8(idx, first_minus_one), + _mm_cmpgt_epi8(last, idx)); + const __m128i masked_candidates = + _mm_blendv_epi8(_mm_set1_epi8(127), candidates, active); + + __m128i min128 = masked_candidates; + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 8)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 4)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 2)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 1)); + + const int candidate_min = + static_cast(static_cast(_mm_extract_epi8(min128, 0))); + if (candidate_min < best) { + const __m128i equal_min = _mm_cmpeq_epi8( + masked_candidates, _mm_set1_epi8(static_cast(candidate_min))); + const uint32_t equal_mask = + static_cast(_mm_movemask_epi8(equal_min)); + const uint32_t nibble_index = std::countr_zero(equal_mask); + const uint8_t nibble = + static_cast((word >> (nibble_index * 4u)) & 0xFu); + best = candidate_min; + best_offset = lane_base_offset + static_cast(nibble_index) * 4u + + static_cast(kNibbleMinOffset[nibble]); + } +} + +static inline ExcessResult excess_min_128_split64_sse_impl( + const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + size_t bit = left; + for (; bit < right && (bit & 3u) != 0; ++bit) { + scan_bit(s, bit, current, best, best_offset); + } + + size_t first_full_nibble = bit >> 2; + const size_t last_full_nibble = right >> 2; + while (first_full_nibble < last_full_nibble) { + const size_t word_index = first_full_nibble >> 4; + const size_t lane_first = first_full_nibble & 15u; + const size_t lane_last = + std::min(last_full_nibble - word_index * 16u, 16); + const size_t lane_base_offset = word_index * 64u; + scan_full_nibbles_64_sse( + s[word_index], prefix_excess_128(s, lane_base_offset), lane_base_offset, + lane_first, lane_last, best, best_offset); + first_full_nibble = word_index * 16u + lane_last; + } + + bit = std::max(bit, first_full_nibble * 4u); + current = prefix_excess_128(s, bit); + for (; bit < right; ++bit) { + scan_bit(s, bit, current, best, best_offset); + } + + return {best, best_offset}; +} + +} // namespace detail + +/** + * @brief Single-64-bit-lane SSE excess_min_128 experiment. + * + * @details Workflow: + * + * scalar boundary -> one 64-bit word as 16 nibbles -> scalar tail + * fallback to production if full nibbles cross words + * + * This variant tests whether short ranges benefit from avoiding the 128-bit + * cross-lane prefix work used by broader vector paths. It only handles the + * single-word full-nibble case; multi-word ranges fall back to production. + */ +static inline ExcessResult excess_min_128_lane64_sse(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + const size_t clamped_left = std::min(left, 128); + const size_t clamped_right = std::min(right, 128); + const size_t first_full_nibble = ((clamped_left + 3u) & ~size_t{3}) >> 2; + const size_t last_full_nibble = clamped_right >> 2; + if (first_full_nibble < last_full_nibble && + (first_full_nibble >> 4) != ((last_full_nibble - 1u) >> 4)) { + return excess_min_128(s, left, right); + } + return detail::excess_min_128_split64_sse_impl(s, left, right); +} + +/** + * @brief Split-64-bit-lane SSE excess_min_128 experiment. + * + * @details Workflow: + * + * scalar boundary -> word 0 full nibbles -> word 1 full nibbles -> tail + * 16-nibble SSE 16-nibble SSE + * + * Each 64-bit word is processed as an independent 16-nibble vector scan. The + * base excess for a word is recomputed from prefix_excess_128, avoiding + * vector-prefix carry propagation across the 64-bit boundary. + */ +static inline ExcessResult excess_min_128_split64_sse(const uint64_t* s, + size_t left, + size_t right) noexcept { + return detail::excess_min_128_split64_sse_impl(s, left, right); +} + +/** + * @brief Short-range dispatch experiment. + * + * @details Workflow: + * + * width <= 2 -> scalar bits + * full nibbles in 1 word -> lane64 SSE + * width <= 80 -> split64 SSE + * otherwise -> production excess_min_128 + * + * This variant tests two ideas together: avoid 128-bit lane crossing when a + * query is contained in one 64-bit word, and skip a few production iterations + * for medium ranges where split-lane scans may be cheaper. + */ +static inline ExcessResult excess_min_128_short_skip(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + const size_t clamped_left = std::min(left, 128); + const size_t clamped_right = std::min(right, 128); + const size_t width = clamped_right - clamped_left; + if (width <= 2) { + return excess_min_128_scalar_bits(s, left, right); + } + + const size_t first_full_nibble = ((clamped_left + 3u) & ~size_t{3}) >> 2; + const size_t last_full_nibble = clamped_right >> 2; + if (first_full_nibble < last_full_nibble && + (first_full_nibble >> 4) == ((last_full_nibble - 1u) >> 4)) { + return excess_min_128_lane64_sse(s, left, right); + } + if (width <= 80) { + return excess_min_128_split64_sse(s, left, right); + } + return excess_min_128(s, left, right); +} + // clang-format off static inline const __m256i excess_branch_lut_em4 = _mm256_setr_epi8( 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -133,6 +699,88 @@ static inline int8_t excess_last_prefix_32x_i8(__m256i pref) noexcept { return (int8_t)_mm_extract_epi8(hi, 15); } +/** + * @brief AVX2 expand-to-i16 excess_min_128 experiment. + * + * @details Workflow: + * + * 16 input bits -> 16 x i16 +/-1 -> vector prefix sum -> store -> scalar min + * + * The implementation scans eight 16-bit chunks. For chunks overlapping the + * query, bits are expanded to signed +/-1 i16 lanes and prefix summed with the + * running carry. The vector result is stored to memory, then relevant lanes are + * checked scalar for the first strict minimum. + */ +static inline ExcessResult excess_min_128_expand16_avx2(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + const __m256i masks = excess_bit_masks_16x(); + const __m256i zero = _mm256_setzero_si256(); + const __m256i pos = _mm256_set1_epi16(1); + const __m256i neg = _mm256_set1_epi16(-1); + + int carry = 0; + alignas(32) int16_t prefix_values[16]; + for (size_t chunk = 0; chunk < 8; ++chunk) { + const size_t chunk_bit = chunk * 16; + const uint16_t bits = + chunk < 4 + ? static_cast((s[0] >> (chunk * 16)) & 0xFFFFu) + : static_cast((s[1] >> ((chunk - 4) * 16)) & 0xFFFFu); + const int delta = 2 * static_cast(std::popcount(bits)) - 16; + + if (chunk_bit + 1 <= right && chunk_bit + 16 >= left) { + const __m256i selected = _mm256_and_si256( + _mm256_set1_epi16(static_cast(bits)), masks); + const __m256i is_zero = _mm256_cmpeq_epi16(selected, zero); + const __m256i steps = _mm256_blendv_epi8(pos, neg, is_zero); + const __m256i pref = + _mm256_add_epi16(excess_prefix_sum_16x_i16(steps), + _mm256_set1_epi16(static_cast(carry))); + _mm256_store_si256(reinterpret_cast<__m256i*>(prefix_values), pref); + + for (size_t lane = 0; lane < 16; ++lane) { + const size_t offset = chunk_bit + lane + 1; + if (offset < left || offset > right) { + continue; + } + const int value = prefix_values[lane]; + if (value < best) { + best = value; + best_offset = offset; + } + } + } + carry += delta; + } + + return {best, best_offset}; +} + +/** + * @brief Historical AVX2 branching-LUT excess_positions_512 variant. + * + * @details Workflow: + * + * 128-bit block -> 32 nibbles -> prefix sums -> branch by target-relative + * -> LUT masks -> packed output bits + * + * Each 128-bit block is converted to nibbles, prefix-summed, and filtered by + * reachability. A family of per-target LUTs produces within-nibble match masks, + * which are packed back to the output words. + */ static inline void excess_positions_512_branching_lut(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -249,6 +897,13 @@ static inline void excess_positions_512_branching_lut(const uint64_t* s, } } #else +/** + * @brief Scalar fallback for the branching-LUT positions variant. + * + * @details Used when AVX2 is not enabled. Delegates to production + * excess_positions_512 so callers can benchmark the same symbol across build + * configurations. + */ static inline void excess_positions_512_branching_lut(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -378,6 +1033,19 @@ static inline uint64_t excess_repeat_byte(int value) noexcept { static_cast(static_cast(value)); } +/** + * @brief AVX-512 nibble-LUT excess_positions_512 experiment. + * + * @details Workflow: + * + * 256 input bits -> 64 nibbles -> two 128-bit logical halves + * -> nibble prefix/LUT matches -> packed output masks + * + * The implementation processes four words at a time. It computes reachability + * for each 128-bit half, converts bytes to nibbles, builds exclusive prefix + * sums, compares nibble-local positions against the target, and packs matches + * back to four output words. + */ static inline void excess_positions_512_lut_avx512(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -463,6 +1131,13 @@ static inline void excess_positions_512_lut_avx512(const uint64_t* s, } } #else +/** + * @brief Fallback for the AVX-512 LUT positions variant. + * + * @details Used when AVX-512 is not enabled. Delegates to production + * excess_positions_512 so callers can benchmark the same symbol across build + * configurations. + */ static inline void excess_positions_512_lut_avx512(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -470,6 +1145,18 @@ static inline void excess_positions_512_lut_avx512(const uint64_t* s, } #endif +/** + * @brief Expand-to-i16 excess_positions_512 experiment. + * + * @details Workflow: + * + * 16 input bits -> 16 x i16 +/-1 -> vector prefix sum -> compare target + * -> pext mask -> output word + * + * With AVX2, this variant scans 16-bit chunks, expands them to i16 prefix + * lanes, compares absolute prefix values against the target, and compresses the + * comparison mask into output bits. Without AVX2 it uses a scalar scan. + */ static inline void excess_positions_512_expand(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -541,6 +1228,18 @@ static inline void excess_positions_512_expand(const uint64_t* s, #endif } +/** + * @brief AVX2 expand-to-i8 excess_positions_512 experiment. + * + * @details Workflow: + * + * 32 input bits -> 32 x i8 +/-1 -> byte prefix sum -> compare target + * -> movemask -> output word + * + * This variant handles 32-bit chunks. It first checks whether the target is + * reachable within the chunk, then expands bits to byte lanes and uses the + * vector comparison mask directly as output bits. + */ static inline void excess_positions_512_expand8(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -601,6 +1300,18 @@ static inline void excess_positions_512_expand8(const uint64_t* s, #endif } +/** + * @brief AVX-512 expand-to-i8 excess_positions_512 experiment. + * + * @details Workflow: + * + * 64 input bits -> 64 x i8 +/-1 -> byte prefix sum -> k-mask output + * + * This variant processes one 64-bit word per vector. If the target is + * unreachable in the word, it advances by popcount delta only. Otherwise it + * expands the word to byte lanes, prefix-sums the lanes, compares against the + * target, and stores the resulting AVX-512 mask as the output word. + */ static inline void excess_positions_512_expand_avx512(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -681,6 +1392,19 @@ struct ExcessByteLut { inline constexpr ExcessByteLut kExcessByteLut; +/** + * @brief Scalar byte-LUT excess_positions_512 experiment. + * + * @details Workflow: + * + * byte -> relative target in [-8, 8] -> LUT match mask + * -> byte delta -> next byte base excess + * + * The table stores, for each byte, all bit positions that reach each local + * target and the byte delta. The scan walks 64 bytes, emits a mask when the + * relative target is in range, and then advances the running excess by the + * byte delta. + */ static inline void excess_positions_512_byte_lut(const uint64_t* s, int target_x, uint64_t* out) noexcept { diff --git a/include/pixie/rmq/bp_plus_minus_one_rmq.h b/include/pixie/rmq/bp_plus_minus_one_rmq.h index 57af0e1..e796b93 100644 --- a/include/pixie/rmq/bp_plus_minus_one_rmq.h +++ b/include/pixie/rmq/bp_plus_minus_one_rmq.h @@ -212,8 +212,7 @@ class BpPlusMinusOneRmq { std::int64_t block_base_depth(std::size_t block, const std::array& bits, std::size_t size) const { - const ExcessMin128Result full_min = - excess_min_128(bits.data(), 0, size - 1); + const ExcessResult full_min = excess_min_128(bits.data(), 0, size - 1); return block_min_values_[block] - full_min.min_excess; } @@ -225,7 +224,7 @@ class BpPlusMinusOneRmq { right_offset = std::min(right_offset, size - 1); const auto bits = block_bits(block); const std::int64_t base_depth = block_base_depth(block, bits, size); - const ExcessMin128Result result = + const ExcessResult result = excess_min_128(bits.data(), left_offset, right_offset); if (result.offset == npos || result.offset >= size) { return {}; @@ -237,7 +236,7 @@ class BpPlusMinusOneRmq { const std::size_t begin = block * BlockSize; const std::size_t size = block_size(block); const auto bits = block_bits(block); - const ExcessMin128Result result = excess_min_128(bits.data(), 0, size - 1); + const ExcessResult result = excess_min_128(bits.data(), 0, size - 1); if (result.offset == npos || result.offset >= size) { return {}; } diff --git a/src/benchmarks/excess_positions_benchmarks.cpp b/src/benchmarks/excess_positions_benchmarks.cpp index 4bbf28f..0e0850f 100644 --- a/src/benchmarks/excess_positions_benchmarks.cpp +++ b/src/benchmarks/excess_positions_benchmarks.cpp @@ -8,12 +8,22 @@ #include #include +using pixie::experimental::excess_min_128_byte_lut; +using pixie::experimental::excess_min_128_hybrid_lut; using pixie::experimental::excess_positions_512_branching_lut; using pixie::experimental::excess_positions_512_byte_lut; using pixie::experimental::excess_positions_512_expand; using pixie::experimental::excess_positions_512_expand8; using pixie::experimental::excess_positions_512_expand_avx512; using pixie::experimental::excess_positions_512_lut_avx512; +#ifdef PIXIE_AVX2_SUPPORT +using pixie::experimental::excess_min_128_expand16_avx2; +using pixie::experimental::excess_min_128_lane64_sse; +using pixie::experimental::excess_min_128_short_skip; +using pixie::experimental::excess_min_128_split64_sse; +#endif +using pixie::experimental::excess_min_128_nibble_lut; +using pixie::experimental::excess_min_128_scalar_bits; static std::vector> make_blocks( size_t num_blocks = 4096) { @@ -27,6 +37,193 @@ static std::vector> make_blocks( return blocks; } +static std::vector> make_128_blocks( + size_t num_blocks = 4096) { + std::mt19937_64 rng(42); + std::vector> blocks(num_blocks); + for (auto& b : blocks) { + b = {rng(), rng()}; + } + return blocks; +} + +static std::vector> make_128_ranges( + size_t num_ranges = 4096) { + std::mt19937_64 rng(43); + std::uniform_int_distribution offset_dist(0, 128); + std::vector> ranges(num_ranges); + for (auto& range : ranges) { + size_t left = offset_dist(rng); + size_t right = offset_dist(rng); + if (left > right) { + std::swap(left, right); + } + range = {left, right}; + } + return ranges; +} + +static std::vector make_512_targets(size_t num_targets = 4096) { + std::mt19937 rng(44); + std::uniform_int_distribution target_dist(-128, 128); + std::vector targets(num_targets); + for (int& target : targets) { + target = target_dist(rng); + } + return targets; +} + +static void BM_ExcessMin128(benchmark::State& state) { + const size_t left = static_cast(state.range(0)); + const size_t right = static_cast(state.range(1)); + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + ExcessResult result = excess_min_128(s.data(), left, right); + benchmark::DoNotOptimize(result.min_excess); + benchmark::DoNotOptimize(result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin128) + ->ArgNames({"left", "right"}) + ->Args({0, 128}) + ->Args({0, 127}) + ->Args({0, 16}) + ->Args({0, 32}) + ->Args({0, 48}) + ->Args({0, 64}) + ->Args({0, 31}) + ->Args({1, 17}) + ->Args({3, 35}) + ->Args({5, 37}) + ->Args({32, 64}) + ->Args({33, 65}) + ->Args({64, 96}) + ->Args({61, 93}) + ->Args({96, 128}) + ->Args({56, 72}) + ->Args({60, 68}) + ->Args({63, 64}) + ->Args({17, 17}); + +template +static void BM_ExcessMin128Variant(benchmark::State& state) { + const size_t left = static_cast(state.range(0)); + const size_t right = static_cast(state.range(1)); + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + ExcessResult result = Fn(s.data(), left, right); + benchmark::DoNotOptimize(result.min_excess); + benchmark::DoNotOptimize(result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +#define PIXIE_BENCH_EXCESS_MIN_VARIANT(name, fn) \ + BENCHMARK_TEMPLATE(BM_ExcessMin128Variant, fn) \ + ->Name(name) \ + ->ArgNames({"left", "right"}) \ + ->Args({0, 128}) \ + ->Args({0, 127}) \ + ->Args({0, 16}) \ + ->Args({0, 32}) \ + ->Args({0, 48}) \ + ->Args({0, 64}) \ + ->Args({0, 31}) \ + ->Args({1, 17}) \ + ->Args({3, 35}) \ + ->Args({5, 37}) \ + ->Args({32, 64}) \ + ->Args({33, 65}) \ + ->Args({64, 96}) \ + ->Args({61, 93}) \ + ->Args({96, 128}) \ + ->Args({56, 72}) \ + ->Args({60, 68}) \ + ->Args({63, 64}) \ + ->Args({17, 17}) + +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_ScalarBits", + excess_min_128_scalar_bits); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_NibbleLUT", + excess_min_128_nibble_lut); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_ByteLUT", + excess_min_128_byte_lut); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_HybridLUT", + excess_min_128_hybrid_lut); +#ifdef PIXIE_AVX2_SUPPORT +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_Expand16AVX2", + excess_min_128_expand16_avx2); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_Lane64SSE", + excess_min_128_lane64_sse); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_Split64SSE", + excess_min_128_split64_sse); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_ShortSkip", + excess_min_128_short_skip); +#endif + +#undef PIXIE_BENCH_EXCESS_MIN_VARIANT + +template +static void BM_ExcessMin128RandomRange(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const auto ranges = make_128_ranges(); + const size_t num_blocks = blocks.size(); + const size_t num_ranges = ranges.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + const auto [left, right] = ranges[idx % num_ranges]; + ExcessResult result = Fn(s.data(), left, right); + benchmark::DoNotOptimize(result.min_excess); + benchmark::DoNotOptimize(result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +#define PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT(name, fn) \ + BENCHMARK_TEMPLATE(BM_ExcessMin128RandomRange, fn)->Name(name) + +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_RandomRange", + excess_min_128); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_ScalarBits_RandomRange", + excess_min_128_scalar_bits); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_NibbleLUT_RandomRange", + excess_min_128_nibble_lut); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_ByteLUT_RandomRange", + excess_min_128_byte_lut); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_HybridLUT_RandomRange", + excess_min_128_hybrid_lut); +#ifdef PIXIE_AVX2_SUPPORT +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT( + "BM_ExcessMin128_Expand16AVX2_RandomRange", + excess_min_128_expand16_avx2); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_Lane64SSE_RandomRange", + excess_min_128_lane64_sse); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_Split64SSE_RandomRange", + excess_min_128_split64_sse); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_ShortSkip_RandomRange", + excess_min_128_short_skip); +#endif + +#undef PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT + static void BM_ExcessPositions512(benchmark::State& state) { const int target_x = state.range(0); const auto blocks = make_blocks(); @@ -183,6 +380,22 @@ BENCHMARK(BM_ExcessPositions512_ExpandAVX512) ->Args({8}) ->Args({64}); +static void excess_positions_512_scalar_benchmark(const uint64_t* s, + int target_x, + uint64_t* out) noexcept { + for (int w = 0; w < 8; ++w) { + out[w] = 0; + } + int cur = 0; + for (size_t i = 0; i < 512; ++i) { + const int bit = int((s[i >> 6] >> (i & 63)) & 1ull); + cur += bit ? +1 : -1; + if (cur == target_x) { + out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } +} + static void BM_ExcessPositions512_Scalar(benchmark::State& state) { const int target_x = state.range(0); const auto blocks = make_blocks(); @@ -193,17 +406,7 @@ static void BM_ExcessPositions512_Scalar(benchmark::State& state) { for (auto _ : state) { const auto& s = blocks[idx % num_blocks]; - for (int w = 0; w < 8; ++w) { - out[w] = 0; - } - int cur = 0; - for (size_t i = 0; i < 512; ++i) { - const int bit = int((s[i >> 6] >> (i & 63)) & 1ull); - cur += bit ? +1 : -1; - if (cur == target_x) { - out[i >> 6] |= (uint64_t{1} << (i & 63)); - } - } + excess_positions_512_scalar_benchmark(s.data(), target_x, out); benchmark::DoNotOptimize(out); ++idx; } @@ -244,3 +447,52 @@ BENCHMARK(BM_ExcessPositions512_ByteLUT) ->Args({0}) ->Args({8}) ->Args({64}); + +template +static void BM_ExcessPositions512RandomTarget(benchmark::State& state) { + const auto blocks = make_blocks(); + const auto targets = make_512_targets(); + const size_t num_blocks = blocks.size(); + const size_t num_targets = targets.size(); + + alignas(64) uint64_t out[8]; + size_t idx = 0; + + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + Fn(s.data(), targets[idx % num_targets], out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +#define PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM(name, fn) \ + BENCHMARK_TEMPLATE(BM_ExcessPositions512RandomTarget, fn)->Name(name) + +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM("BM_ExcessPositions512_RandomTarget", + excess_positions_512); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_BranchingLUT_RandomTarget", + excess_positions_512_branching_lut); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_LUTAVX512_RandomTarget", + excess_positions_512_lut_avx512); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_Expand_RandomTarget", + excess_positions_512_expand); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_Expand8_RandomTarget", + excess_positions_512_expand8); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_ExpandAVX512_RandomTarget", + excess_positions_512_expand_avx512); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_Scalar_RandomTarget", + excess_positions_512_scalar_benchmark); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_ByteLUT_RandomTarget", + excess_positions_512_byte_lut); + +#undef PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM diff --git a/src/docs/excess_positions_benchmark_results.md b/src/docs/excess_positions_benchmark_results.md new file mode 100644 index 0000000..17adf71 --- /dev/null +++ b/src/docs/excess_positions_benchmark_results.md @@ -0,0 +1,9 @@ +| Method | X=-64 | X=-8 | X=0 | X=8 | X=64 | +|---|---:|---:|---:|---:|---:| +| BranchingLUT | 15.71 ns | 26.56 ns | 26.55 ns | 26.43 ns | 15.37 ns | +| Current | 10.73 ns | 18.09 ns | 18.58 ns | 18.24 ns | 10.43 ns | +| Expand | 60.70 ns | 88.41 ns | 87.38 ns | 88.50 ns | 56.77 ns | +| Expand8 | 19.13 ns | 53.05 ns | 47.83 ns | 49.35 ns | 17.44 ns | +| ExpandAVX512 | 23.13 ns | 38.39 ns | 38.56 ns | 39.24 ns | 23.19 ns | +| LUTAVX512 | 12.33 ns | 18.34 ns | 18.06 ns | 18.21 ns | 12.75 ns | +| Scalar | 304.42 ns | 389.58 ns | 446.94 ns | 399.80 ns | 316.23 ns | diff --git a/src/tests/excess_positions_tests.cpp b/src/tests/excess_positions_tests.cpp index d37a7d3..62cdf3a 100644 --- a/src/tests/excess_positions_tests.cpp +++ b/src/tests/excess_positions_tests.cpp @@ -10,12 +10,22 @@ #include #include +using pixie::experimental::excess_min_128_byte_lut; +using pixie::experimental::excess_min_128_hybrid_lut; using pixie::experimental::excess_positions_512_branching_lut; using pixie::experimental::excess_positions_512_byte_lut; using pixie::experimental::excess_positions_512_expand; using pixie::experimental::excess_positions_512_expand8; using pixie::experimental::excess_positions_512_expand_avx512; using pixie::experimental::excess_positions_512_lut_avx512; +#ifdef PIXIE_AVX2_SUPPORT +using pixie::experimental::excess_min_128_expand16_avx2; +using pixie::experimental::excess_min_128_lane64_sse; +using pixie::experimental::excess_min_128_short_skip; +using pixie::experimental::excess_min_128_split64_sse; +#endif +using pixie::experimental::excess_min_128_nibble_lut; +using pixie::experimental::excess_min_128_scalar_bits; static void naive_excess_positions_512(const uint64_t* s, int target_x, @@ -69,9 +79,9 @@ static int naive_prefix_excess_128(const uint64_t* s, size_t end_offset) { return cur; } -static ExcessMin128Result naive_excess_min_128(const uint64_t* s, - size_t left, - size_t right) { +static ExcessResult naive_excess_min_128(const uint64_t* s, + size_t left, + size_t right) { if (left > right) { return {}; } @@ -170,6 +180,23 @@ static void check_matches_naive(Fn fn, << fn_name << " case=" << case_id << " x=" << target_x; } +template +static void check_min_matches_naive(Fn fn, + const char* fn_name, + const uint64_t* s, + size_t left, + size_t right, + int case_id = 0) { + const ExcessResult result = fn(s, left, right); + const ExcessResult expected = naive_excess_min_128(s, left, right); + ASSERT_EQ(result.min_excess, expected.min_excess) + << fn_name << " case=" << case_id << " left=" << left + << " right=" << right; + ASSERT_EQ(result.offset, expected.offset) + << fn_name << " case=" << case_id << " left=" << left + << " right=" << right; +} + TEST(ExcessPositions128, MatchesNaiveMasksAndDelta) { const std::array, 4> cases = {{ {0, 0}, @@ -231,9 +258,8 @@ TEST(ExcessPositions128, MinMatchesNaiveFixedCases) { for (const auto& s : cases) { for (const auto [left, right] : ranges) { - const ExcessMin128Result result = excess_min_128(s.data(), left, right); - const ExcessMin128Result expected = - naive_excess_min_128(s.data(), left, right); + const ExcessResult result = excess_min_128(s.data(), left, right); + const ExcessResult expected = naive_excess_min_128(s.data(), left, right); EXPECT_EQ(result.min_excess, expected.min_excess) << "left=" << left << " right=" << right; EXPECT_EQ(result.offset, expected.offset) @@ -245,18 +271,54 @@ TEST(ExcessPositions128, MinMatchesNaiveFixedCases) { TEST(ExcessPositions128, MinReturnsFirstTie) { const std::array s = {0x5555555555555555ull, 0x5555555555555555ull}; - const ExcessMin128Result result = excess_min_128(s.data(), 0, 128); + const ExcessResult result = excess_min_128(s.data(), 0, 128); EXPECT_EQ(result.min_excess, 0); EXPECT_EQ(result.offset, 0u); - const ExcessMin128Result shifted = excess_min_128(s.data(), 1, 128); + const ExcessResult shifted = excess_min_128(s.data(), 1, 128); EXPECT_EQ(shifted.min_excess, 0); EXPECT_EQ(shifted.offset, 2u); + + const ExcessResult left_tie = excess_min_128(s.data(), 2, 128); + EXPECT_EQ(left_tie.min_excess, 0); + EXPECT_EQ(left_tie.offset, 2u); +} + +TEST(ExcessPositions128, MinHandlesRightBoundary) { + const std::array s = {0, 0}; + + const ExcessResult without_last = excess_min_128(s.data(), 0, 127); + EXPECT_EQ(without_last.min_excess, -127); + EXPECT_EQ(without_last.offset, 127u); + + const ExcessResult with_last = excess_min_128(s.data(), 0, 128); + EXPECT_EQ(with_last.min_excess, -128); + EXPECT_EQ(with_last.offset, 128u); +} + +TEST(ExcessPositions128, MinPartialNibbleBoundsExcludeOuterMin) { + const std::array s = {0, 0}; + + const ExcessResult short_prefix = excess_min_128(s.data(), 1, 2); + EXPECT_EQ(short_prefix.min_excess, -2); + EXPECT_EQ(short_prefix.offset, 2u); + + const ExcessResult short_suffix = excess_min_128(s.data(), 2, 3); + EXPECT_EQ(short_suffix.min_excess, -3); + EXPECT_EQ(short_suffix.offset, 3u); +} + +TEST(ExcessPositions128, MinPositiveRangeKeepsLeftBoundary) { + const std::array s = {UINT64_MAX, UINT64_MAX}; + + const ExcessResult result = excess_min_128(s.data(), 64, 128); + EXPECT_EQ(result.min_excess, 64); + EXPECT_EQ(result.offset, 64u); } TEST(ExcessPositions128, MinInvalidRangeUsesSentinel) { const std::array s = {0, 0}; - const ExcessMin128Result result = excess_min_128(s.data(), 17, 16); + const ExcessResult result = excess_min_128(s.data(), 17, 16); EXPECT_EQ(result.min_excess, 0); EXPECT_EQ(result.offset, 128u); } @@ -273,9 +335,8 @@ TEST(ExcessPositions128, MinMatchesNaiveRandom) { if (left > right) { std::swap(left, right); } - const ExcessMin128Result result = excess_min_128(s.data(), left, right); - const ExcessMin128Result expected = - naive_excess_min_128(s.data(), left, right); + const ExcessResult result = excess_min_128(s.data(), left, right); + const ExcessResult expected = naive_excess_min_128(s.data(), left, right); ASSERT_EQ(result.min_excess, expected.min_excess) << "case=" << t << " left=" << left << " right=" << right; ASSERT_EQ(result.offset, expected.offset) @@ -284,6 +345,80 @@ TEST(ExcessPositions128, MinMatchesNaiveRandom) { } } +TEST(ExcessPositions128Experimental, MinVariantsMatchNaive) { + const std::array, 6> cases = {{ + {0, 0}, + {UINT64_MAX, UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull}, + {0x0000FFFF0000FFFFull, 0xFFFF0000FFFF0000ull}, + {0x1111222233334444ull, 0x8888777766665555ull}, + }}; + const std::array, 25> ranges = {{ + {0, 128}, {0, 127}, {0, 16}, {0, 31}, {0, 32}, + {0, 48}, {0, 64}, {32, 64}, {64, 96}, {96, 128}, + {56, 72}, {60, 68}, {63, 64}, {17, 17}, {1, 2}, + {2, 3}, {3, 6}, {5, 5}, {64, 64}, {64, 128}, + {127, 128}, {128, 128}, {120, 127}, {129, 140}, {17, 16}, + }}; + + int case_id = 0; + for (const auto& s : cases) { + for (const auto [left, right] : ranges) { + check_min_matches_naive(excess_min_128_scalar_bits, "scalar_bits", + s.data(), left, right, case_id); + check_min_matches_naive(excess_min_128_nibble_lut, "nibble_lut", s.data(), + left, right, case_id); + check_min_matches_naive(excess_min_128_byte_lut, "byte_lut", s.data(), + left, right, case_id); + check_min_matches_naive(excess_min_128_hybrid_lut, "hybrid_lut", s.data(), + left, right, case_id); +#ifdef PIXIE_AVX2_SUPPORT + check_min_matches_naive(excess_min_128_expand16_avx2, "expand16_avx2", + s.data(), left, right, case_id); + check_min_matches_naive(excess_min_128_lane64_sse, "lane64_sse", s.data(), + left, right, case_id); + check_min_matches_naive(excess_min_128_split64_sse, "split64_sse", + s.data(), left, right, case_id); + check_min_matches_naive(excess_min_128_short_skip, "short_skip", s.data(), + left, right, case_id); +#endif + ++case_id; + } + } +} + +TEST(ExcessPositions128Experimental, MinVariantsMatchNaiveRandom) { + std::mt19937_64 rng(44); + std::uniform_int_distribution offset_dist(0, 140); + + for (int t = 0; t < 500; ++t) { + const std::array s = {rng(), rng()}; + for (int q = 0; q < 16; ++q) { + const size_t left = offset_dist(rng); + const size_t right = offset_dist(rng); + check_min_matches_naive(excess_min_128_scalar_bits, "scalar_bits", + s.data(), left, right, t); + check_min_matches_naive(excess_min_128_nibble_lut, "nibble_lut", s.data(), + left, right, t); + check_min_matches_naive(excess_min_128_byte_lut, "byte_lut", s.data(), + left, right, t); + check_min_matches_naive(excess_min_128_hybrid_lut, "hybrid_lut", s.data(), + left, right, t); +#ifdef PIXIE_AVX2_SUPPORT + check_min_matches_naive(excess_min_128_expand16_avx2, "expand16_avx2", + s.data(), left, right, t); + check_min_matches_naive(excess_min_128_lane64_sse, "lane64_sse", s.data(), + left, right, t); + check_min_matches_naive(excess_min_128_split64_sse, "split64_sse", + s.data(), left, right, t); + check_min_matches_naive(excess_min_128_short_skip, "short_skip", s.data(), + left, right, t); +#endif + } + } +} + TEST(ExcessPositions128, ForwardAndBackwardSearchMatchNaive) { std::mt19937_64 rng(42); const std::array offsets = {0, 1, 63, 64, 65, 126, 127, 128}; From 3e6f2b0f13c1be2a169a1279567e9c25c3dd94f1 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 30 May 2026 22:50:46 +0300 Subject: [PATCH 04/27] chore: remove agentic/cpp subtree for submodule migration --- agentic/cpp/README.md | 15 - agentic/cpp/commands/README.md | 6 - agentic/cpp/commands/benchmarks-affected.md | 34 - agentic/cpp/commands/perf-review.md | 149 --- agentic/cpp/commands/ping.md | 7 - .../cpp/skills/benchmarks-affected/SKILL.md | 81 -- .../analyze_benchmarks_affected.py | 1132 ----------------- .../benchmarks-compare-revisions/SKILL.md | 225 ---- agentic/cpp/skills/benchmarks/SKILL.md | 209 --- .../benchmarks/scripts/plot_benchmarks.py | 159 --- agentic/cpp/skills/cmake/SKILL.md | 96 -- agentic/cpp/skills/diagnose-segfault/SKILL.md | 195 --- .../skills/optimization-experiment/SKILL.md | 193 --- agentic/cpp/skills/paper-search/SKILL.md | 106 -- .../paper-search/references/api_reference.md | 32 - .../paper-search/scripts/search_papers.py | 333 ----- agentic/cpp/skills/pdf/SKILL.md | 112 -- agentic/cpp/skills/setup-cpp-repo/SKILL.md | 136 -- .../references/project_structure.md | 117 -- .../scripts/init_cpp_project.py | 1053 --------------- 20 files changed, 4390 deletions(-) delete mode 100644 agentic/cpp/README.md delete mode 100644 agentic/cpp/commands/README.md delete mode 100644 agentic/cpp/commands/benchmarks-affected.md delete mode 100644 agentic/cpp/commands/perf-review.md delete mode 100644 agentic/cpp/commands/ping.md delete mode 100644 agentic/cpp/skills/benchmarks-affected/SKILL.md delete mode 100755 agentic/cpp/skills/benchmarks-affected/analyze_benchmarks_affected.py delete mode 100644 agentic/cpp/skills/benchmarks-compare-revisions/SKILL.md delete mode 100644 agentic/cpp/skills/benchmarks/SKILL.md delete mode 100644 agentic/cpp/skills/benchmarks/scripts/plot_benchmarks.py delete mode 100644 agentic/cpp/skills/cmake/SKILL.md delete mode 100644 agentic/cpp/skills/diagnose-segfault/SKILL.md delete mode 100644 agentic/cpp/skills/optimization-experiment/SKILL.md delete mode 100644 agentic/cpp/skills/paper-search/SKILL.md delete mode 100644 agentic/cpp/skills/paper-search/references/api_reference.md delete mode 100755 agentic/cpp/skills/paper-search/scripts/search_papers.py delete mode 100644 agentic/cpp/skills/pdf/SKILL.md delete mode 100644 agentic/cpp/skills/setup-cpp-repo/SKILL.md delete mode 100644 agentic/cpp/skills/setup-cpp-repo/references/project_structure.md delete mode 100755 agentic/cpp/skills/setup-cpp-repo/scripts/init_cpp_project.py diff --git a/agentic/cpp/README.md b/agentic/cpp/README.md deleted file mode 100644 index 68a2262..0000000 --- a/agentic/cpp/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Shared C++ Agent Skills - -This subtree contains reusable C++ agent skills and related commands. - -Keep this tree generic: - -- Do not add project-specific benchmark names, CMake options, or paths. -- Keep reusable scripts beside the skills that use them. -- Put project-specific examples in the consuming repository under - `agentic/local/cpp/skills//EXAMPLES.md`. - -When using a skill in a project, read: - -1. `agentic/cpp/skills//SKILL.md` -2. `agentic/local/cpp/skills//EXAMPLES.md`, if present diff --git a/agentic/cpp/commands/README.md b/agentic/cpp/commands/README.md deleted file mode 100644 index 2a24b20..0000000 --- a/agentic/cpp/commands/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Shared C++ Agent Commands - -Reusable command definitions for C++ projects belong here. - -Keep project-specific commands in the consuming repository under -`agentic/local/cpp/commands`. diff --git a/agentic/cpp/commands/benchmarks-affected.md b/agentic/cpp/commands/benchmarks-affected.md deleted file mode 100644 index 158cb9e..0000000 --- a/agentic/cpp/commands/benchmarks-affected.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -description: Scan current branch and report impacted benchmark targets/functions. ---- - -# Benchmarks Affected - -Identify which benchmark binaries and benchmark functions are affected by changes on the current branch. - -Use the `benchmarks-affected` skill as the single source of truth for workflow details and guardrails. -Do not duplicate or override the skill instructions in this command. - -## Inputs - -- Optional `--baseline ` (default: `main`) -- Optional `--compile-commands ` -- Optional `--no-include-working-tree` -- Optional `--format ` (default: `text`) - -## Workflow - -1. Execute the `benchmarks-affected` skill workflow. -2. Pass through command inputs to the analyzer invocation defined by the skill. -3. Report results with these sections: - - Changed files - - Affected benchmark targets - - Affected benchmark functions - - Suggested `--benchmark_filter` regex - - Any warnings/failures - -## Output rules - -1. If `affected_benchmarks` is non-empty, prioritize those names. -2. If `affected_benchmarks` is empty but benchmark targets are affected, mark result as partial and include target-level impact. -3. Do not run full benchmark suites in this command; this command is for impact discovery only. diff --git a/agentic/cpp/commands/perf-review.md b/agentic/cpp/commands/perf-review.md deleted file mode 100644 index 8ef1865..0000000 --- a/agentic/cpp/commands/perf-review.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -description: Benchmark-driven PR performance review versus target branch ---- - -# Perf Review Workflow - -You are performing a performance review for the current PR branch. - -Non-negotiable requirements: -1. Benchmark timing plus profiling data is the highest-priority judgment tool. -2. Compare source branch versus target branch and report relevant benchmark metric changes. -3. Provide analysis and a final verdict: does the PR improve performance or not. - -## Inputs - -- Optional argument `--target `: target branch override. -- Optional argument `--filter `: benchmark filter regex. -- Optional argument `--no-counters`: disable hardware-counter collection. - -If arguments are omitted: -- Default target branch to PR base branch from `gh pr view --json baseRefName` when available. -- Fall back target branch to `main`. - -Filter handling: -- If `--filter` is provided, pass it through. -- Else use the filter produced by `benchmarks-affected` through `benchmarks-compare-revisions`. -- If no filter can be derived, run conservative full-binary compare for impacted binaries. - -## Step 1 - Resolve branches and hashes - -1. Resolve contender from current checkout (`HEAD`) and compute short hash. -2. Resolve baseline branch using precedence: `--target` -> PR base from `gh pr view --json baseRefName` -> `main`. -3. Resolve baseline short hash. -4. Print branch/hash mapping before benchmark execution. - -## Step 2 - Run timing and hardware-counter comparison via skill (single source of truth) - -Use `benchmarks-compare-revisions` as the single source of truth for revision builds, benchmark scope, compare.py flow, retry policy, and guardrails. - -Pass-through inputs: -- Baseline ref/hash from Step 1. -- Contender ref/hash from Step 1. -- Optional `--filter` override. -- Counter mode: default on (`COLLECT_COUNTERS=1`) on Linux, disabled when `--no-counters` is provided. - -Consume outputs from `benchmarks-compare-revisions`: -- Baseline and contender benchmark JSON artifacts. -- compare.py output per binary. -- Effective filter used. -- Scope metadata from `benchmarks-affected` (`affected_benchmark_targets`, `affected_benchmarks`) when available. -- `counters_available` status and, when unavailable, explicit reason. -- Baseline and contender counter JSON artifacts (when available). -- Derived counter metrics per benchmark (IPC, cache miss rate, branch mispredict rate). -- Counter anomaly list and ready-to-embed counter summary table. - -Execution guardrails: -- Run benchmarks sequentially. -- No background jobs (`nohup`, `&`). -- Use Release timing builds only. -- If timing comparison fails, return blocked verdict with exact failure points. - -## Step 3 - Consume delegated hardware-counter outputs - -Hardware-counter collection is delegated to `benchmarks-compare-revisions`. - -Pass-through inputs: -- `COLLECT_COUNTERS=1` by default on Linux (unless `--no-counters` is provided). -- Same baseline/contender refs and effective filter used in Step 2. - -Consume outputs: -- Counter preflight result. -- Counter JSON artifacts for both revisions. -- Derived metrics (IPC, cache miss rate, branch mispredict rate). -- Anomaly list and counter summary table for report embedding. - -If counters are unavailable (`counters_available=false`), continue with timing-only review and explicitly mark profiling as unavailable in the report. - -## Step 4 - Analyze timing and counter data - -Timing classification per benchmark entry: -- Improvement: time delta < -5% -- Regression: time delta > +5% -- Neutral: between -5% and +5% - -Aggregate per binary: -- Number of improvements/regressions/neutral -- Net average percentage change -- Largest regression and largest improvement - -Counter correlation: -- Use skill-provided hardware counter summary and anomaly list to explain major timing changes. -- Do not recompute derived counter metrics in this command. - -Judgment priority: -- Base verdict primarily on benchmark timing comparison. -- Use counter data as explanatory evidence and confidence signal. - -Noise-control expectations: -- Include at least one control benchmark family expected to be unaffected by the code change. -- Treat isolated swings without pattern as noise unless reproduced across related sizes/fill ratios. - -## Step 5 - Produce final markdown report - -Return a structured markdown report with this shape: - -```markdown -## Performance Review: vs - -### Configuration -- Baseline: () -- Contender: () -- Platform: -- Benchmarks run: -- Filter: -- Hardware counters: available / unavailable - -### Timing Summary -| Binary | Improvements | Regressions | Neutral | Net Change | -|---|---:|---:|---:|---:| -| ... | N | N | N | +/-X% | - -### Detailed Timing Results - - -### Hardware Counter Profile (if available) -| Benchmark | IPC (base->new) | Cache Miss Rate (base->new) | Branch Mispredict (base->new) | -|---|---:|---:|---:| -| ... | X.XX -> Y.YY | A.A% -> B.B% | C.C% -> D.D% | - -### Key Findings -- -- - -### Verdict -**[IMPROVES PERFORMANCE | REGRESSES PERFORMANCE | NO SIGNIFICANT CHANGE]** - -<1-2 sentence justification grounded in benchmark metrics, with profiling context if available> -``` - -Verdict rules: -- `IMPROVES PERFORMANCE`: improvements outnumber regressions, no severe regression (>10%), and net average change is favorable. -- `REGRESSES PERFORMANCE`: any severe regression (>10%) or regressions dominate with net unfavorable average. -- `NO SIGNIFICANT CHANGE`: mostly neutral changes or mixed results that approximately cancel out. - -## Failure Handling - -- If required builds fail or timing comparison cannot run, output a blocked review with exact failure points and no misleading verdict. -- If only profiling fails (`counters_available=false` from delegated skill output), continue with timing-based verdict and explicitly list profiling limitation. -- If JSON output is invalid/truncated, discard it and rerun that benchmark command once with tighter filter and explicit output redirection. diff --git a/agentic/cpp/commands/ping.md b/agentic/cpp/commands/ping.md deleted file mode 100644 index b5edaf7..0000000 --- a/agentic/cpp/commands/ping.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -description: Test command that replies with pong ---- - -Respond with exactly `pong`. -Do not add any other words. -Do not add quotes or punctuation. diff --git a/agentic/cpp/skills/benchmarks-affected/SKILL.md b/agentic/cpp/skills/benchmarks-affected/SKILL.md deleted file mode 100644 index 2072dcb..0000000 --- a/agentic/cpp/skills/benchmarks-affected/SKILL.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -name: benchmarks-affected -description: Analyze current branch versus a baseline and extract affected benchmark targets and benchmark functions using compile_commands and clang AST. ---- - -# Benchmarks Affected Skill - -Use this skill to identify exactly which benchmark binaries and benchmark functions are affected by code changes on the current branch. - -It implements a two-stage workflow: - -1. `compile_commands.json` analysis to determine affected compile targets. -2. Clang AST analysis to determine affected benchmark functions. - -## Goal - -Given `HEAD` and a baseline branch (default `main`), produce: - -- Changed files. -- Affected targets (with emphasis on benchmark targets). -- Exact benchmark functions impacted by the changes. -- A ready-to-use Google Benchmark filter regex. - -## Prerequisites - -1. Build tree with benchmarks enabled and compile database exported. Use the -repository's normal benchmark-enabling CMake options: - -```bash -BUILD_SUFFIX=local -cmake -B build/benchmarks-all_${BUILD_SUFFIX} \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -cmake --build build/benchmarks-all_${BUILD_SUFFIX} --config Release -j -``` - -2. `clang++` must be available on `PATH` (used for AST dump). - -For repository-specific invocations, check -`agentic/local/cpp/skills/benchmarks-affected/EXAMPLES.md` when present. - -## Run - -```bash -python3 agentic/cpp/skills/benchmarks-affected/analyze_benchmarks_affected.py \ - --baseline main \ - --compile-commands build/benchmarks-all_local/compile_commands.json \ - --format json -``` - -If `--compile-commands` is omitted, the script auto-selects the most recently modified `build/**/compile_commands.json`. -Working tree changes are included by default. Use `--no-include-working-tree` to restrict analysis to `...HEAD` only. - -## Output - -The analyzer reports: - -- `affected_targets`: impacted CMake targets inferred from compile dependency analysis. -- `affected_benchmark_targets`: subset of benchmark binaries impacted. -- `affected_benchmarks`: precise benchmark function names from AST-level call analysis. -- `suggested_filter_regex`: regex to pass as `--benchmark_filter`. - -## How to Use Findings - -1. Build only impacted benchmark binaries where feasible. -2. Run benchmark binaries with the suggested filter: - -```bash -FILTER='^(BM_Foo|BM_Bar)(/|$)' -BENCH_CPU=${BENCH_CPU:-0} -taskset -c "${BENCH_CPU}" build/benchmarks-all_local/benchmarks --benchmark_filter="${FILTER}" -``` - -3. If impact mapping is broad/uncertain, run full binary for selected benchmark target(s). - -## Guardrails - -1. Keep baseline comparison at merge-base style diff: `...HEAD`. -2. Use Release binaries for timing runs. -3. If AST parse fails for a TU, still trust compile target impact and mark benchmark-function scope as partial. -4. If benchmark infra (`CMakeLists.txt`, benchmark source layout) changed, fall back to conservative benchmark selection. diff --git a/agentic/cpp/skills/benchmarks-affected/analyze_benchmarks_affected.py b/agentic/cpp/skills/benchmarks-affected/analyze_benchmarks_affected.py deleted file mode 100755 index 4dcae85..0000000 --- a/agentic/cpp/skills/benchmarks-affected/analyze_benchmarks_affected.py +++ /dev/null @@ -1,1132 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import argparse -import concurrent.futures -import json -import os -import re -import shlex -import shutil -import subprocess -import sys -from collections import defaultdict -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - - -def is_project_source(source: Path, repo_root: Path) -> bool: - """Exclude third-party deps and generated build files.""" - try: - rel = source.relative_to(repo_root) - except ValueError: - return False - rel_text = rel.as_posix() - if rel_text.startswith("build/") or "_deps/" in rel_text: - return False - return True - - -KNOWN_BENCHMARK_TARGETS = {"benchmarks"} - -HEADER_EXTENSIONS = { - ".h", - ".hh", - ".hpp", - ".hxx", - ".inc", - ".ipp", - ".tcc", -} - -BUILD_INFRA_FILES = { - "CMakeLists.txt", - "CMakePresets.json", -} - -DIFF_HUNK_RE = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") - -CPP_FUNCTION_START_RE = re.compile( - r"^\s*" - r"(?:template\s*<[^>]*>\s*)?" - r"(?:(?:inline|constexpr|consteval|constinit|static|friend|virtual|explicit)\s+)*" - r"[A-Za-z_~][\w:<>,\s\*&\[\]]*\s+" - r"([~A-Za-z_][A-Za-z0-9_]*)\s*" - r"\([^;{}]*\)\s*" - r"(?:const\s*)?" - r"(?:noexcept\s*)?" - r"(?:->\s*[^\{]+)?\{" -) - - -@dataclass -class CompileCommandEntry: - directory: Path - source: Path - arguments: list[str] - output: Path | None - target: str | None - dependencies: set[Path] = field(default_factory=set) - dep_error: str | None = None - - -@dataclass -class AstImpactResult: - benchmark_names: set[str] = field(default_factory=set) - affected_names: set[str] = field(default_factory=set) - ast_error: str | None = None - - -def run_command( - args: list[str], - cwd: Path, - check: bool = True, - timeout: float | None = 60.0, -) -> subprocess.CompletedProcess[str]: - return subprocess.run( - args, - cwd=str(cwd), - text=True, - capture_output=True, - check=check, - timeout=timeout, - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=( - "Analyze benchmark impact between baseline and HEAD via " - "compile_commands dependency mapping and clang AST analysis." - ) - ) - parser.add_argument( - "--baseline", - default="main", - help="Baseline ref used as ...HEAD (default: main).", - ) - parser.add_argument( - "--head", - default="HEAD", - help="Contender ref (default: HEAD).", - ) - parser.add_argument( - "--compile-commands", - default=None, - help=( - "Path to compile_commands.json. If omitted, auto-discovers most " - "recent build/**/compile_commands.json." - ), - ) - parser.add_argument( - "--clangxx", - default=None, - help="clang++ executable for AST dump (auto-detected if omitted).", - ) - parser.add_argument( - "--format", - choices=["text", "json"], - default="text", - help="Output format (default: text).", - ) - parser.add_argument( - "--include-working-tree", - dest="include_working_tree", - action="store_true", - default=True, - help=( - "Include local unstaged/staged changes in changed-files set, " - "in addition to ... (default: enabled)." - ), - ) - parser.add_argument( - "--no-include-working-tree", - dest="include_working_tree", - action="store_false", - help="Disable working-tree inclusion and only analyze ....", - ) - return parser.parse_args() - - -def git_repo_root() -> Path: - proc = run_command(["git", "rev-parse", "--show-toplevel"], cwd=Path.cwd()) - return Path(proc.stdout.strip()).resolve() - - -def resolve_compile_commands(repo_root: Path, explicit_path: str | None) -> Path: - if explicit_path: - compile_path = Path(explicit_path) - if not compile_path.is_absolute(): - compile_path = (repo_root / compile_path).resolve() - if not compile_path.exists(): - raise FileNotFoundError(f"compile_commands.json not found: {compile_path}") - return compile_path - - candidates = sorted( - repo_root.glob("build/**/compile_commands.json"), - key=lambda path: path.stat().st_mtime, - reverse=True, - ) - if not candidates: - raise FileNotFoundError( - "No compile_commands.json found under build/**. " - "Configure with -DCMAKE_EXPORT_COMPILE_COMMANDS=ON first." - ) - return candidates[0].resolve() - - -def load_compile_commands( - compile_commands_path: Path, - repo_root: Path, -) -> list[CompileCommandEntry]: - entries: list[CompileCommandEntry] = [] - data = json.loads(compile_commands_path.read_text(encoding="utf-8")) - for raw_entry in data: - directory = Path(raw_entry["directory"]).resolve() - - raw_source = Path(raw_entry["file"]) - if raw_source.is_absolute(): - source = raw_source.resolve() - else: - source = (directory / raw_source).resolve() - - if not is_project_source(source, repo_root): - continue - - if "arguments" in raw_entry: - arguments = [str(arg) for arg in raw_entry["arguments"]] - else: - arguments = shlex.split(raw_entry["command"]) - - output = infer_output_path(arguments, directory) - target = infer_cmake_target_from_output(output) - - entries.append( - CompileCommandEntry( - directory=directory, - source=source, - arguments=arguments, - output=output, - target=target, - ) - ) - return entries - - -def infer_output_path(arguments: list[str], directory: Path) -> Path | None: - output_token: str | None = None - for idx, arg in enumerate(arguments): - if arg == "-o" and idx + 1 < len(arguments): - output_token = arguments[idx + 1] - elif arg.startswith("-o") and len(arg) > 2: - output_token = arg[2:] - elif arg.startswith("/Fo") and len(arg) > 3: - output_token = arg[3:] - - if output_token is None: - return None - - out_path = Path(output_token) - if out_path.is_absolute(): - return out_path.resolve() - return (directory / out_path).resolve() - - -def infer_cmake_target_from_output(output: Path | None) -> str | None: - if output is None: - return None - parts = output.parts - for index, part in enumerate(parts): - if part == "CMakeFiles" and index + 1 < len(parts): - target_part = parts[index + 1] - if target_part.endswith(".dir"): - return target_part[: -len(".dir")] - return target_part - return None - - -def git_changed_files(repo_root: Path, baseline: str, head: str) -> set[Path]: - diff_range = f"{baseline}...{head}" - proc = run_command(["git", "diff", "--name-only", diff_range], cwd=repo_root) - changed_files: set[Path] = set() - for line in proc.stdout.splitlines(): - line = line.strip() - if not line: - continue - changed_files.add((repo_root / line).resolve()) - return changed_files - - -def git_working_tree_changed_files(repo_root: Path) -> set[Path]: - changed_files: set[Path] = set() - commands = [ - ["git", "diff", "--name-only"], - ["git", "diff", "--name-only", "--cached"], - ] - for cmd in commands: - proc = run_command(cmd, cwd=repo_root) - for line in proc.stdout.splitlines(): - line = line.strip() - if not line: - continue - changed_files.add((repo_root / line).resolve()) - return changed_files - - -def parse_changed_lines_from_diff_text( - diff_text: str, - repo_root: Path, -) -> dict[Path, set[int]]: - changed_lines: dict[Path, set[int]] = defaultdict(set) - - current_file: Path | None = None - in_hunk = False - new_line = 0 - - for raw_line in diff_text.splitlines(): - if raw_line.startswith("+++ "): - file_token = raw_line[4:].strip() - if file_token == "/dev/null": - current_file = None - in_hunk = False - continue - if file_token.startswith("b/"): - file_token = file_token[2:] - current_file = (repo_root / file_token).resolve() - in_hunk = False - continue - - hunk_match = DIFF_HUNK_RE.match(raw_line) - if hunk_match: - in_hunk = current_file is not None - new_line = int(hunk_match.group(1)) - continue - - if not in_hunk or current_file is None: - continue - - if raw_line.startswith("+") and not raw_line.startswith("+++"): - changed_lines[current_file].add(new_line) - new_line += 1 - continue - - if raw_line.startswith("-") and not raw_line.startswith("---"): - continue - - if raw_line.startswith(" "): - new_line += 1 - continue - - return changed_lines - - -def git_changed_line_map( - repo_root: Path, - baseline: str, - head: str, - include_working_tree: bool, -) -> dict[Path, set[int]]: - changed_lines: dict[Path, set[int]] = defaultdict(set) - - proc = run_command( - ["git", "diff", "--unified=0", f"{baseline}...{head}"], - cwd=repo_root, - ) - baseline_map = parse_changed_lines_from_diff_text(proc.stdout, repo_root) - for path, lines in baseline_map.items(): - changed_lines[path].update(lines) - - if include_working_tree: - for cmd in ( - ["git", "diff", "--unified=0"], - ["git", "diff", "--cached", "--unified=0"], - ): - wt_proc = run_command(cmd, cwd=repo_root) - wt_map = parse_changed_lines_from_diff_text(wt_proc.stdout, repo_root) - for path, lines in wt_map.items(): - changed_lines[path].update(lines) - - return changed_lines - - -def extract_changed_symbol_names_from_file( - file_path: Path, - changed_lines: set[int], -) -> set[str]: - if not changed_lines or not file_path.exists(): - return set() - - lines = file_path.read_text(encoding="utf-8", errors="replace").splitlines() - symbols: set[str] = set() - - line_index = 1 - max_line = len(lines) - while line_index <= max_line: - line = lines[line_index - 1] - match = CPP_FUNCTION_START_RE.match(line) - if not match: - line_index += 1 - continue - - symbol_name = match.group(1) - start_line = line_index - brace_depth = line.count("{") - line.count("}") - end_line = start_line - - while brace_depth > 0 and end_line < max_line: - end_line += 1 - body_line = lines[end_line - 1] - brace_depth += body_line.count("{") - body_line.count("}") - - if any(start_line <= line_no <= end_line for line_no in changed_lines): - symbols.add(symbol_name) - - line_index = end_line + 1 - - return symbols - - -def collect_changed_symbol_names( - changed_line_map: dict[Path, set[int]], -) -> set[str]: - symbol_names: set[str] = set() - for file_path, changed_lines in changed_line_map.items(): - symbol_names.update( - extract_changed_symbol_names_from_file(file_path, changed_lines) - ) - return symbol_names - - -def clean_command_for_dependency_scan(arguments: list[str]) -> list[str]: - cleaned: list[str] = [] - skip_next = False - flags_with_value = { - "-o", - "-MF", - "-MT", - "-MQ", - "-MJ", - "-Xclang", - } - standalone_drop = { - "-c", - "-MD", - "-MMD", - "-MP", - "-MM", - "-M", - "-E", - "-S", - } - - index = 0 - while index < len(arguments): - arg = arguments[index] - if skip_next: - skip_next = False - index += 1 - continue - - if arg in flags_with_value: - skip_next = True - index += 1 - continue - if arg in standalone_drop: - index += 1 - continue - if arg.startswith("-o") and len(arg) > 2: - index += 1 - continue - if arg.startswith("-MF") and len(arg) > 3: - index += 1 - continue - if arg.startswith("-MT") and len(arg) > 3: - index += 1 - continue - if arg.startswith("-MQ") and len(arg) > 3: - index += 1 - continue - if arg.startswith("-MJ") and len(arg) > 3: - index += 1 - continue - - cleaned.append(arg) - index += 1 - - return cleaned - - -def parse_makefile_dependencies(stdout_text: str) -> list[str]: - flattened = stdout_text.replace("\\\n", " ").replace("\n", " ") - if ":" not in flattened: - return [] - dep_payload = flattened.split(":", 1)[1].strip() - if not dep_payload: - return [] - return shlex.split(dep_payload) - - -def compute_tu_dependencies(entry: CompileCommandEntry) -> None: - dep_cmd = clean_command_for_dependency_scan(entry.arguments) - if not dep_cmd: - entry.dep_error = "Empty compile command after sanitization" - entry.dependencies = {entry.source} - return - - dep_cmd.extend(["-MM", "-MF", "-", "-MT", "__benchmark_affected_tu__"]) - source_arg = str(entry.source) - if source_arg not in dep_cmd: - dep_cmd.append(source_arg) - - try: - proc = run_command(dep_cmd, cwd=entry.directory, check=False) - except FileNotFoundError as exc: - entry.dep_error = str(exc) - entry.dependencies = {entry.source} - return - - dependencies: set[Path] = {entry.source} - if proc.returncode != 0: - stderr = proc.stderr.strip() - entry.dep_error = ( - stderr if stderr else f"Dependency scan failed ({proc.returncode})" - ) - entry.dependencies = dependencies - return - - for dep in parse_makefile_dependencies(proc.stdout): - dep_path = Path(dep) - resolved = ( - dep_path.resolve() - if dep_path.is_absolute() - else (entry.directory / dep_path).resolve() - ) - dependencies.add(resolved) - - entry.dependencies = dependencies - - -def is_build_infra_change(repo_root: Path, changed: set[Path]) -> bool: - for path in changed: - if path.name in BUILD_INFRA_FILES: - return True - try: - rel = path.relative_to(repo_root) - except ValueError: - continue - rel_text = rel.as_posix() - if rel_text.startswith("cmake/"): - return True - return False - - -def identify_benchmark_targets( - entries: list[CompileCommandEntry], repo_root: Path -) -> set[str]: - benchmark_targets: set[str] = set() - targets_present = {entry.target for entry in entries if entry.target} - for entry in entries: - if entry.target is None: - continue - try: - rel = entry.source.relative_to(repo_root) - rel_text = rel.as_posix() - except ValueError: - rel_text = entry.source.as_posix() - - if rel_text.startswith("src/benchmarks/"): - benchmark_targets.add(entry.target) - - benchmark_targets.update(targets_present.intersection(KNOWN_BENCHMARK_TARGETS)) - return benchmark_targets - - -def is_benchmark_source(source: Path, repo_root: Path) -> bool: - try: - rel_text = source.relative_to(repo_root).as_posix() - except ValueError: - return False - return rel_text.startswith("src/benchmarks/") - - -def dedupe_entries_by_target_source( - entries: list[CompileCommandEntry], -) -> list[CompileCommandEntry]: - deduped: list[CompileCommandEntry] = [] - seen: set[tuple[str | None, Path]] = set() - for entry in entries: - key = (entry.target, entry.source) - if key in seen: - continue - seen.add(key) - deduped.append(entry) - return deduped - - -def discover_clangxx(explicit: str | None) -> str: - if explicit: - return explicit - - candidates = [ - "clang++", - "clang++-19", - "clang++-18", - "clang++-17", - "clang++-16", - ] - for candidate in candidates: - resolved = shutil.which(candidate) - if resolved: - return resolved - raise FileNotFoundError( - "clang++ was not found on PATH. Provide --clangxx to select a clang compiler." - ) - - -def clean_command_for_ast(arguments: list[str], clangxx: str) -> list[str]: - cleaned = clean_command_for_dependency_scan(arguments) - if not cleaned: - return [] - cleaned[0] = clangxx - cleaned.extend(["-Xclang", "-ast-dump=json", "-fsyntax-only"]) - return cleaned - - -def normalize_path_candidate(path_text: str | None, working_dir: Path) -> Path | None: - if not path_text: - return None - path = Path(path_text) - if path.is_absolute(): - return path.resolve() - return (working_dir / path).resolve() - - -def file_from_loc(loc: dict[str, Any] | None, working_dir: Path) -> Path | None: - if not isinstance(loc, dict): - return None - if "file" in loc: - return normalize_path_candidate(str(loc["file"]), working_dir) - for nested_key in ("spellingLoc", "expansionLoc", "includedFrom"): - nested_loc = loc.get(nested_key) - if isinstance(nested_loc, dict): - resolved = file_from_loc(nested_loc, working_dir) - if resolved is not None: - return resolved - return None - - -def iter_ast_nodes(node: Any): - if isinstance(node, dict): - yield node - inner = node.get("inner", []) - if isinstance(inner, list): - for child in inner: - yield from iter_ast_nodes(child) - elif isinstance(node, list): - for item in node: - yield from iter_ast_nodes(item) - - -def referenced_decl_file(node: dict[str, Any], working_dir: Path) -> Path | None: - referenced = node.get("referencedDecl") - if not isinstance(referenced, dict): - return None - return file_from_loc(referenced.get("loc"), working_dir) - - -def node_references_changed_symbol( - node: dict[str, Any], - changed_symbol_names: set[str], -) -> bool: - if not changed_symbol_names: - return False - - for subnode in iter_ast_nodes(node): - if not isinstance(subnode, dict): - continue - - kind = subnode.get("kind") - if kind == "MemberExpr": - member_name = subnode.get("name") - if isinstance(member_name, str) and member_name in changed_symbol_names: - return True - - if kind == "DeclRefExpr": - ref_decl = subnode.get("referencedDecl") - if not isinstance(ref_decl, dict): - continue - ref_name = ref_decl.get("name") - if isinstance(ref_name, str) and ref_name in changed_symbol_names: - return True - - return False - - -def call_expr_callee_name(call_expr: dict[str, Any]) -> str | None: - for node in iter_ast_nodes(call_expr): - if not isinstance(node, dict): - continue - if node.get("kind") != "DeclRefExpr": - continue - referenced = node.get("referencedDecl") - if isinstance(referenced, dict) and isinstance(referenced.get("name"), str): - return referenced["name"] - return None - - -def string_literals_in_node(node: dict[str, Any]) -> list[str]: - values: list[str] = [] - for cur in iter_ast_nodes(node): - if not isinstance(cur, dict): - continue - if cur.get("kind") != "StringLiteral": - continue - value = cur.get("value") - if isinstance(value, str): - if len(value) >= 2 and value[0] == '"' and value[-1] == '"': - value = value[1:-1] - values.append(value) - return values - - -def benchmark_names_from_source(source: Path) -> set[str]: - names: set[str] = set() - if not source.exists(): - return names - text = source.read_text(encoding="utf-8", errors="replace") - for match in re.finditer(r"BENCHMARK\(\s*([A-Za-z_][A-Za-z0-9_]*)\s*\)", text): - names.add(match.group(1)) - for match in re.finditer(r"register_op\(\s*\"([^\"]+)\"", text): - names.add(match.group(1)) - return names - - -def ast_analyze_entry( - entry: CompileCommandEntry, - changed_files: set[Path], - changed_symbol_names: set[str], - clangxx: str, -) -> AstImpactResult: - result = AstImpactResult() - - ast_cmd = clean_command_for_ast(entry.arguments, clangxx) - if not ast_cmd: - result.ast_error = "Failed to build AST command" - return result - - try: - proc = run_command(ast_cmd, cwd=entry.directory, check=False) - except FileNotFoundError as exc: - result.ast_error = str(exc) - return result - - if proc.returncode != 0: - stderr = proc.stderr.strip() - result.ast_error = ( - stderr if stderr else f"AST command failed ({proc.returncode})" - ) - return result - - try: - ast_root = json.loads(proc.stdout) - except json.JSONDecodeError as exc: - result.ast_error = f"Invalid AST JSON: {exc}" - return result - - function_callees: dict[str, set[str]] = defaultdict(set) - direct_impacted_functions: set[str] = set() - dynamic_benchmarks_by_function: dict[str, set[str]] = defaultdict(set) - - for node in iter_ast_nodes(ast_root): - if not isinstance(node, dict): - continue - - if node.get("kind") not in {"FunctionDecl", "CXXMethodDecl"}: - continue - - function_name = node.get("name") - if not isinstance(function_name, str) or not function_name: - continue - - if function_name.startswith("BM_"): - result.benchmark_names.add(function_name) - - function_callees.setdefault(function_name, set()) - - function_loc = file_from_loc(node.get("loc"), entry.directory) - is_directly_impacted = function_loc in changed_files - if not is_directly_impacted: - is_directly_impacted = node_references_changed_symbol( - node, changed_symbol_names - ) - - for subnode in iter_ast_nodes(node): - if not isinstance(subnode, dict): - continue - - sub_kind = subnode.get("kind") - if sub_kind in {"CallExpr", "CXXMemberCallExpr", "CXXOperatorCallExpr"}: - callee = call_expr_callee_name(subnode) - if callee: - function_callees[function_name].add(callee) - - if callee == "register_op": - literal_values = string_literals_in_node(subnode) - if literal_values: - dynamic_benchmarks_by_function[function_name].add( - literal_values[0] - ) - - if not is_directly_impacted: - ref_file = referenced_decl_file(subnode, entry.directory) - if ref_file in changed_files: - is_directly_impacted = True - - if is_directly_impacted: - direct_impacted_functions.add(function_name) - - # Reverse call-graph propagation: if a function is directly impacted, - # every caller in this TU is impacted as well (fixed-point DFS/BFS). - callers_of: dict[str, set[str]] = defaultdict(set) - for caller, callees in function_callees.items(): - for callee in callees: - callers_of[callee].add(caller) - - impacted_functions = set(direct_impacted_functions) - stack = list(direct_impacted_functions) - while stack: - callee_name = stack.pop() - for caller_name in callers_of.get(callee_name, set()): - if caller_name in impacted_functions: - continue - impacted_functions.add(caller_name) - stack.append(caller_name) - - for function_name in impacted_functions: - if function_name.startswith("BM_"): - result.affected_names.add(function_name) - - for function_name, names in dynamic_benchmarks_by_function.items(): - result.benchmark_names.update(names) - if function_name in impacted_functions: - result.affected_names.update(names) - - return result - - -def regex_for_benchmarks(names: set[str]) -> str | None: - if not names: - return None - ordered = sorted(names) - body = "|".join(re.escape(name) for name in ordered) - return rf"^({body})(/|$)" - - -def relpath_or_abs(path: Path, root: Path) -> str: - try: - return path.relative_to(root).as_posix() - except ValueError: - return path.as_posix() - - -def main() -> int: - cli = parse_args() - - try: - repo_root = git_repo_root() - changed_files = git_changed_files(repo_root, cli.baseline, cli.head) - if cli.include_working_tree: - changed_files.update(git_working_tree_changed_files(repo_root)) - changed_line_map = git_changed_line_map( - repo_root, - cli.baseline, - cli.head, - cli.include_working_tree, - ) - changed_symbol_names = collect_changed_symbol_names(changed_line_map) - compile_commands_path = resolve_compile_commands( - repo_root, cli.compile_commands - ) - entries = load_compile_commands(compile_commands_path, repo_root) - except FileNotFoundError as exc: - print(f"error: {exc}", file=sys.stderr) - return 2 - except subprocess.CalledProcessError as exc: - stderr = (exc.stderr or "").strip() - if stderr: - print(f"error: {stderr}", file=sys.stderr) - else: - print(f"error: command failed: {' '.join(exc.cmd)}", file=sys.stderr) - return 2 - - target_to_entries: dict[str, list[CompileCommandEntry]] = defaultdict(list) - source_to_entries: dict[Path, list[CompileCommandEntry]] = defaultdict(list) - for entry in entries: - source_to_entries[entry.source].append(entry) - if entry.target: - target_to_entries[entry.target].append(entry) - - benchmark_targets = identify_benchmark_targets(entries, repo_root) - all_targets = {entry.target for entry in entries if entry.target} - benchmark_entries = dedupe_entries_by_target_source( - [entry for entry in entries if entry.target in benchmark_targets] - ) - - infra_change = is_build_infra_change(repo_root, changed_files) - relevant_changed_files = { - path - for path in changed_files - if is_project_source(path, repo_root) - or path.name in BUILD_INFRA_FILES - or relpath_or_abs(path, repo_root).startswith("cmake/") - } - has_header_changes = any( - path.suffix.lower() in HEADER_EXTENSIONS for path in relevant_changed_files - ) - benchmark_source_extensions = {".c", ".cc", ".cpp", ".cxx"} - only_benchmark_source_changes = bool(relevant_changed_files) and all( - is_benchmark_source(path, repo_root) - and path.suffix.lower() in benchmark_source_extensions - for path in relevant_changed_files - ) - - directly_affected_targets: set[str] = set() - for changed_path in changed_files: - for entry in source_to_entries.get(changed_path, []): - if entry.target: - directly_affected_targets.add(entry.target) - - dependency_scan_entries: list[CompileCommandEntry] = [] - if not infra_change and not only_benchmark_source_changes: - if has_header_changes: - dependency_scan_entries = dedupe_entries_by_target_source(entries) - else: - dependency_scan_entries = benchmark_entries - - if dependency_scan_entries: - with concurrent.futures.ThreadPoolExecutor( - max_workers=min(8, (os.cpu_count() or 4)) - ) as pool: - list(pool.map(compute_tu_dependencies, dependency_scan_entries)) - - affected_targets: set[str] = set(directly_affected_targets) - for entry in dependency_scan_entries: - has_changed_dependency = any(dep in changed_files for dep in entry.dependencies) - if has_changed_dependency and entry.target: - affected_targets.add(entry.target) - - if infra_change: - affected_targets.update(all_targets) - - dependency_impacted_benchmark_targets = affected_targets.intersection( - benchmark_targets - ) - impacted_benchmark_entries = [ - entry - for entry in benchmark_entries - if entry.target in dependency_impacted_benchmark_targets - ] - - ast_errors: dict[str, str] = {} - benchmark_target_to_names: dict[str, set[str]] = defaultdict(set) - benchmark_target_to_affected: dict[str, set[str]] = defaultdict(set) - warnings: list[str] = [] - ast_fallback_used = False - ast_entries_scanned = 0 - - if impacted_benchmark_entries: - try: - clangxx = discover_clangxx(cli.clangxx) - except FileNotFoundError as exc: - clangxx = "" - warnings.append(str(exc)) - - if not clangxx: - ast_fallback_used = True - for entry in impacted_benchmark_entries: - target_name = entry.target or "" - fallback_names = benchmark_names_from_source(entry.source) - benchmark_target_to_names[target_name].update(fallback_names) - benchmark_target_to_affected[target_name].update(fallback_names) - else: - max_ast_workers = min(2, (os.cpu_count() or 2)) - with concurrent.futures.ThreadPoolExecutor( - max_workers=max_ast_workers - ) as pool: - futures = { - pool.submit( - ast_analyze_entry, - entry, - changed_files, - changed_symbol_names, - clangxx, - ): entry - for entry in impacted_benchmark_entries - } - ast_entries_scanned = len(futures) - for future in concurrent.futures.as_completed(futures): - entry = futures[future] - target_name = entry.target or "" - source_path = entry.source - source_is_changed = source_path in changed_files - - try: - ast_result = future.result(timeout=120) - except Exception as exc: - ast_result = AstImpactResult( - ast_error=f"AST worker failed: {exc}" - ) - - if ast_result.ast_error: - ast_errors[relpath_or_abs(source_path, repo_root)] = ( - ast_result.ast_error - ) - - benchmark_names = ast_result.benchmark_names - if not benchmark_names: - benchmark_names = benchmark_names_from_source(source_path) - benchmark_target_to_names[target_name].update(benchmark_names) - - if ast_result.affected_names: - benchmark_target_to_affected[target_name].update( - ast_result.affected_names - ) - elif source_is_changed or ast_result.ast_error: - benchmark_target_to_affected[target_name].update( - benchmark_names - ) - if benchmark_names: - ast_fallback_used = True - - if infra_change and benchmark_targets: - for target_name in sorted(benchmark_targets): - for entry in target_to_entries.get(target_name, []): - names = benchmark_names_from_source(entry.source) - benchmark_target_to_names[target_name].update(names) - benchmark_target_to_affected[target_name].update(names) - - if infra_change: - affected_benchmark_targets = sorted(benchmark_targets) - else: - affected_benchmark_targets = sorted( - target for target, names in benchmark_target_to_affected.items() if names - ) - - all_affected_benchmarks: set[str] = set() - for names in benchmark_target_to_affected.values(): - all_affected_benchmarks.update(names) - - dep_scan_failures = { - relpath_or_abs(entry.source, repo_root): entry.dep_error - for entry in dependency_scan_entries - if entry.dep_error - } - - scope_mode = "normal" - if infra_change: - scope_mode = "infra_fallback" - elif ast_fallback_used: - scope_mode = "ast_fallback" - - report: dict[str, Any] = { - "baseline": cli.baseline, - "head": cli.head, - "include_working_tree": cli.include_working_tree, - "changed_symbols": sorted(changed_symbol_names), - "compile_commands": relpath_or_abs(compile_commands_path, repo_root), - "changed_files": sorted( - relpath_or_abs(path, repo_root) for path in changed_files - ), - "affected_targets": sorted(affected_targets), - "affected_benchmark_targets": affected_benchmark_targets, - "affected_benchmarks": { - target: sorted(names) - for target, names in sorted(benchmark_target_to_affected.items()) - if names - }, - "suggested_filter_regex": regex_for_benchmarks(all_affected_benchmarks), - "dependency_entries_scanned": len(dependency_scan_entries), - "benchmark_entries_scanned": len(benchmark_entries), - "ast_entries_scanned": ast_entries_scanned, - "scope_mode": scope_mode, - "dependency_scan_failures": dep_scan_failures, - "ast_failures": ast_errors, - "warnings": warnings, - } - - if cli.format == "json": - json.dump(report, sys.stdout, indent=2) - sys.stdout.write("\n") - return 0 - - print(f"Baseline: {cli.baseline}") - print(f"Head: {cli.head}") - print(f"Compile commands: {report['compile_commands']}") - print(f"Scope mode: {report['scope_mode']}") - print( - "Scan counts: " - f"dependency={report['dependency_entries_scanned']}, " - f"benchmark={report['benchmark_entries_scanned']}, " - f"ast={report['ast_entries_scanned']}" - ) - print("") - - print(f"Changed files ({len(report['changed_files'])}):") - for item in report["changed_files"]: - print(f"- {item}") - if not report["changed_files"]: - print("- none") - - print("") - print(f"Affected targets ({len(report['affected_targets'])}):") - for item in report["affected_targets"]: - print(f"- {item}") - if not report["affected_targets"]: - print("- none") - - print("") - print(f"Affected benchmark targets ({len(report['affected_benchmark_targets'])}):") - for item in report["affected_benchmark_targets"]: - print(f"- {item}") - if not report["affected_benchmark_targets"]: - print("- none") - - print("") - print("Affected benchmark functions:") - if report["affected_benchmarks"]: - for target, names in report["affected_benchmarks"].items(): - print(f"- {target}:") - for name in names: - print(f" - {name}") - else: - print("- none") - - print("") - print("Suggested --benchmark_filter regex:") - print(report["suggested_filter_regex"] or "none") - - if dep_scan_failures: - print("") - print("Dependency scan failures:") - for source, error in dep_scan_failures.items(): - print(f"- {source}: {error}") - - if ast_errors: - print("") - print("AST failures:") - for source, error in ast_errors.items(): - print(f"- {source}: {error}") - - if warnings: - print("") - print("Warnings:") - for warning in warnings: - print(f"- {warning}") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/agentic/cpp/skills/benchmarks-compare-revisions/SKILL.md b/agentic/cpp/skills/benchmarks-compare-revisions/SKILL.md deleted file mode 100644 index caa9542..0000000 --- a/agentic/cpp/skills/benchmarks-compare-revisions/SKILL.md +++ /dev/null @@ -1,225 +0,0 @@ ---- -name: benchmarks-compare-revisions -description: Compare benchmark performance between two git revisions via Google Benchmark compare.py, with optional hardware-counter comparison from diagnostic libpfm builds. ---- - -# Benchmarks Compare Revisions Skill - -Use this skill to compare performance between two git revisions. - -This workflow now depends on: - -1. `../benchmarks-affected/SKILL.md` to determine affected benchmark targets/functions and produce a benchmark filter. -2. `../benchmarks/SKILL.md` for build/run operational details. - -## Goal - -Build two separate benchmark binaries using short commit hashes as build suffixes, compare timing results with Google Benchmark compare.py, and optionally compare hardware counters across the same revisions. - -## Step 0 — Choose revisions, hashes, and options - -Pick a baseline and a contender revision. Use short commit hashes to suffix build directories so builds do not collide. - -Optional behavior flags: - -- `COLLECT_COUNTERS=1` to enable hardware-counter collection and analysis in addition to timing comparison. -- `COLLECT_COUNTERS=0` to run timing-only comparison. - -Counter collection is Linux-only and requires: - -- diagnostic builds with `BENCHMARK_ENABLE_LIBPFM=ON` -- perf permissions on the host (for access to performance counters) - -Example: -```bash -BASELINE=abc1234 -CONTENDER=def5678 -``` - -## Step 1 — Compute affected benchmark scope first - -Run `benchmarks-affected` from the contender checkout to derive the compare scope. - -Do not duplicate `benchmarks-affected` internals here (compile database selection, AST analysis, or fallback heuristics). Follow that skill directly and consume only its outputs. - -Inputs to pass through: - -- `--baseline ${BASELINE}` -- optional compile-commands path if auto-detection is not desired -- optional output format (`json` recommended for parsing) - -Consume these outputs from `benchmarks-affected`: - -- `suggested_filter_regex` -> set `FILTER` -- `affected_benchmark_targets` -> optionally constrain which benchmark binary/binaries to run -- `affected_benchmarks` -> function-level scope for validation/reporting - -If `FILTER` is empty, fall back to full benchmark binary compare (conservative mode). - -## Step 2 — Build both revisions - -Use the existing benchmarks skill build steps, but set the build suffix to include the short hash for each revision. - -Always build Release timing binaries. - -If `COLLECT_COUNTERS=1`, also build diagnostic binaries (RelWithDebInfo + libpfm) for both revisions. - -```bash -# Baseline -BUILD_SUFFIX=bench_${BASELINE} -git checkout ${BASELINE} -# Follow ../benchmarks/SKILL.md timing build instructions with this suffix -# If COLLECT_COUNTERS=1, also follow the diagnostic build instructions with this suffix - -# Contender -BUILD_SUFFIX=bench_${CONTENDER} -git checkout ${CONTENDER} -# Follow ../benchmarks/SKILL.md timing build instructions with this suffix -# If COLLECT_COUNTERS=1, also follow the diagnostic build instructions with this suffix -``` - -Expected build trees: - -- Timing: `build/benchmarks-all_bench_` -- Counters (optional): `build/benchmarks-diagnostic_bench_` - -## Step 3 — Compare using compare.py - -Use Google Benchmark compare tooling with a JSON-first flow to avoid long-running binary-vs-binary retries. - -Locate compare.py from the Google Benchmark dependency (installed under the build tree): -```bash -COMPARE_PY=build/benchmarks-all_bench_${BASELINE}/_deps/googlebenchmark-src/tools/compare.py -``` - -Verify Python deps once (compare.py imports numpy/scipy): -```bash -python3 -c "import numpy, scipy" -``` - -Generate baseline/contender JSON sequentially with explicit file outputs: -```bash -BENCH_CPU=${BENCH_CPU:-0} -BENCH_RUN="taskset -c ${BENCH_CPU}" -BASE_JSON=/tmp/bench_${BASELINE}.json -CONT_JSON=/tmp/bench_${CONTENDER}.json - -${BENCH_RUN} build/benchmarks-all_bench_${BASELINE}/benchmarks \ - --benchmark_report_aggregates_only=true \ - --benchmark_display_aggregates_only=true \ - --benchmark_format=json \ - --benchmark_out=${BASE_JSON} > /tmp/bench_${BASELINE}.log 2>&1 - -${BENCH_RUN} build/benchmarks-all_bench_${CONTENDER}/benchmarks \ - --benchmark_report_aggregates_only=true \ - --benchmark_display_aggregates_only=true \ - --benchmark_format=json \ - --benchmark_out=${CONT_JSON} > /tmp/bench_${CONTENDER}.log 2>&1 -``` - -Validate JSON before comparing: -```bash -python3 -m json.tool ${BASE_JSON} > /dev/null -python3 -m json.tool ${CONT_JSON} > /dev/null -``` - -Run the comparison: -```bash -python3 ${COMPARE_PY} -a benchmarks ${BASE_JSON} ${CONT_JSON} -``` - -Use the affected filter from Step 1 when generating JSON files: -```bash -if [ -n "${FILTER}" ]; then - FILTER_ARG="--benchmark_filter=${FILTER}" -else - FILTER_ARG="" -fi - -${BENCH_RUN} build/benchmarks-all_bench_${BASELINE}/benchmarks ${FILTER_ARG} --benchmark_report_aggregates_only=true --benchmark_display_aggregates_only=true ... -${BENCH_RUN} build/benchmarks-all_bench_${CONTENDER}/benchmarks ${FILTER_ARG} --benchmark_report_aggregates_only=true --benchmark_display_aggregates_only=true ... -``` - -## Step 3b — Compare hardware counters (optional, Linux only) - -Run this step only when `COLLECT_COUNTERS=1`. - -1. Preflight first with one tiny counter-enabled benchmark run from a diagnostic binary. If output includes warnings such as `Failed to get a file descriptor for performance counter`, mark counters unavailable and skip counter collection. -2. Run baseline and contender diagnostic binaries sequentially with explicit JSON outputs and the same filter scope: - -```bash -BASE_COUNTERS_JSON=/tmp/bench_counters_${BASELINE}.json -CONT_COUNTERS_JSON=/tmp/bench_counters_${CONTENDER}.json - -${BENCH_RUN} build/benchmarks-diagnostic_bench_${BASELINE}/benchmarks \ - ${FILTER_ARG} \ - --benchmark_counters_tabular=true \ - --benchmark_format=json \ - --benchmark_out=${BASE_COUNTERS_JSON} > /tmp/bench_counters_${BASELINE}.log 2>&1 - -${BENCH_RUN} build/benchmarks-diagnostic_bench_${CONTENDER}/benchmarks \ - ${FILTER_ARG} \ - --benchmark_counters_tabular=true \ - --benchmark_format=json \ - --benchmark_out=${CONT_COUNTERS_JSON} > /tmp/bench_counters_${CONTENDER}.log 2>&1 -``` - -3. Validate JSON files before consuming: - -```bash -python3 -m json.tool ${BASE_COUNTERS_JSON} > /dev/null -python3 -m json.tool ${CONT_COUNTERS_JSON} > /dev/null -``` - -4. Collect and compare these counter families when present: - -- `instructions`, `cycles` -- `cache-misses`, `cache-references` -- `branch-misses`, `branches` -- `L1-dcache-load-misses` - -5. Compute derived metrics when denominators are non-zero: - -- IPC = `instructions / cycles` -- Cache miss rate = `cache-misses / cache-references` -- Branch mispredict rate = `branch-misses / branches` - -6. Pair baseline and contender rows by benchmark name, compute deltas, and flag anomalies where timing direction conflicts with key counter direction. - -7. Emit a canonical summary table for downstream consumers: - -```markdown -| Benchmark | IPC (base -> new) | Cache Miss Rate (base -> new) | Branch Mispredict (base -> new) | Anomaly? | -|---|---:|---:|---:|---| -``` - -## Retry and Timeout Policy - -1. Run benchmarks sequentially; do not background with `nohup`/`&`. -2. If a run times out, narrow filter and retry once. -3. Maximum retries per benchmark group: 1. -4. If still failing, emit blocked/partial findings instead of repeated attempts. - -Apply this policy to both timing and counter runs. - -## Step 4 — Record findings - -Capture and return: - -- compare.py output (terminal transcript or redirected file) -- effective filter used -- timing JSON artifacts for baseline and contender -- `counters_available` (`true`/`false`) -- if `counters_available=false`, a reason string (unsupported OS, missing libpfm, perf permission denied, preflight failure) -- if counters are available: counter JSON artifacts, derived metrics table, and anomaly list - -## Best Practices / Guardrails - -1. **Release only**: never compare Debug binaries. -2. **Short hash suffixes**: keep build dirs isolated per revision (example: `bench_`). -3. **Same host, same conditions**: do not compare across different machines or power profiles. -4. **Filter from analysis**: use `benchmarks-affected` output instead of hand-crafted filters whenever possible. -5. **Pin process and frequency**: use `taskset -c ${BENCH_CPU:-0}` for all benchmark executions and follow benchmark skill guidance on CPU governor. -6. **Counter collection is optional and Linux-only**: when unavailable, return timing-only outputs with `counters_available=false`. -7. **Always preflight counters**: do not run full counter collection if preflight fails. -8. **Keep build types separated**: timing uses `benchmarks-all_*` Release builds; counters use `benchmarks-diagnostic_*` RelWithDebInfo builds; never Debug. diff --git a/agentic/cpp/skills/benchmarks/SKILL.md b/agentic/cpp/skills/benchmarks/SKILL.md deleted file mode 100644 index 92af231..0000000 --- a/agentic/cpp/skills/benchmarks/SKILL.md +++ /dev/null @@ -1,209 +0,0 @@ ---- -name: benchmarks -description: Run Google Benchmark binaries, including filtering, hardware counters, and perf profiling. ---- - -# Benchmarks Skill - -You now have expertise in running and interpreting Google Benchmark suites. -Follow these workflows: - -## Build Directory Convention - -Use a short commit hash suffix for committed revisions: - -```bash -BUILD_SUFFIX=$(git rev-parse --short HEAD) -``` - -If the worktree has uncommitted changes, append a descriptive suffix so results -cannot be confused with a clean HEAD build: - -```bash -BUILD_SUFFIX=$(git rev-parse --short HEAD)-dirty -``` - -If not a git repository, use - -```bash -BUILD_SUFFIX=agent -``` - -## CRITICAL: Never Run Benchmarks from a Debug Build - -> **Always pass `--config Release` (or `--config RelWithDebInfo`) to `cmake --build`.** -> Multi-config generators (MSVC, Xcode) default to `Debug` if no `--config` is given. -> Google Benchmark will print `***WARNING*** Library was built as DEBUG` and timings will -> be 3-10x slower and meaningless. Always verify the binary path contains `Release/` or -> `RelWithDebInfo/`, never `Debug/`. - -## Step 1 — Build - -If benchmarks affected by the changes are easily tractable build only related targets. - -**Pure timing (benchmarks, Release):** -```bash -cmake -B build/benchmarks_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Release -cmake --build build/benchmarks_${BUILD_SUFFIX} --config Release -j -``` - -**Hardware counters / verbose report (benchmarks-diagnostic, RelWithDebInfo, Linux only):** -```bash -cmake -B build/benchmarks-diagnostic_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBENCHMARK_ENABLE_LIBPFM=ON -cmake --build build/benchmarks-diagnostic_${BUILD_SUFFIX} --config RelWithDebInfo -j -``` - -For repository-specific benchmark examples, check -`agentic/local/cpp/skills/benchmarks/EXAMPLES.md` when present. - -## Step 2 — Run - -Prefer running benchmarks with filtering passing the benchmarks that should be affected. - -Unless the user explicitly asks otherwise, pin benchmark execution to one CPU with -`taskset` to reduce scheduler noise. Use CPU 0 by default, or override with -`BENCH_CPU=` when a better isolated/performance core is known: - -```bash -BENCH_CPU=${BENCH_CPU:-0} -BENCH_RUN="taskset -c ${BENCH_CPU}" -``` - -If `taskset` is unavailable or fails on the host, report that benchmark results -are unpinned and more noisy. - -Execution guardrails: -- Run benchmark commands sequentially in CI. -- Avoid background jobs (`nohup`, `&`) for benchmark collection. -- Always write machine-readable results with `--benchmark_out` when data is later parsed. - -### Available benchmark binaries - -Discover benchmark binary names from the repository's build system. Common -locations include `build/**/` for single-config generators and -`build/**/Release/` for multi-config generators. Repository-specific -binary lists belong in the repo-local benchmark examples overlay. - -Binary paths vary by generator type: - -| Generator | Path pattern | -|-----------|-------------| -| MSVC / Xcode (multi-config) | `build/_/Release/` | -| Ninja / Make (single-config) | `build/_/` | - -### Run all benchmarks in a binary - -```bash -# Multi-config (MSVC/Xcode) -${BENCH_RUN} build/benchmarks_${BUILD_SUFFIX}/Release/benchmarks - -# Single-config (Ninja/Make) -${BENCH_RUN} build/benchmarks_${BUILD_SUFFIX}/benchmarks -``` - -### Filter benchmarks with a regex (FILTER parameter) - -```bash -FILTER="BM_Foo" # change to match benchmark names in the target binary - -# Multi-config -${BENCH_RUN} build/benchmarks_${BUILD_SUFFIX}/Release/benchmarks --benchmark_filter="${FILTER}" - -# Single-config -${BENCH_RUN} build/benchmarks_${BUILD_SUFFIX}/benchmarks --benchmark_filter="${FILTER}" -``` - -Examples: -```bash -# Only one benchmark family -... --benchmark_filter="BM_Foo" - -# Only one layout/parameter family -... --benchmark_filter="BM_Foo.*Variant" - -# List all available benchmark names without running -... --benchmark_list_tests=true -``` - -### Run with hardware counters (benchmarks-diagnostic build, Linux only) - -The `--benchmark_perf_counters` flag requests hardware counter collection via libpfm. Counter names are platform-specific but common ones include `CYCLES`, `INSTRUCTIONS`, `CACHE-MISSES`, `CACHE-REFERENCES`, `BRANCH-MISSES`, `BRANCH-INSTRUCTIONS`. - -```bash -${BENCH_RUN} build/benchmarks-diagnostic_${BUILD_SUFFIX}/RelWithDebInfo/benchmarks \ - --benchmark_filter="${FILTER}" \ - --benchmark_perf_counters=CYCLES,INSTRUCTIONS,CACHE-MISSES \ - --benchmark_counters_tabular=true -``` - -### Save results to file - -```bash -${BENCH_RUN} build/benchmarks_${BUILD_SUFFIX}/Release/benchmarks \ - --benchmark_filter="${FILTER}" \ - --benchmark_report_aggregates_only=true \ - --benchmark_display_aggregates_only=true \ - --benchmark_format=json \ - --benchmark_out=results.json -``` - -Validate output before consuming: -```bash -python3 -m json.tool results.json > /dev/null -``` - -## Step 3 — Profile with perf (Linux only) - -Use when hardware counters alone are not enough and you need a full call-graph profile for post-processing. - -**Record:** -```bash -perf record -g -F 999 \ - -- ${BENCH_RUN} build/benchmarks-diagnostic_${BUILD_SUFFIX}/benchmarks \ - --benchmark_filter="${FILTER}" \ - --benchmark_min_time=5s -``` - -**Quick report (terminal):** -```bash -perf report --stdio -``` - -**Flame graph (requires FlameGraph scripts):** -```bash -perf script | stackcollapse-perf.pl | flamegraph.pl > flamegraph.html -``` - -**Export for external tools (Hotspot, Firefox Profiler):** -```bash -perf script -F +pid > perf.data.txt -# or open with `hotspot perf.data` -``` - -## Useful Benchmark Flags - -| Flag | Purpose | -|------|---------| -| `--benchmark_filter=` | Run only matching benchmarks | -| `--benchmark_list_tests=true` | List names without running | -| `--benchmark_repetitions=` | Repeat each benchmark n times | -| `--benchmark_min_time=` | Minimum run time per benchmark | -| `--benchmark_format=json` | Machine-readable output | -| `--benchmark_out=` | Save output to file | -| `--benchmark_perf_counters=CYCLES,INSTRUCTIONS,...` | Collect hardware perf counters (requires libpfm build) | -| `--benchmark_counters_tabular=true` | Align user/perf counter columns into a table | -| `--benchmark_time_unit=ms` | Change display unit (ns/us/ms/s) | - -## Best Practices - -1. **Never run from a Debug binary**: always use `--config Release` at build time; check path contains `Release/` -2. **Use benchmarks for clean timing**: Release optimizations, no debug info, no libpfm overhead -3. **Use benchmarks-diagnostic for hardware counters**: RelWithDebInfo + libpfm; Linux only -4. **Use perf for deep profiling**: when counters point to a hotspot but don't explain it -5. **Pin benchmark process** with `taskset -c ${BENCH_CPU:-0}` unless unavailable -6. **Pin CPU frequency** before timing runs: `sudo cpupower frequency-set -g performance` -7. **Filter to reduce noise**: narrow the filter regex to the benchmark under investigation -8. **Save JSON output** when comparing before/after changes: use `--benchmark_out` and diff the files -9. **Fail fast on environment issues**: precheck Python deps used by compare tooling (`numpy`, `scipy`) -10. **Use explicit retry limits**: on timeout, narrow scope and retry once; avoid repeated full-suite attempts -11. **Preflight perf counters**: run a tiny counter-enabled benchmark first; if counters unavailable, skip counter workflow diff --git a/agentic/cpp/skills/benchmarks/scripts/plot_benchmarks.py b/agentic/cpp/skills/benchmarks/scripts/plot_benchmarks.py deleted file mode 100644 index abe5040..0000000 --- a/agentic/cpp/skills/benchmarks/scripts/plot_benchmarks.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python3 -"""Generic Google Benchmark JSON plotter. - -Plots timing and (optionally) hardware counter data from Google Benchmark ---benchmark_format=json output. Correctly handles repetition output by -averaging raw iterations and skipping aggregate entries. - -Usage: - python3 plot_benchmarks.py results.json [output_prefix] - python3 plot_benchmarks.py results.json report --min-n 16 --max-n 1048576 -""" -import json -import sys -import argparse -import matplotlib.pyplot as plt -import numpy as np - -def load_benchmark_json(path): - with open(path, 'r') as f: - return json.load(f) - -def extract_series(data, metric='time', min_n=None, max_n=None): - """Extract series from benchmark JSON. - - metric='time' -> real_time (ns) - metric='counter' -> first available hardware counter (e.g. CACHE-MISSES) - """ - # Determine counter key if requested - counter_key = None - if metric == 'counter': - # Find first counter key from the first iteration entry - for bench in data.get('benchmarks', []): - if bench.get('run_type', 'iteration') != 'iteration': - continue - for k in bench.keys(): - if k not in ('name', 'run_name', 'run_type', 'repetitions', - 'repetition_index', 'threads', 'iterations', - 'real_time', 'cpu_time', 'time_unit', - 'items_per_second', 'aggregate_name', - 'aggregate_unit', 'family_index', - 'per_family_instance_index'): - counter_key = k - break - if counter_key: - break - if not counter_key: - return {} - - raw = {} - for bench in data.get('benchmarks', []): - if bench.get('run_type', 'iteration') != 'iteration': - continue - - name = bench['name'] - parts = name.split('/') - if len(parts) < 2: - continue - - bench_name = parts[0] - try: - n = int(parts[1]) - except ValueError: - continue - - if min_n is not None and n < min_n: - continue - if max_n is not None and n > max_n: - continue - - if metric == 'time': - val = bench.get('real_time', bench.get('cpu_time', 0)) - else: - val = bench.get(counter_key) - - if val is None or val == 0: - continue - - key = (bench_name, n) - raw.setdefault(key, []).append(val) - - series = {} - for (bench_name, n), vals in raw.items(): - series.setdefault(bench_name, []).append((n, sum(vals) / len(vals))) - - for name in series: - series[name].sort(key=lambda x: x[0]) - - return series - -def plot_series(series, output_prefix, ylabel, title_suffix): - if not series: - print(f"No data to plot for {title_suffix}") - return - - fig, ax = plt.subplots(figsize=(12, 8)) - colors = plt.cm.tab10(np.linspace(0, 1, len(series))) - - for idx, (name, points) in enumerate(sorted(series.items())): - xs = [p[0] for p in points] - ys = [p[1] for p in points] - ax.plot(xs, ys, marker='o', markersize=3, label=name, color=colors[idx]) - - ax.set_xscale('log') - ax.set_xlabel('Benchmark parameter n') - ax.set_ylabel(ylabel) - ax.set_title(f'Benchmark Results - {title_suffix}') - ax.legend(loc='upper left', fontsize='small') - ax.grid(True, which='both', ls='--', alpha=0.5) - fig.tight_layout() - fig.savefig(f'{output_prefix}.png', dpi=150) - fig.savefig(f'{output_prefix}.svg') - print(f'Saved {output_prefix}.png and {output_prefix}.svg') - plt.close(fig) - -def main(): - parser = argparse.ArgumentParser( - description='Plot Google Benchmark JSON results.') - parser.add_argument('json_path', help='Path to benchmark JSON file') - parser.add_argument('output_prefix', nargs='?', default='report', - help='Output file prefix (default: report)') - parser.add_argument('--min-n', type=int, default=None, - help='Minimum parameter value to include (inclusive)') - parser.add_argument('--max-n', type=int, default=None, - help='Maximum parameter value to include (inclusive)') - args = parser.parse_args() - - data = load_benchmark_json(args.json_path) - - # Timing plot - time_series = extract_series(data, metric='time', - min_n=args.min_n, max_n=args.max_n) - if time_series: - plot_series(time_series, args.output_prefix, - 'Time per iteration (ns)', 'Time') - - # Counter plot - counter_series = extract_series(data, metric='counter', - min_n=args.min_n, max_n=args.max_n) - if counter_series: - # Determine counter name for labels - counter_name = 'Counter' - for bench in data.get('benchmarks', []): - if bench.get('run_type', 'iteration') != 'iteration': - continue - for k in bench.keys(): - if k not in ('name', 'run_name', 'run_type', 'repetitions', - 'repetition_index', 'threads', 'iterations', - 'real_time', 'cpu_time', 'time_unit', - 'items_per_second', 'aggregate_name', - 'aggregate_unit', 'family_index', - 'per_family_instance_index'): - counter_name = k - break - break - plot_series(counter_series, f'{args.output_prefix}_{counter_name.lower().replace("-", "_")}', - f'{counter_name} per iteration', counter_name) - -if __name__ == '__main__': - main() diff --git a/agentic/cpp/skills/cmake/SKILL.md b/agentic/cpp/skills/cmake/SKILL.md deleted file mode 100644 index a659e1a..0000000 --- a/agentic/cpp/skills/cmake/SKILL.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -name: cmake -description: Compile and build CMake projects, including configuring build types, options, and running test binaries. ---- - -# CMake Build Skill - -You now have expertise in building and configuring CMake projects. Follow these workflows: - -## Build Directory Convention - -Use a short commit hash suffix for committed revisions: - -```bash -BUILD_SUFFIX=$(git rev-parse --short HEAD) -``` - -If the worktree has uncommitted changes, append a descriptive suffix so generated -artifacts cannot be confused with a clean HEAD build: - -```bash -BUILD_SUFFIX=$(git rev-parse --short HEAD)-dirty -``` - -If not a git repository, use - -```bash -BUILD_SUFFIX=agent -``` - -Build directories follow the pattern `build/_`. - -## Using Presets (Preferred When Available) - -> **Important**: `cmake --preset` sets cache variables and generator but its `binaryDir` cannot be -> overridden from the command line. To use a preset's settings with a custom build dir, pass the -> relevant `-D` flags explicitly together with `-B`. Use `--preset` only to discover what flags a -> preset applies. - -**List available presets:** -```bash -cmake --list-presets -``` - -**Replicate a preset's settings with a custom suffix build dir:** - -Release: -```bash -cmake -B build/release_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Release -cmake --build build/release_${BUILD_SUFFIX} -j -``` - -Debug: -```bash -cmake -B build/debug_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Debug -cmake --build build/debug_${BUILD_SUFFIX} -j -``` - -AddressSanitizer: -```bash -cmake -B build/asan_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Debug -DENABLE_ADDRESS_SANITIZER=ON -cmake --build build/asan_${BUILD_SUFFIX} -j -``` - -Coverage: -```bash -cmake -B build/coverage_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Debug -DENABLE_COVERAGE=ON -cmake --build build/coverage_${BUILD_SUFFIX} -j -``` - -Benchmarks: -```bash -cmake -B build/benchmarks_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Release -DBUILD_BENCHMARKS=ON -cmake --build build/benchmarks_${BUILD_SUFFIX} -j -``` - -## Additional Feature Options - -Feature flags are project-specific. Inspect `CMakeLists.txt`, -`CMakePresets.json`, or `cmake -LAH ` before toggling options. For -repository-specific examples, check -`agentic/local/cpp/skills/cmake/EXAMPLES.md` when present. - -**Example feature toggle:** -```bash -cmake -B build/release_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Release -DENABLE_FEATURE=ON -cmake --build build/release_${BUILD_SUFFIX} -j -``` - -## Best Practices - -1. **Use out-of-source builds**: Keep build artifacts in `build/_` directories -2. **Presets fix binaryDir**: `--preset` cannot be combined with `-B` to change the build dir; replicate `-D` flags manually with `-B` instead -3. **Reconfigure when options change**: Rerun the `cmake -B ...` step when toggling options -4. **Clean build directory when needed**: Delete the entire build folder for a fresh configuration -5. **Match build type to task**: Release for performance work, Debug/ASan for correctness diff --git a/agentic/cpp/skills/diagnose-segfault/SKILL.md b/agentic/cpp/skills/diagnose-segfault/SKILL.md deleted file mode 100644 index 91c8950..0000000 --- a/agentic/cpp/skills/diagnose-segfault/SKILL.md +++ /dev/null @@ -1,195 +0,0 @@ ---- -name: diagnose-segfault -description: Diagnose C++ crashes and memory-safety errors with AddressSanitizer, GDB, and core dumps. Use when a C++ binary crashes with SIGSEGV, SIGABRT, heap-buffer-overflow, use-after-free, stack-buffer-overflow, double-free, suspected memory corruption, or an available core file. ---- - -# C++ Segfault and Memory Error Diagnosis - -Use this skill to find the first bad access or corrupting operation, not just -the frame where the process finally crashed. - -## When To Use - -- A C++ binary crashes with `Segmentation fault`, `SIGSEGV`, or `SIGABRT`. -- AddressSanitizer reports `ERROR: AddressSanitizer:`. -- A test reports heap-buffer-overflow, stack-buffer-overflow, use-after-free, - double-free, global-buffer-overflow, or similar memory-safety failures. -- A core dump exists and the user wants root-cause analysis. -- Memory corruption is suspected but the immediate failure is ambiguous. - -For repository-specific binary names, CMake options, or known reproducer -patterns, also read `agentic/local/cpp/skills/diagnose-segfault/EXAMPLES.md` -when present. - -## Workflow 1: ASan First - -Prefer AddressSanitizer when the issue is reproducible. It usually reports the -bad access with file and line information. - -### Build With ASan - -For CMake projects, first check whether the repository already has an ASan -preset or option. If not, configure a dedicated debug build: - -```bash -cmake -B build/asan -DCMAKE_BUILD_TYPE=Debug -DENABLE_ADDRESS_SANITIZER=ON -cmake --build build/asan -j -``` - -For non-CMake builds, compile and link with: - -```bash --fsanitize=address -fno-omit-frame-pointer -g -O1 -``` - -Use `-O0` instead of `-O1` when you expect to inspect many local variables in -GDB. - -### Run The Minimal Reproducer - -Run the specific binary, test case, or input that triggers the crash. For Google -Test binaries, prefer a narrow filter: - -```bash -./build/asan/unittests --gtest_filter="SuiteName.TestName" -``` - -### Read The ASan Report - -Focus on: - -| Report section | Meaning | -|---|---| -| `ERROR: AddressSanitizer: ` | Error class | -| `READ/WRITE of size N` | Access direction and size | -| First user-code frame | Exact bad access | -| Allocation/deallocation stack | Object lifetime and ownership | -| Shadow-byte legend | Boundary or lifetime category | -| `SUMMARY:` | One-line location summary | - -Useful options: - -```bash -ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=1 -ASAN_OPTIONS=halt_on_error=0 -ASAN_OPTIONS=print_stats=1 -``` - -Disable leak detection while diagnosing a crash if leak noise hides the primary -failure: - -```bash -ASAN_OPTIONS=detect_leaks=0 ./build/asan/unittests -``` - -## Workflow 2: GDB Live Debugging - -Use GDB when ASan is unavailable, when the crash is not a direct memory-safety -violation, or when variable inspection is needed. - -Build with debug symbols: - -```bash -cmake -B build/debug -DCMAKE_BUILD_TYPE=Debug -cmake --build build/debug -j -``` - -Run under GDB: - -```bash -gdb --args [arguments...] -``` - -Core commands: - -```gdb -run -bt full -info registers -info locals -info args -frame N -list -print -thread apply all bt -``` - -Make C++ values easier to inspect: - -```gdb -set print pretty on -set print object on -set pagination off -``` - -## Workflow 3: ASan Under GDB - -Use this when ASan points at a bad access but the pointer or lifetime corruption -comes from an earlier frame. - -```bash -gdb --args [arguments...] -``` - -Break on ASan reporting or abort: - -```gdb -break __asan::ReportGenericError -catch signal SIGABRT -run -bt full -``` - -Then inspect the last user-code frames before ASan internals. - -## Workflow 4: Core Dump Analysis - -Use when the crash already happened or reproduction is expensive. - -Enable core dumps for future runs if needed: - -```bash -ulimit -c unlimited -``` - -Analyze: - -```bash -gdb -``` - -Useful commands: - -```gdb -bt full -info threads -thread apply all bt -frame N -info locals -info args -print -``` - -## Common ASan Errors - -| Error type | Typical cause | -|---|---| -| `heap-buffer-overflow` | Read or write past heap allocation bounds | -| `stack-buffer-overflow` | Read or write past a local stack object | -| `global-buffer-overflow` | Read or write past global/static storage | -| `heap-use-after-free` | Access after `delete`, `free`, or container invalidation | -| `stack-use-after-return` | Pointer/reference to a returned stack frame | -| `double-free` | Object released twice | -| `alloc-dealloc-mismatch` | Mixed allocation APIs, such as `new[]` with `free` | - -## Best Practices - -1. Build with `-g`; reports without symbols are often not actionable. -2. Prefer the smallest reproducer over full-suite runs. -3. Rebuild after toggling sanitizer or debug options. -4. Treat the first ASan error as primary; later errors are often fallout. -5. Check container iterator/reference invalidation around the reported object. -6. Validate the fix with the same reproducer under ASan before running broader - tests. -7. If ASan is too slow for a large input, use GDB on the same input or reduce - the input while preserving the crash. diff --git a/agentic/cpp/skills/optimization-experiment/SKILL.md b/agentic/cpp/skills/optimization-experiment/SKILL.md deleted file mode 100644 index 75904f6..0000000 --- a/agentic/cpp/skills/optimization-experiment/SKILL.md +++ /dev/null @@ -1,193 +0,0 @@ ---- -name: optimization-experiment -description: Run iterative C++ optimization experiments for a target function or class by adding same-API experimental variants, validating correctness, benchmarking, comparing results, and deciding whether to promote a faster implementation. ---- - -# Optimization Experiment Skill - -Use this skill when a user wants to improve performance of a specific C++ -function, class, algorithm, or hot path through benchmark-driven experiments. - -This workflow depends on: - -1. `../benchmarks/SKILL.md` for Google Benchmark build/run commands, JSON output, - hardware counters, pinning, and perf profiling. -2. `../benchmarks-affected/SKILL.md` when changes need an affected benchmark - scope. -3. `../benchmarks-compare-revisions/SKILL.md` when comparing committed - revisions. - -## Goal - -Iterate from a production implementation to one or more experimental -implementations, prove semantic equivalence, measure the impact, and decide -whether a candidate is worth promoting. - -The standard loop is: - -```text -target -> benchmark baseline -> experimental same-API variant - -> correctness check -> benchmark compare -> keep / revise / discard -``` - -Stop when a candidate is clearly better on the intended workload without -correctness or maintenance regressions, or when the remaining ideas are too weak -to justify more iteration. - -## Step 1 - Identify the Target and Contract - -Start from the requested function/class and inspect the real implementation. -Record: - -- public signature/API and call sites that must not change -- input domains, invalid-input behavior, boundary conditions, and tie-breaking -- compile-time feature gates such as SIMD flags or platform-specific paths -- existing tests and reference implementations -- existing benchmarks that should move if the optimization succeeds - -Do not optimize before the contract is clear. If behavior is ambiguous, add or -find tests before changing implementation. - -## Step 2 - Establish Benchmark Coverage - -Find benchmark rows that directly exercise the target. Prefer narrow benchmark -filters over full-suite runs during iteration. - -If coverage is missing or too broad, add focused benchmark cases before adding -the optimized implementation. Include cases for: - -- the expected common path -- boundary and alignment-sensitive paths -- short, medium, and long ranges or sizes when width matters -- random or mixed workloads when real calls are not fixed-shape -- current production behavior and each experimental variant - -Capture a baseline JSON before implementation changes: - -```bash -BENCH_CPU=${BENCH_CPU:-0} -taskset -c "${BENCH_CPU}" \ - --benchmark_filter="${FILTER}" \ - --benchmark_report_aggregates_only=true \ - --benchmark_display_aggregates_only=true \ - --benchmark_out=/tmp/_baseline.json \ - --benchmark_out_format=json -``` - -Use `../benchmarks/SKILL.md` for exact build directories, Release versus -diagnostic builds, hardware-counter setup, and retry policy. - -## Step 3 - Add Experimental Same-API Variants - -Add candidate implementations beside production code in an experimental area, -namespace, header, or benchmark-local adapter that is already consistent with -the repository. - -Rules: - -- keep the callable signature/API identical to production where practical -- preserve public semantics exactly, including invalid inputs and tie-breaking -- keep production callers unchanged during experiments -- make variants benchmark-selectable by name -- avoid unrelated refactors while measuring -- keep losing variants only when they document useful evidence or support future - comparison - -For C++ libraries with feature-gated implementations, provide correct fallbacks -for unsupported targets or compile configurations. - -## Step 4 - Validate Correctness Before Timing - -Run relevant tests before trusting benchmark numbers. Add tests when the -experimental implementation introduces new risk. - -Prefer: - -- fixed edge cases for boundaries, empty/sentinel behavior, and exact ties -- randomized differential tests against a scalar or naive reference -- tests for feature-gated fallback builds when the code has SIMD or platform - branches -- targeted regression tests for any bug found during benchmarking - -Do not compare performance for a candidate that has not passed the correctness -checks for the same semantics as production. - -## Step 5 - Benchmark and Compare - -Run timing benchmarks from Release builds. Save JSON for every meaningful -baseline and candidate. - -Use diagnostic builds with hardware counters when timing changes need -explanation: - -- cycles and instructions for core execution cost -- cache counters for memory behavior -- branch counters when early exits or dispatch logic are involved - -Compare both absolute timings and relative deltas. Watch for cases where a -candidate wins the cherry-picked row but regresses neighboring or realistic -workloads. - -When results are noisy: - -- pin to a CPU with `taskset` when available -- increase repetitions or minimum benchmark time -- rerun the narrow benchmark filter once -- avoid changing benchmark scope between baseline and candidate - -## Step 6 - Iterate Deliberately - -For each candidate, decide one of: - -- **Promote**: repeatedly faster on intended rows, no important regressions, - correct and maintainable. -- **Keep experimental**: interesting or workload-specific, but not production - ready. -- **Discard**: slower, too complex, too narrow, or semantically risky. - -Use benchmark data to choose the next idea. Examples: - -- higher instruction count suggests fewer operations or simpler dispatch -- lower instructions but higher cycles suggests stalls, memory, or dependency - chains -- short-range regressions suggest a narrower dispatch condition -- alignment-sensitive rows suggest splitting aligned and unaligned paths - -When no idea wins convincingly, document the best result and stop rather than -overfitting. - -## Step 7 - Finalize the Result - -If promoting a candidate to production: - -- keep the public API unchanged unless the user explicitly requested otherwise -- keep or update tests that protect the optimized behavior -- remove accidental benchmark-only scaffolding from production code -- preserve experimental variants only when useful for future research - -If leaving work experimental: - -- add a short note near the experimental code with benchmark date, command, and - the relevant table or JSON artifact path -- clearly state that production callers do not use the experimental variant -- explain which workload the variant helps and where it loses - -The final response should include: - -- what changed -- correctness checks run -- benchmark command or JSON artifacts -- concise result table -- recommendation: promote, keep experimenting, or stop - -## Guardrails - -1. Benchmark before optimizing; otherwise there is no trustworthy baseline. -2. Never change semantics to win a benchmark. -3. Never compare Debug timings. -4. Keep production and experimental code paths distinguishable. -5. Prefer focused benchmark filters during iteration, then broaden before - promotion. -6. Treat hardware counters as explanatory data, not a replacement for timing. -7. Record enough benchmark context that future agents do not confuse - experimental wins with production behavior. diff --git a/agentic/cpp/skills/paper-search/SKILL.md b/agentic/cpp/skills/paper-search/SKILL.md deleted file mode 100644 index 27f108c..0000000 --- a/agentic/cpp/skills/paper-search/SKILL.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -name: paper-search -description: "Search for academic papers across Semantic Scholar, arXiv, and CrossRef APIs. Returns unified results with title, authors, year, abstract, DOI, venue, and citation counts. Integrates with Zotero MCP tools for adding found papers to a Zotero library and generating BibTeX entries. Use when the user asks to find papers, search for related work, look up a DOI, or discover references on a topic." ---- - -# Paper Search - -Search external academic APIs for papers. Provides a unified interface across Semantic Scholar, arXiv, and CrossRef with optional Zotero integration. - -## Workflow - -### 1. Search for Papers - -Run the search script from the skill's `scripts/` directory: - -```bash -python3 scripts/search_papers.py --query "topic" --source semantic_scholar --limit 10 --format compact -``` - -Available sources: -- `semantic_scholar` — Default. Best for comprehensive search with citation counts. -- `arxiv` — Best for preprints and recent unpublished work. -- `crossref` — Best for published works and DOI-based metadata. -- `all` — Query all three sources (slower, results combined). - -Output formats: -- `json` — Full JSON output (default). Good for programmatic use. -- `compact` — Human-readable summary with title, authors, year, venue, citations, and truncated abstract. - -### 2. DOI Lookup - -Look up a specific paper by DOI: - -```bash -python3 scripts/search_papers.py --doi "10.1145/1234567.1234568" --format compact -``` - -### 3. Download PDFs - -Download open-access PDFs directly from search results: - -```bash -python3 scripts/search_papers.py --query "wavelet tree" --source arxiv --limit 3 --download ~/papers -``` - -- arXiv papers always have PDFs available. -- Semantic Scholar provides `openAccessPdf` URLs when available. -- CrossRef may provide PDF links via publisher APIs. - -The `--download` flag adds a `downloaded_path` field to each result in JSON output. - -### 4. Add to Zotero - -**Option A: Via DOI/URL (metadata only)** - -After finding relevant papers, add them to Zotero using the Zotero MCP tools: - -- `zotero_add_by_doi` — Preferred when DOI is available. Fetches full metadata from CrossRef. -- `zotero_add_by_url` — Use for arXiv papers or when only a URL is available. - -**Option B: Via downloaded PDF (metadata + attachment)** - -Download the PDF first, then add to Zotero with the PDF file: - -```bash -# Step 1: Download PDFs and get paths in JSON -python3 scripts/search_papers.py --doi "10.1007/978-3-540-73420-8_13" --download ~/papers --format json - -# Step 2: Use zotero_zotero_add_from_file with the downloaded_path -``` - -The agent should call `zotero_zotero_add_from_file` with the `downloaded_path` from the JSON output. This attaches the PDF to the Zotero item and attempts DOI-based metadata extraction. - -**Option C: Download + Zotero in one step** - -Use `--zotero` to download PDFs with paths formatted for easy Zotero import: - -```bash -python3 scripts/search_papers.py -q "succinct data structures" -s arxiv -n 3 --zotero --download ~/papers -``` - -After adding papers, update the semantic search database: - -``` -zotero_update_search_database -``` - -### 5. Generate BibTeX - -For papers already in Zotero, use `zotero_get_item_metadata` with `format: "bibtex"` to get BibTeX entries. Alternatively, use `zotero_fetch` for full metadata. - -For papers NOT in Zotero, BibTeX can be constructed from the search results' JSON fields (`authors`, `year`, `title`, `venue`, `doi`). - -## Guidance - -- Start with `semantic_scholar` for general queries — it has the broadest coverage and citation data. -- Use `arxiv` when looking for very recent work or preprints in CS/ML/physics. -- Use `crossref` for DOI lookups or when Semantic Scholar returns no results. -- When using `--source all`, results may contain duplicates (same paper from different sources). Deduplicate by DOI or title similarity. -- Citation counts are approximate and may differ across sources. -- arXiv results return the arXiv ID (e.g., `2301.12345`) which can be used with `zotero_add_by_url` via `https://arxiv.org/abs/2301.12345`. - -## API Quirks - -- **arXiv `atom:id` is NOT a DOI** — it contains an arXiv URL like `http://arxiv.org/abs/2301.12345`. Store the extracted ID in `arxiv_id` only; set `doi` to `None` for arXiv results. Writing the arXiv URL into `doi` produces invalid DOI metadata downstream (e.g., Zotero import). -- **CrossRef `select` must include `link`** — the `link` field is needed for `pdf_url` extraction. If omitted from `select`, the API won't return link metadata and `pdf_url` will silently be empty for all CrossRef results. diff --git a/agentic/cpp/skills/paper-search/references/api_reference.md b/agentic/cpp/skills/paper-search/references/api_reference.md deleted file mode 100644 index dcb5aa5..0000000 --- a/agentic/cpp/skills/paper-search/references/api_reference.md +++ /dev/null @@ -1,32 +0,0 @@ -# External Paper Search APIs - -## Semantic Scholar - -- **Base URL**: `https://api.semanticscholar.org/graph/v1/` -- **Rate limit**: 1 req/sec without API key, 10 req/sec with key -- **No auth required** for basic usage -- **Fields**: title, authors, year, abstract, externalIds (DOI, ArXiv), venue, citationCount -- **Best for**: Comprehensive academic search with citation counts - -## arXiv - -- **Base URL**: `http://export.arxiv.org/api/query` -- **Rate limit**: Be nice, ~3 sec between requests -- **No auth required** -- **Returns**: XML (Atom feed) -- **Best for**: Preprints, recent work not yet published - -## CrossRef - -- **Base URL**: `https://api.crossref.org/` -- **Rate limit**: 50 req/sec with polite pool (include `mailto` header) -- **No auth required** -- **Best for**: DOI lookup, published works, metadata enrichment - -## Zotero Integration - -After finding papers via external search, use Zotero MCP tools: - -1. `zotero_add_by_doi` — Add paper by DOI (fetches metadata from CrossRef) -2. `zotero_add_by_url` — Add paper by URL (arXiv, DOI URLs) -3. `zotero_update_search_database` — Update semantic search index after adding diff --git a/agentic/cpp/skills/paper-search/scripts/search_papers.py b/agentic/cpp/skills/paper-search/scripts/search_papers.py deleted file mode 100755 index c65351d..0000000 --- a/agentic/cpp/skills/paper-search/scripts/search_papers.py +++ /dev/null @@ -1,333 +0,0 @@ -#!/usr/bin/env python3 -"""Search external APIs for academic papers. - -Sources: Semantic Scholar, arXiv, CrossRef. -Outputs unified JSON to stdout. - -Usage: - python3 search_papers.py --query "wavelet tree succinct" --source semantic_scholar --limit 10 - python3 search_papers.py --query "succinct data structures" --source arxiv --limit 5 - python3 search_papers.py --doi "10.1145/123" --source crossref - python3 search_papers.py --query "rank select" --source all --limit 5 - python3 search_papers.py --query "wavelet tree" --source arxiv --limit 1 --download ~/papers - python3 search_papers.py --doi "10.1007/978-3-540-73420-8_13" --download ~/papers --zotero -""" - -import argparse -import json -import os -import re -import sys -import time -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path -from typing import Any - - -def _get(url: str, headers: dict[str, str] | None = None, timeout: int = 30, - retries: int = 2) -> dict: - for attempt in range(retries + 1): - req = urllib.request.Request(url, headers=headers or {}) - try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - return json.loads(resp.read().decode()) - except urllib.error.HTTPError as e: - if e.code == 429 and attempt < retries: - wait = 2 ** attempt - print(f"Rate limited, retrying in {wait}s...", file=sys.stderr) - time.sleep(wait) - continue - print(f"HTTP {e.code}: {e.reason} for {url}", file=sys.stderr) - return {} - except urllib.error.URLError as e: - print(f"URL error: {e.reason} for {url}", file=sys.stderr) - return {} - return {} - - -def search_semantic_scholar(query: str, limit: int = 10) -> list[dict[str, Any]]: - """Search Semantic Scholar API.""" - params = urllib.parse.urlencode({ - "query": query, - "limit": limit, - "fields": "title,authors,year,abstract,externalIds,venue,publicationDate,citationCount,url,openAccessPdf", - }) - url = f"https://api.semanticscholar.org/graph/v1/paper/search?{params}" - data = _get(url, headers={"Accept": "application/json"}) - results = [] - for paper in data.get("data", []): - ext_ids = paper.get("externalIds") or {} - pdf_info = paper.get("openAccessPdf") or {} - results.append({ - "source": "semantic_scholar", - "title": paper.get("title", ""), - "authors": [a.get("name", "") for a in paper.get("authors", [])], - "year": paper.get("year"), - "abstract": paper.get("abstract", ""), - "doi": ext_ids.get("DOI"), - "arxiv_id": ext_ids.get("ArXiv"), - "venue": paper.get("venue", ""), - "citation_count": paper.get("citationCount"), - "url": paper.get("url", ""), - "pdf_url": pdf_info.get("url"), - }) - return results - - -def search_arxiv(query: str, limit: int = 10) -> list[dict[str, Any]]: - """Search arXiv API.""" - words = query.split() - if len(words) == 1: - search_term = f"all:{query}" - elif len(words) == 2: - # Phrase search for 2-word queries - search_term = f'all:"{query}"' - else: - # Use OR of phrase and individual terms for 3+ words - # This catches exact phrase matches AND papers with all terms - phrase = f'all:"{query}"' - and_terms = " AND ".join(f"all:{w}" for w in words) - search_term = f"({phrase}) OR ({and_terms})" - params = urllib.parse.urlencode({ - "search_query": search_term, - "start": 0, - "max_results": limit, - }) - url = f"http://export.arxiv.org/api/query?{params}" - req = urllib.request.Request(url) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - xml_data = resp.read().decode() - except (urllib.error.URLError, urllib.error.HTTPError) as e: - print(f"arXiv API error: {e}", file=sys.stderr) - return [] - - import xml.etree.ElementTree as ET - root = ET.fromstring(xml_data) - ns = {"atom": "http://www.w3.org/2005/Atom"} - results = [] - for entry in root.findall("atom:entry", ns): - title = entry.findtext("atom:title", "", ns).strip().replace("\n", " ") - abstract = entry.findtext("atom:summary", "", ns).strip().replace("\n", " ") - authors = [a.findtext("atom:name", "", ns) for a in entry.findall("atom:author", ns)] - published = entry.findtext("atom:published", "", ns) - year = int(published[:4]) if published else None - arxiv_id = "" - for link in entry.findall("atom:link", ns): - href = link.get("href", "") - if "arxiv.org/abs/" in href: - arxiv_id = href.split("/abs/")[-1] - break - results.append({ - "source": "arxiv", - "title": title, - "authors": authors, - "year": year, - "abstract": abstract, - "doi": None, - "arxiv_id": arxiv_id, - "venue": "arXiv", - "citation_count": None, - "url": f"https://arxiv.org/abs/{arxiv_id}" if arxiv_id else "", - "pdf_url": f"https://arxiv.org/pdf/{arxiv_id}" if arxiv_id else None, - }) - return results - - -def search_crossref(query: str, limit: int = 10) -> list[dict[str, Any]]: - """Search CrossRef API.""" - params = urllib.parse.urlencode({ - "query": query, - "rows": limit, - "select": "DOI,title,author,published-print,abstract,container-title,is-referenced-by-count,URL,type,link", - }) - url = f"https://api.crossref.org/works?{params}" - data = _get(url, headers={"Accept": "application/json"}) - results = [] - for item in data.get("message", {}).get("items", []): - title_list = item.get("title", []) - title = title_list[0] if title_list else "" - authors = [] - for a in item.get("author", []): - name = f"{a.get('given', '')} {a.get('family', '')}".strip() - if name: - authors.append(name) - pub_date = item.get("published-print", {}).get("date-parts", [[None]]) - year = pub_date[0][0] if pub_date and pub_date[0] else None - venue_list = item.get("container-title", []) - venue = venue_list[0] if venue_list else "" - pdf_url = None - for link in item.get("link", []): - if "pdf" in link.get("content-type", ""): - pdf_url = link.get("URL") - break - results.append({ - "source": "crossref", - "title": title, - "authors": authors, - "year": year, - "abstract": item.get("abstract", ""), - "doi": item.get("DOI"), - "arxiv_id": None, - "venue": venue, - "citation_count": item.get("is-referenced-by-count"), - "url": item.get("URL", ""), - "pdf_url": pdf_url, - }) - return results - - -def lookup_doi(doi: str) -> dict[str, Any] | None: - """Look up a single paper by DOI via CrossRef.""" - url = f"https://api.crossref.org/works/{urllib.parse.quote(doi, safe='')}" - data = _get(url) - item = data.get("message") - if not item: - return None - title_list = item.get("title", []) - title = title_list[0] if title_list else "" - authors = [] - for a in item.get("author", []): - name = f"{a.get('given', '')} {a.get('family', '')}".strip() - if name: - authors.append(name) - pub_date = item.get("published-print", {}).get("date-parts", [[None]]) - year = pub_date[0][0] if pub_date and pub_date[0] else None - venue_list = item.get("container-title", []) - venue = venue_list[0] if venue_list else "" - pdf_url = None - for link in item.get("link", []): - if "pdf" in link.get("content-type", ""): - pdf_url = link.get("URL") - break - return { - "source": "crossref", - "title": title, - "authors": authors, - "year": year, - "abstract": item.get("abstract", ""), - "doi": item.get("DOI"), - "arxiv_id": None, - "venue": venue, - "citation_count": item.get("is-referenced-by-count"), - "url": item.get("URL", ""), - "pdf_url": pdf_url, - } - - -SOURCES = { - "semantic_scholar": search_semantic_scholar, - "arxiv": search_arxiv, - "crossref": search_crossref, -} - - -def _sanitize_filename(title: str) -> str: - """Generate a clean filename from paper title.""" - name = re.sub(r'[^\w\s-]', '', title.lower()) - name = re.sub(r'[\s]+', '_', name.strip()) - return name[:80] - - -def download_pdf(url: str, output_dir: str, paper: dict[str, Any]) -> str | None: - """Download a PDF and return the local path.""" - filename = _sanitize_filename(paper.get("title", "paper")) + ".pdf" - output_path = Path(output_dir) / filename - output_path.parent.mkdir(parents=True, exist_ok=True) - - req = urllib.request.Request(url, headers={ - "User-Agent": "Mozilla/5.0 (academic paper-search script)" - }) - try: - with urllib.request.urlopen(req, timeout=60) as resp: - content_type = resp.headers.get("Content-Type", "") - if "pdf" not in content_type and "octet-stream" not in content_type: - print(f"Warning: unexpected content type '{content_type}' for {url}", - file=sys.stderr) - with open(output_path, "wb") as f: - f.write(resp.read()) - print(f"Downloaded: {output_path}", file=sys.stderr) - return str(output_path) - except (urllib.error.URLError, urllib.error.HTTPError) as e: - print(f"Download failed for {url}: {e}", file=sys.stderr) - return None - - -def main(): - parser = argparse.ArgumentParser(description="Search for academic papers") - parser.add_argument("--query", "-q", help="Search query") - parser.add_argument("--doi", help="Look up a specific DOI") - parser.add_argument("--source", "-s", default="semantic_scholar", - choices=["semantic_scholar", "arxiv", "crossref", "all"], - help="API source (default: semantic_scholar)") - parser.add_argument("--limit", "-n", type=int, default=10, - help="Max results per source (default: 10)") - parser.add_argument("--format", "-f", default="json", - choices=["json", "compact"], - help="Output format (default: json)") - parser.add_argument("--download", "-d", metavar="DIR", - help="Download PDFs to DIR (requires pdf_url in results)") - parser.add_argument("--zotero", "-z", action="store_true", - help="Download PDFs and output paths for Zotero import (implies --download)") - args = parser.parse_args() - - if not args.query and not args.doi: - parser.error("Either --query or --doi is required") - - if args.zotero and not args.download: - args.download = "." - - results = [] - if args.doi: - paper = lookup_doi(args.doi) - if paper: - results.append(paper) - elif args.source == "all": - for name, func in SOURCES.items(): - try: - results.extend(func(args.query, args.limit)) - except Exception as e: - print(f"Error searching {name}: {e}", file=sys.stderr) - time.sleep(1) - else: - results = SOURCES[args.source](args.query, args.limit) - - if args.download: - for r in results: - pdf_url = r.get("pdf_url") - if pdf_url: - path = download_pdf(pdf_url, args.download, r) - r["downloaded_path"] = path - else: - r["downloaded_path"] = None - - if args.format == "json": - print(json.dumps(results, indent=2)) - else: - for i, r in enumerate(results, 1): - authors = ", ".join(r["authors"][:3]) - if len(r["authors"]) > 3: - authors += " et al." - doi_str = f" DOI: {r['doi']}" if r.get("doi") else "" - arxiv_str = f" arXiv: {r['arxiv_id']}" if r.get("arxiv_id") else "" - cite_str = f" Citations: {r['citation_count']}" if r.get("citation_count") else "" - pdf_str = f" PDF: {r['pdf_url']}" if r.get("pdf_url") else " PDF: N/A" - dl_str = "" - if r.get("downloaded_path"): - dl_str = f" Downloaded: {r['downloaded_path']}" - print(f"[{i}] {r['title']}") - print(f" {authors} ({r.get('year', '?')}) — {r.get('venue', '')}") - print(f" {r.get('url', '')}{doi_str}{arxiv_str}{cite_str}") - print(f" {pdf_str}{dl_str}") - if r.get("abstract"): - abstract = r["abstract"][:200] - if len(r["abstract"]) > 200: - abstract += "..." - print(f" {abstract}") - print() - - -if __name__ == "__main__": - main() diff --git a/agentic/cpp/skills/pdf/SKILL.md b/agentic/cpp/skills/pdf/SKILL.md deleted file mode 100644 index ddbce00..0000000 --- a/agentic/cpp/skills/pdf/SKILL.md +++ /dev/null @@ -1,112 +0,0 @@ ---- -name: pdf -description: Process PDF files - extract text, create PDFs, merge documents. Use when user asks to read PDF, create PDF, or work with PDF files. ---- - -# PDF Processing Skill - -You now have expertise in PDF manipulation. Follow these workflows: - -## Reading PDFs - -**Option 1: Quick text extraction (preferred)** -```bash -# Using pdftotext (poppler-utils) -pdftotext input.pdf - # Output to stdout -pdftotext input.pdf output.txt # Output to file - -# If pdftotext not available, try: -python3 -c " -import fitz # PyMuPDF -doc = fitz.open('input.pdf') -for page in doc: - print(page.get_text()) -" -``` - -**Option 2: Page-by-page with metadata** -```python -import fitz # pip install pymupdf - -doc = fitz.open("input.pdf") -print(f"Pages: {len(doc)}") -print(f"Metadata: {doc.metadata}") - -for i, page in enumerate(doc): - text = page.get_text() - print(f"--- Page {i+1} ---") - print(text) -``` - -## Creating PDFs - -**Option 1: From Markdown (recommended)** -```bash -# Using pandoc -pandoc input.md -o output.pdf - -# With custom styling -pandoc input.md -o output.pdf --pdf-engine=xelatex -V geometry:margin=1in -``` - -**Option 2: Programmatically** -```python -from reportlab.lib.pagesizes import letter -from reportlab.pdfgen import canvas - -c = canvas.Canvas("output.pdf", pagesize=letter) -c.drawString(100, 750, "Hello, PDF!") -c.save() -``` - -**Option 3: From HTML** -```bash -# Using wkhtmltopdf -wkhtmltopdf input.html output.pdf - -# Or with Python -python3 -c " -import pdfkit -pdfkit.from_file('input.html', 'output.pdf') -" -``` - -## Merging PDFs - -```python -import fitz - -result = fitz.open() -for pdf_path in ["file1.pdf", "file2.pdf", "file3.pdf"]: - doc = fitz.open(pdf_path) - result.insert_pdf(doc) -result.save("merged.pdf") -``` - -## Splitting PDFs - -```python -import fitz - -doc = fitz.open("input.pdf") -for i in range(len(doc)): - single = fitz.open() - single.insert_pdf(doc, from_page=i, to_page=i) - single.save(f"page_{i+1}.pdf") -``` - -## Key Libraries - -| Task | Library | Install | -|------|---------|---------| -| Read/Write/Merge | PyMuPDF | `pip install pymupdf` | -| Create from scratch | ReportLab | `pip install reportlab` | -| HTML to PDF | pdfkit | `pip install pdfkit` + wkhtmltopdf | -| Text extraction | pdftotext | `brew install poppler` / `apt install poppler-utils` | - -## Best Practices - -1. **Always check if tools are installed** before using them -2. **Handle encoding issues** - PDFs may contain various character encodings -3. **Large PDFs**: Process page by page to avoid memory issues -4. **OCR for scanned PDFs**: Use `pytesseract` if text extraction returns empty diff --git a/agentic/cpp/skills/setup-cpp-repo/SKILL.md b/agentic/cpp/skills/setup-cpp-repo/SKILL.md deleted file mode 100644 index 83201a4..0000000 --- a/agentic/cpp/skills/setup-cpp-repo/SKILL.md +++ /dev/null @@ -1,136 +0,0 @@ ---- -name: setup-cpp-repo -description: Scaffold a new C++20 repository with CMake, Google Test, Google Benchmark, CI workflows, Doxygen docs, and Chromium code style. Use when the user asks to create a new C++ project, set up a C++ library, or initialize a C++ repository with modern tooling. ---- - -# setup-cpp-repo - -## Overview - -This skill generates a complete modern C++20 project scaffold. The generated -repository is header-only by default and includes: - -- CMake build system with presets -- Google Test for unit testing -- Google Benchmark for performance benchmarks -- Doxygen documentation with doxygen-awesome-css theme -- GitHub Actions CI workflows (ASan, lint, coverage, docs) -- Chromium C++ code style via `.clang-format` -- `AGENTS.md` for AI coding assistant guidelines - -## When to Use This Skill - -Use this skill when: -- The user wants to create a new C++ library or project from scratch -- The user asks for a "C++ project template" or "C++ repo setup" -- The user needs CMake + Google Test + benchmark scaffolding -- The user wants header-only library conventions with optional SIMD-oriented - build flags and Doxygen docs - -Do **not** use this skill when: -- Working with an existing codebase (use the `cmake` skill instead) -- The project is not C++ (use a different skill) -- The user only wants a single file or snippet - -## Workflow - -### Step 1: Gather Parameters - -Ask the user for (or infer from context): -- **Project name** (required): Hyphenated lowercase identifier, e.g., `my-lib` -- **Namespace** (optional): C++ namespace. Defaults to project name with hyphens removed, e.g., `mylib` -- **Output directory** (optional): Where to create the project. Defaults to current directory. - -### Step 2: Run the Generator - -Execute the generation script: - -```bash -python3 agentic/cpp/skills/setup-cpp-repo/scripts/init_cpp_project.py \ - --name \ - [--namespace ] \ - [--output-dir ] -``` - -For concrete examples, check -`agentic/local/cpp/skills/setup-cpp-repo/EXAMPLES.md` when present. - -### Step 3: Verify the Scaffold - -After generation, the project structure should look like: - -``` -/ -├── CMakeLists.txt -├── CMakePresets.json -├── .clang-format -├── .gitignore -├── README.md -├── AGENTS.md -├── include/ -│ └── / -│ └── .hpp -├── src/ -│ ├── tests/ -│ │ └── unittests.cpp -│ ├── benchmarks/ -│ │ └── benchmarks.cpp -│ └── docs/ -│ ├── Doxyfile.in -│ └── images/ -├── scripts/ -│ └── coverage_report.sh -└── .github/ - └── workflows/ - ├── build-test.yml - ├── linter.yml - ├── coverage.yml - └── doxygen.yml -``` - -### Step 4: Initial Build and Test - -Change into the project directory and run an initial build to verify everything works: - -```bash -cd -cmake --preset release -cmake --build --preset release -j -./build/release/unittests -``` - -If the build and tests pass, the scaffold is ready. - -### Step 5: Hand Off to cmake Skill - -After project creation, use the **`cmake` skill** (`../cmake/SKILL.md`) for all subsequent build operations. The `cmake` skill documents: -- Build directory conventions with git short-hash suffixes -- How to replicate preset settings with custom build directories -- AddressSanitizer, coverage, and benchmark workflows -- Best practices for out-of-source builds - -## Customization Guide - -### Adding More Test Executables - -Edit `CMakeLists.txt` and add new `add_executable` blocks under the `if(_TESTS)` section, following the pattern of the existing `unittests` target. - -Update `scripts/coverage_report.sh` to run any new test binaries. - -Update `.github/workflows/build-test.yml` to execute new test binaries in CI. - -### Adding More Benchmark Executables - -Edit `CMakeLists.txt` and add new `add_executable` blocks under the `if(_BENCHMARKS)` section, following the pattern of the existing `benchmarks` target. - -### Adding Third-Party Dependencies - -For header-only libraries, prefer `FetchContent` in `CMakeLists.txt`. For compiled libraries, consider vendoring or using a package manager (Conan, vcpkg). - -### Modifying Doxygen Configuration - -Edit `src/docs/Doxyfile.in`. The generated version is intentionally minimal (only non-default settings). Add or override settings as needed. Run `doxygen -g` to see all available options. - -## Reference - -See `references/project_structure.md` for a detailed breakdown of every generated file and its purpose. diff --git a/agentic/cpp/skills/setup-cpp-repo/references/project_structure.md b/agentic/cpp/skills/setup-cpp-repo/references/project_structure.md deleted file mode 100644 index 6bf3236..0000000 --- a/agentic/cpp/skills/setup-cpp-repo/references/project_structure.md +++ /dev/null @@ -1,117 +0,0 @@ -# Generated Project Structure Reference - -This document describes every file and directory generated by `init_cpp_project.py` and its purpose. - -## Root Files - -### `CMakeLists.txt` -Main CMake configuration. Defines: -- C++20 standard requirements -- `MARCH` cache variable (defaults to `native`) -- Optional SIMD fallback flag when the generated project enables SIMD-specific - code paths -- `ENABLE_ADDRESS_SANITIZER` option for ASan builds -- `_COVERAGE` option for gcov instrumentation -- Build options: `_TESTS`, `_BENCHMARKS`, `_DIAGNOSTICS`, `_DOCS` -- FetchContent dependencies: Google Test, Google Benchmark, spdlog (diagnostics only), Doxygen theme -- Test executable: `unittests` -- Benchmark executable: `benchmarks` -- Custom target: `docs` (when Doxygen is enabled) - -### `CMakePresets.json` -CMake presets (version 4) with a hidden `base` preset. Defines presets for: -- `debug` — Debug build -- `release` — Release build -- `benchmarks` — Release with benchmarks enabled -- `benchmarks-diagnostic` — RelWithDebInfo with diagnostics and libpfm -- `docs` — Documentation build -- `coverage` — Debug with coverage instrumentation -- `asan` — Debug with AddressSanitizer - -### `.clang-format` -Chromium-based C++ formatting configuration. Simplified from the full Chromium style by removing Windows-specific include priorities and IPC macro block definitions. Key settings: -- `BasedOnStyle: Chromium` -- `Standard: Cpp11` -- `InsertBraces: true` -- `InsertNewlineAtEOF: true` -- `IncludeBlocks: Regroup` with generic priority categories - -### `.gitignore` -Standard C++ project ignores: -- `build/`, `.vscode/`, `Testing/` -- `plans/*`, `venv/`, `docs/*` -- `CMakeUserPresets.json` -- `_deps/`, gcov outputs (`*.gcda`, `*.gcno`, `*.gcov`) - -### `README.md` -Minimal project README used as the Doxygen main page. - -### `AGENTS.md` -Project documentation for AI coding assistants. Contains: -- Project overview and architecture conventions -- Technology stack (C++20, CMake, Google Test, Google Benchmark) -- Build commands with all CMake options -- Testing patterns and style guidelines -- Common tasks for AI agents (adding components, modifying SIMD code, adding tests) -- Performance philosophy - -## Directories - -### `include//` -Header-only library API. Contains a placeholder header (`.hpp`) with: -- Doxygen file documentation -- Example function in the project's namespace -- `#pragma once` guard - -### `src/tests/` -Unit test scaffold. Contains `unittests.cpp` with: -- Google Test includes -- Basic assertion test against the placeholder header -- `gtest_main` supplies the test runner entry point - -### `src/benchmarks/` -Benchmark scaffold. Contains `benchmarks.cpp` with: -- Google Benchmark includes -- Example benchmark using `benchmark::DoNotOptimize` -- `BENCHMARK_MAIN()` macro - -### `src/docs/` -Doxygen configuration. Contains: -- `Doxyfile.in` — Trimmed Doxygen config (~300 lines vs. 1100+ in full). Only non-default settings are specified. Key templated values: - - `PROJECT_NAME` - - `INPUT` (points to `include/` and `README.md`) - - `STRIP_FROM_PATH` (strips source dir from file paths) - - `IMAGE_PATH` - - `HTML_EXTRA_STYLESHEET` (doxygen-awesome-css) - - `USE_MDFILE_AS_MAINPAGE` -- `images/` — Empty directory for documentation images - -### `scripts/` -Utility scripts. Contains: -- `coverage_report.sh` — Runs the `coverage` CMake preset, executes tests, and generates gcov reports. Excludes `_deps/`, `third_party/`, and `src/benchmarks/` from coverage. - -### `.github/workflows/` -CI/CD workflows: - -#### `build-test.yml` -Builds the project with AddressSanitizer and runs unit tests on `ubuntu-latest`. Triggered on pushes and PRs to `main`. - -#### `linter.yml` -Runs `clang-format --dry-run --Werror` on all C/C++ files. Triggered on pushes to `main` and all PRs. - -#### `coverage.yml` -Runs the coverage script and uploads results to Codecov. Also uploads coverage artifacts. Triggered on pushes and PRs to `main`. - -#### `doxygen.yml` -Installs Doxygen, builds documentation with the `docs` preset, and deploys HTML output to GitHub Pages. Triggered on pushes to `main` and manual dispatch. - -## Template Substitution - -All generated files use these placeholders, replaced by the script: - -| Placeholder | Example input | Example output | -|-------------|---------------|----------------| -| `{{PROJECT_NAME}}` | `my-lib` | `my-lib` | -| `{{NAMESPACE}}` | `mylib` | `mylib` | -| `{{PROJECT_NAME_UPPER}}` | `MY_LIB` | `MY_LIB` | -| `{{HEADER_NAME}}` | `my_lib.hpp` | `my_lib.hpp` | diff --git a/agentic/cpp/skills/setup-cpp-repo/scripts/init_cpp_project.py b/agentic/cpp/skills/setup-cpp-repo/scripts/init_cpp_project.py deleted file mode 100755 index d76624f..0000000 --- a/agentic/cpp/skills/setup-cpp-repo/scripts/init_cpp_project.py +++ /dev/null @@ -1,1053 +0,0 @@ -#!/usr/bin/env python3 -""" -init_cpp_project.py - Scaffold a new C++20 repository following modern C++ conventions. - -Usage: - init_cpp_project.py --name [--namespace ] [--output-dir ] - -Example: - init_cpp_project.py --name my-lib --namespace mylib --output-dir . -""" - -import argparse -import os -import sys -from pathlib import Path - - -# --------------------------------------------------------------------------- -# Helper functions -# --------------------------------------------------------------------------- - -def to_upper(name: str) -> str: - """Convert project name to uppercase with underscores.""" - return name.replace("-", "_").upper() - - -def to_snake(name: str) -> str: - """Convert project name to snake_case for filenames.""" - return name.replace("-", "_") - - -# --------------------------------------------------------------------------- -# Templates -# --------------------------------------------------------------------------- - -CMAKE_LISTS_TXT = """cmake_minimum_required(VERSION 3.18) -project({{PROJECT_NAME}}) - -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) - -set(MARCH "native" CACHE STRING "march compiler flag") -add_compile_options("-march=${MARCH}") -message(STATUS "MARCH is '${MARCH}'") - -option({{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD "Disable wide SIMD instructions" OFF) -if({{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD) - add_compile_options("-mno-avx512f") - message(STATUS "{{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD is ON") -endif() - -option(ENABLE_ADDRESS_SANITIZER "Enable AddressSanitizer" OFF) -if(ENABLE_ADDRESS_SANITIZER) - add_compile_options(-fsanitize=address -fno-omit-frame-pointer) - add_link_options(-fsanitize=address) - message(STATUS "AddressSanitizer is ON") -endif() - -option({{PROJECT_NAME_UPPER}}_COVERAGE "Enable coverage instrumentation" OFF) -if({{PROJECT_NAME_UPPER}}_COVERAGE) - add_compile_options(-O0 -g --coverage) - add_link_options(--coverage) - message(STATUS "Coverage instrumentation is ON") -endif() - -# --------------------------------------------------------------------------- -# Build options -# --------------------------------------------------------------------------- -option({{PROJECT_NAME_UPPER}}_TESTS "Build unit tests" ON) -option({{PROJECT_NAME_UPPER}}_BENCHMARKS "Build benchmarks" OFF) -option({{PROJECT_NAME_UPPER}}_DIAGNOSTICS "Include diagnostic logs" OFF) -option({{PROJECT_NAME_UPPER}}_DOCS "Build Doxygen documentation" OFF) - -if({{PROJECT_NAME_UPPER}}_DIAGNOSTICS) - add_compile_definitions({{PROJECT_NAME_UPPER}}_DIAGNOSTICS) - set({{PROJECT_NAME_UPPER}}_DIAGNOSTICS_LIBS spdlog::spdlog_header_only) -endif() - -# --------------------------------------------------------------------------- -# Dependencies (fetched only when needed) -# --------------------------------------------------------------------------- -include(FetchContent) - -if({{PROJECT_NAME_UPPER}}_DIAGNOSTICS) - set(SPDLOG_BUILD_SHARED OFF CACHE BOOL "" FORCE) - set(SPDLOG_BUILD_EXAMPLE OFF CACHE BOOL "" FORCE) - set(SPDLOG_BUILD_TESTING OFF CACHE BOOL "" FORCE) - set(SPDLOG_INSTALL OFF CACHE BOOL "" FORCE) - FetchContent_Declare( - spdlog - GIT_REPOSITORY https://github.com/gabime/spdlog.git - GIT_TAG v1.14.1 - ) - FetchContent_MakeAvailable(spdlog) -endif() - -if({{PROJECT_NAME_UPPER}}_BENCHMARKS) - FetchContent_Declare( - googlebenchmark - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.9.4 - ) - set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable Google Benchmark tests") - FetchContent_MakeAvailable(googlebenchmark) -endif() - -if({{PROJECT_NAME_UPPER}}_TESTS) - if(NOT TARGET gtest_main) - FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.17.0 - ) - FetchContent_MakeAvailable(googletest) - endif() - include(GoogleTest) -endif() - -# --------------------------------------------------------------------------- -# Unit tests -# --------------------------------------------------------------------------- -if({{PROJECT_NAME_UPPER}}_TESTS) - enable_testing() - - add_executable(unittests - src/tests/unittests.cpp) - target_include_directories(unittests - PUBLIC include) - target_link_libraries(unittests - gtest_main - ${{{PROJECT_NAME_UPPER}}_DIAGNOSTICS_LIBS}) - gtest_discover_tests(unittests) -endif() - -# --------------------------------------------------------------------------- -# Benchmarks -# --------------------------------------------------------------------------- -if({{PROJECT_NAME_UPPER}}_BENCHMARKS) - add_executable(benchmarks - src/benchmarks/benchmarks.cpp) - target_include_directories(benchmarks - PUBLIC include) - target_link_libraries(benchmarks - benchmark - benchmark_main - ${{{PROJECT_NAME_UPPER}}_DIAGNOSTICS_LIBS}) -endif() - -# --------------------------------------------------------------------------- -# Documentation (Doxygen) -# --------------------------------------------------------------------------- -if({{PROJECT_NAME_UPPER}}_DOCS) - find_package(Doxygen REQUIRED) - - FetchContent_Declare( - doxygen-awesome-css - URL https://github.com/jothepro/doxygen-awesome-css/archive/refs/heads/main.zip - ) - FetchContent_MakeAvailable(doxygen-awesome-css) - - FetchContent_GetProperties(doxygen-awesome-css SOURCE_DIR AWESOME_CSS_DIR) - - set(DOXYFILE_IN ${CMAKE_CURRENT_SOURCE_DIR}/src/docs/Doxyfile.in) - set(DOXYFILE_OUT ${CMAKE_CURRENT_BINARY_DIR}/docs/Doxyfile) - configure_file(${DOXYFILE_IN} ${DOXYFILE_OUT} @ONLY) - - add_custom_target(docs - COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_OUT} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Generating API documentation with Doxygen" - VERBATIM) -endif() -""" - -CMAKE_PRESETS_JSON = """{ - "version": 4, - "cmakeMinimumRequired": { - "major": 3, - "minor": 18, - "patch": 0 - }, - "configurePresets": [ - { - "name": "base", - "hidden": true, - "cacheVariables": { - "CMAKE_EXPORT_COMPILE_COMMANDS": "ON" - } - }, - { - "name": "debug", - "displayName": "Debug", - "inherits": "base", - "binaryDir": "${sourceDir}/build/debug", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" - } - }, - { - "name": "release", - "displayName": "Release", - "inherits": "base", - "binaryDir": "${sourceDir}/build/release", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" - } - }, - { - "name": "benchmarks", - "displayName": "Benchmarks", - "inherits": "base", - "binaryDir": "${sourceDir}/build/benchmarks", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release", - "{{PROJECT_NAME_UPPER}}_BENCHMARKS": "ON" - } - }, - { - "name": "benchmarks-diagnostic", - "displayName": "Benchmarks diagnostic build", - "inherits": "base", - "binaryDir": "${sourceDir}/build/release-with-deb", - "cacheVariables": { - "BENCHMARK_ENABLE_LIBPFM": "ON", - "CMAKE_BUILD_TYPE": "RelWithDebInfo", - "{{PROJECT_NAME_UPPER}}_DIAGNOSTICS": "ON", - "{{PROJECT_NAME_UPPER}}_BENCHMARKS": "ON" - } - }, - { - "name": "docs", - "displayName": "Docs", - "inherits": "base", - "binaryDir": "${sourceDir}/build/docs", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release", - "{{PROJECT_NAME_UPPER}}_DOCS": "ON" - } - }, - { - "name": "coverage", - "displayName": "Coverage", - "inherits": "base", - "binaryDir": "${sourceDir}/build/coverage", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "{{PROJECT_NAME_UPPER}}_BENCHMARKS": "OFF", - "{{PROJECT_NAME_UPPER}}_COVERAGE": "ON" - } - }, - { - "name": "asan", - "displayName": "AddressSanitizer", - "inherits": "base", - "binaryDir": "${sourceDir}/build/asan", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "{{PROJECT_NAME_UPPER}}_BENCHMARKS": "OFF", - "ENABLE_ADDRESS_SANITIZER": "ON" - } - } - ], - "buildPresets": [ - { - "name": "debug", - "displayName": "Build Debug", - "configurePreset": "debug" - }, - { - "name": "release", - "displayName": "Build Release", - "configurePreset": "release" - }, - { - "name": "benchmarks", - "displayName": "Build Benchmarks", - "configurePreset": "benchmarks" - }, - { - "name": "benchmarks-diagnostic", - "displayName": "Benchmarks diagnostic", - "configurePreset": "benchmarks-diagnostic" - }, - { - "name": "docs", - "displayName": "Build Docs", - "configurePreset": "docs", - "targets": [ - "docs" - ] - }, - { - "name": "coverage", - "displayName": "Build Coverage", - "configurePreset": "coverage" - }, - { - "name": "asan", - "displayName": "Build AddressSanitizer", - "configurePreset": "asan" - } - ] -} -""" - -CLANG_FORMAT = """# Defines the Chromium style for automatic reformatting. -# http://clang.llvm.org/docs/ClangFormatStyleOptions.html -BasedOnStyle: Chromium -# This defaults to 'Auto'. Explicitly set it for a while, so that -# 'vector >' in existing files gets formatted to -# 'vector>'. ('Auto' means that clang-format will only use -# 'int>>' if the file already contains at least one such instance.) -Standard: Cpp11 - -# TODO(crbug.com/1392808): Remove when InsertBraces has been upstreamed into -# the Chromium style (is implied by BasedOnStyle: Chromium). -InsertBraces: true -InsertNewlineAtEOF: true - -# Sort #includes by following -# https://google.github.io/styleguide/cppguide.html#Names_and_Order_of_Includes -IncludeBlocks: Regroup -IncludeCategories: - # C system headers. - - Regex: '^<.*\\.h>' - Priority: 1 - # C++ standard library headers. - - Regex: '^<.*>' - Priority: 2 - # Project headers (quoted includes). - - Regex: '^".*"' - Priority: 3 - # Other libraries. - - Regex: '.*' - Priority: 4 -""" - -GITIGNORE = """build/ -.vscode/ -Testing/ -plans/* -venv/ -docs/* -src/docs/presentations/* -CMakeUserPresets.json -_deps/ -*.gcda -*.gcno -*.gcov -""" - -README_MD = """# {{PROJECT_NAME}} - -{{PROJECT_NAME}} is a C++20 header-only library. - -## Build - -```bash -cmake --preset release -cmake --build --preset release -j -./build/release/unittests -``` -""" - -AGENTS_MD = """# AGENTS.md - AI Coding Assistant Guidelines for {{PROJECT_NAME}} - -## Project Overview - -{{PROJECT_NAME}} is a **C++20 header-only library**. It provides [TODO: brief description]. - -## Skills - -Shared C++ agent skills live in `agentic/cpp/skills` when this repository -vendors the shared skills subtree. Project-specific examples live in -`agentic/local/cpp/skills`. - -## Architecture - -### Project Layout Conventions - -- **`include/`**: Header-only library API (all implementations here, no `.cpp` files) -- **`src/*_tests.cpp`**: Unit tests (Google Test) -- **`src/*_benchmarks.cpp`**: Performance benchmarks (Google Benchmark) -- **`src/docs/`**: Doxygen configuration - -### Key Design Decisions - -1. **Header-only library**: All code in `include/`; no compiled library. -2. **Non-owning spans**: Use `std::span` for external data where appropriate. -3. **SIMD conditional compilation**: Use `#ifdef {{PROJECT_NAME_UPPER}}_AVX512_SUPPORT` / `{{PROJECT_NAME_UPPER}}_AVX2_SUPPORT` with scalar fallbacks. -4. **Target domain**: Optimized for practical data sizes. -5. **Platform**: Linux/Unix is the primary target platform. - -### Why Header-Only? - -- **SIMD flexibility**: Users compile with their target `-march` flags. -- **Better inlining**: Compiler sees full implementation. -- **No ABI issues**: Works across compilers and standard library versions. -- **Easy integration**: Users just `#include` headers. -- **Template-friendly**: No explicit instantiation needed. - -## Technology Stack - -- **Language**: C++20 (required features: `std::span`, `std::popcount`, ``) -- **Build**: CMake >= 3.18 -- **Testing**: Google Test v1.17.0 -- **Benchmarking**: Google Benchmark v1.9.4 -- **SIMD**: AVX-512 (primary), AVX2 (fallback), scalar fallbacks -- **Style**: Chromium C++ style (`.clang-format`) - -### Dependencies - -The library itself is header-only and has **no runtime dependencies**. Build-time dependencies are managed via CMake FetchContent and controlled by options: - -| Option | Default | What it enables | -|--------|---------|-----------------| -| `{{PROJECT_NAME_UPPER}}_TESTS` | `ON` | Unit tests (fetches Google Test) | -| `{{PROJECT_NAME_UPPER}}_BENCHMARKS` | `OFF` | Benchmarks (fetches Google Benchmark) | - -## Build Commands - -```bash -# Standard build (Release) -cmake -B build/release -DCMAKE_BUILD_TYPE=Release -cmake --build build/release -j - -# Debug build -cmake -B build/debug -DCMAKE_BUILD_TYPE=Debug -cmake --build build/debug -j - -# Without wide SIMD -cmake -B build/release -D{{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD=ON -cmake --build build/release -j - -# With AddressSanitizer -cmake -B build/asan -DENABLE_ADDRESS_SANITIZER=ON -cmake --build build/asan -j - -# Custom march flag -cmake -B build/release -DMARCH=icelake-client -cmake --build build/release -j - -# Tests only (no benchmarks) -cmake -B build/release -D{{PROJECT_NAME_UPPER}}_BENCHMARKS=OFF -cmake --build build/release -j -``` - -## Testing - -### Running Tests - -```bash -./build/release/unittests -``` - -### Testing Patterns - -- **Differential testing**: Compare against naive reference implementations. -- **Randomized testing**: Random inputs with configurable seed. -- **Exhaustive short inputs**: Test all patterns for small sizes. - -## Code Style Guidelines - -1. **Formatting**: Run `clang-format` before committing (Chromium style) -2. **Namespace**: All library code in `{{NAMESPACE}}` namespace -3. **Documentation**: Use Doxygen-style comments for public API -4. **Constants**: Use `constexpr` for compile-time values -5. **Alignment**: Be aware of data alignment; prefer 64-byte aligned array allocations where performance matters - -## CI/CD Workflows - -- **build-test.yml**: Builds and runs tests with AddressSanitizer -- **linter.yml**: Clang-format checks on all C/C++ files -- **coverage.yml**: Coverage reporting with codecov upload -- **doxygen.yml**: Documentation generation and GitHub Pages deployment - -## Common Tasks for AI Agents - -### Adding a New Component - -1. Create header in `include/{{NAMESPACE}}/` with Doxygen documentation -2. Add unit tests in `src/tests/_tests.cpp` -3. Add benchmarks in `src/benchmarks/_benchmarks.cpp` -4. Update `CMakeLists.txt` with new executables -5. Run `clang-format` on new files - -### Modifying SIMD Code - -1. Provide implementations for: - - Wide SIMD (`#ifdef {{PROJECT_NAME_UPPER}}_AVX512_SUPPORT`) - - AVX2 (`#ifdef {{PROJECT_NAME_UPPER}}_AVX2_SUPPORT`) - - Scalar fallback -2. Test with `-D{{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD=ON` to verify fallback works -3. Benchmark to ensure performance is maintained - -### Adding Tests - -1. Use Google Test framework -2. Include naive reference implementation for differential testing -3. Add edge cases: empty input, single element, boundary conditions -4. Use random testing with configurable seed for reproducibility - -## Performance Philosophy - -- **Goal**: Best practical performance (not just asymptotic complexity) -- **Approach**: Benchmark-driven optimization using Google Benchmark -- **SIMD**: Leverage vectorized operations where beneficial -- **Cache efficiency**: Align data structures to cache line boundaries (64 bytes) -""" - -DOXYFILE_IN = """# Doxyfile - -DOXYFILE_ENCODING = UTF-8 -PROJECT_NAME = "{{PROJECT_NAME}}" -PROJECT_NUMBER = -PROJECT_BRIEF = -PROJECT_LOGO = -PROJECT_ICON = -OUTPUT_DIRECTORY = docs -CREATE_SUBDIRS = NO -CREATE_SUBDIRS_LEVEL = 8 -ALLOW_UNICODE_NAMES = NO -OUTPUT_LANGUAGE = English -BRIEF_MEMBER_DESC = YES -REPEAT_BRIEF = YES -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the -ALWAYS_DETAILED_SEC = NO -INLINE_INHERITED_MEMB = NO -FULL_PATH_NAMES = YES -STRIP_FROM_PATH = @CMAKE_CURRENT_SOURCE_DIR@ -STRIP_FROM_INC_PATH = -SHORT_NAMES = NO -JAVADOC_AUTOBRIEF = NO -JAVADOC_BANNER = NO -QT_AUTOBRIEF = NO -MULTILINE_CPP_IS_BRIEF = NO -PYTHON_DOCSTRING = YES -INHERIT_DOCS = YES -SEPARATE_MEMBER_PAGES = NO -TAB_SIZE = 4 -ALIASES = -OPTIMIZE_OUTPUT_FOR_C = NO -OPTIMIZE_OUTPUT_JAVA = NO -OPTIMIZE_FOR_FORTRAN = NO -OPTIMIZE_OUTPUT_VHDL = NO -OPTIMIZE_OUTPUT_SLICE = NO -EXTENSION_MAPPING = -MARKDOWN_SUPPORT = YES -MARKDOWN_STRICT = YES -TOC_INCLUDE_HEADINGS = 6 -MARKDOWN_ID_STYLE = DOXYGEN -AUTOLINK_SUPPORT = YES -AUTOLINK_IGNORE_WORDS = -BUILTIN_STL_SUPPORT = NO -CPP_CLI_SUPPORT = NO -SIP_SUPPORT = NO -IDL_PROPERTY_SUPPORT = YES -DISTRIBUTE_GROUP_DOC = NO -GROUP_NESTED_COMPOUNDS = NO -SUBGROUPING = YES -INLINE_GROUPED_CLASSES = NO -INLINE_SIMPLE_STRUCTS = NO -TYPEDEF_HIDES_STRUCT = NO -LOOKUP_CACHE_SIZE = 0 -NUM_PROC_THREADS = 1 -TIMESTAMP = NO -EXTRACT_ALL = NO -EXTRACT_PRIVATE = NO -EXTRACT_PRIV_VIRTUAL = NO -EXTRACT_PACKAGE = NO -EXTRACT_STATIC = NO -EXTRACT_LOCAL_CLASSES = YES -EXTRACT_LOCAL_METHODS = NO -EXTRACT_ANON_NSPACES = NO -RESOLVE_UNNAMED_PARAMS = YES -HIDE_UNDOC_MEMBERS = NO -HIDE_UNDOC_CLASSES = NO -HIDE_UNDOC_NAMESPACES = YES -HIDE_FRIEND_COMPOUNDS = NO -HIDE_IN_BODY_DOCS = NO -INTERNAL_DOCS = NO -CASE_SENSE_NAMES = SYSTEM -HIDE_SCOPE_NAMES = NO -HIDE_COMPOUND_REFERENCE= NO -SHOW_HEADERFILE = YES -SHOW_INCLUDE_FILES = YES -SHOW_GROUPED_MEMB_INC = NO -FORCE_LOCAL_INCLUDES = NO -INLINE_INFO = YES -SORT_MEMBER_DOCS = YES -SORT_BRIEF_DOCS = NO -SORT_MEMBERS_CTORS_1ST = NO -SORT_GROUP_NAMES = NO -SORT_BY_SCOPE_NAME = NO -STRICT_PROTO_MATCHING = NO -GENERATE_TODOLIST = YES -GENERATE_TESTLIST = YES -GENERATE_BUGLIST = YES -GENERATE_DEPRECATEDLIST= YES -ENABLED_SECTIONS = -MAX_INITIALIZER_LINES = 30 -SHOW_USED_FILES = YES -SHOW_FILES = YES -SHOW_NAMESPACES = YES -FILE_VERSION_FILTER = -LAYOUT_FILE = -CITE_BIB_FILES = -EXTERNAL_TOOL_PATH = -QUIET = NO -WARNINGS = YES -WARN_IF_UNDOCUMENTED = YES -WARN_IF_DOC_ERROR = YES -WARN_IF_INCOMPLETE_DOC = YES -WARN_NO_PARAMDOC = NO -WARN_IF_UNDOC_ENUM_VAL = NO -WARN_LAYOUT_FILE = YES -WARN_AS_ERROR = NO -WARN_FORMAT = "$file:$line: $text" -WARN_LINE_FORMAT = "at line $line of file $file" -WARN_LOGFILE = -INPUT = @CMAKE_CURRENT_SOURCE_DIR@/include \ - @CMAKE_CURRENT_SOURCE_DIR@/README.md -INPUT_ENCODING = UTF-8 -INPUT_FILE_ENCODING = -FILE_PATTERNS = *.c \ - *.cc \ - *.cxx \ - *.cpp \ - *.h \ - *.hh \ - *.hxx \ - *.hpp -RECURSIVE = YES -EXCLUDE = -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = -EXCLUDE_SYMBOLS = -EXAMPLE_PATH = -EXAMPLE_PATTERNS = * -EXAMPLE_RECURSIVE = NO -IMAGE_PATH = @CMAKE_CURRENT_SOURCE_DIR@/src/docs/images -INPUT_FILTER = -FILTER_PATTERNS = -FILTER_SOURCE_FILES = NO -FILTER_SOURCE_PATTERNS = -USE_MDFILE_AS_MAINPAGE = @CMAKE_CURRENT_SOURCE_DIR@/README.md -IMPLICIT_DIR_DOCS = YES -FORTRAN_COMMENT_AFTER = 72 -SOURCE_BROWSER = NO -INLINE_SOURCES = NO -STRIP_CODE_COMMENTS = YES -REFERENCED_BY_RELATION = NO -REFERENCES_RELATION = NO -REFERENCES_LINK_SOURCE = YES -SOURCE_TOOLTIPS = YES -USE_HTAGS = NO -VERBATIM_HEADERS = YES -CLANG_ASSISTED_PARSING = NO -CLANG_ADD_INC_PATHS = YES -CLANG_OPTIONS = -CLANG_DATABASE_PATH = -ALPHABETICAL_INDEX = YES -IGNORE_PREFIX = -GENERATE_HTML = YES -HTML_OUTPUT = html -HTML_FILE_EXTENSION = .html -HTML_HEADER = -HTML_FOOTER = -HTML_STYLESHEET = -HTML_EXTRA_STYLESHEET = @AWESOME_CSS_DIR@/doxygen-awesome.css -HTML_EXTRA_FILES = -HTML_COLORSTYLE = AUTO_LIGHT -HTML_COLORSTYLE_HUE = 220 -HTML_COLORSTYLE_SAT = 100 -HTML_COLORSTYLE_GAMMA = 80 -HTML_DYNAMIC_MENUS = YES -HTML_DYNAMIC_SECTIONS = NO -HTML_CODE_FOLDING = YES -HTML_COPY_CLIPBOARD = YES -HTML_PROJECT_COOKIE = -HTML_INDEX_NUM_ENTRIES = 100 -GENERATE_DOCSET = NO -DOCSET_FEEDNAME = "Doxygen generated docs" -DOCSET_FEEDURL = -DOCSET_BUNDLE_ID = org.doxygen.Project -DOCSET_PUBLISHER_ID = org.doxygen.Publisher -DOCSET_PUBLISHER_NAME = Publisher -GENERATE_HTMLHELP = NO -CHM_FILE = -HHC_LOCATION = -GENERATE_CHI = NO -CHM_INDEX_ENCODING = -BINARY_TOC = NO -TOC_EXPAND = NO -SITEMAP_URL = -GENERATE_QHP = NO -QCH_FILE = -QHP_NAMESPACE = org.doxygen.Project -QHP_VIRTUAL_FOLDER = doc -QHP_CUST_FILTER_NAME = -QHP_CUST_FILTER_ATTRS = -QHP_SECT_FILTER_ATTRS = -QHG_LOCATION = -GENERATE_ECLIPSEHELP = NO -ECLIPSE_DOC_ID = org.doxygen.Project -DISABLE_INDEX = NO -GENERATE_TREEVIEW = YES -PAGE_OUTLINE_PANEL = YES -FULL_SIDEBAR = NO -ENUM_VALUES_PER_LINE = 4 -SHOW_ENUM_VALUES = NO -TREEVIEW_WIDTH = 250 -EXT_LINKS_IN_WINDOW = NO -OBFUSCATE_EMAILS = YES -HTML_FORMULA_FORMAT = png -FORMULA_FONTSIZE = 10 -FORMULA_MACROFILE = -USE_MATHJAX = NO -MATHJAX_VERSION = MathJax_2 -MATHJAX_FORMAT = HTML-CSS -MATHJAX_RELPATH = -MATHJAX_EXTENSIONS = -MATHJAX_CODEFILE = -SEARCHENGINE = YES -SERVER_BASED_SEARCH = NO -EXTERNAL_SEARCH = NO -SEARCHENGINE_URL = -SEARCHDATA_FILE = searchdata.xml -EXTERNAL_SEARCH_ID = -EXTRA_SEARCH_MAPPINGS = -GENERATE_LATEX = NO -""" - -COVERAGE_REPORT_SH = """#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -BUILD_DIR="${ROOT_DIR}/build/coverage" - -cmake --preset coverage -cmake --build --preset coverage - -"${BUILD_DIR}/unittests" - -cd "${BUILD_DIR}" -find . -name "*.gcda" > gcov_files.txt -while read -r f; do - case "${f}" in - *"/_deps/"*|*"/third_party/"*|*"/src/benchmarks/"*) - ;; - *) - gcov -pb "${f}" >> coverage.txt - ;; - esac -done < gcov_files.txt -echo "gcov report written to ${BUILD_DIR}/coverage.txt" -""" - -BUILD_TEST_YML = """name: Tests (ASan) - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build-and-test: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Create Build Directory - run: mkdir build - - - name: Configure CMake - working-directory: ./build - run: cmake -D{{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD=ON -DENABLE_ADDRESS_SANITIZER=ON -D{{PROJECT_NAME_UPPER}}_BENCHMARKS=OFF .. - - - name: Build Project - working-directory: ./build - run: make -j - - - name: Run Unittests - working-directory: ./build - run: ./unittests -""" - -LINTER_YML = """name: Clang Format Lint - -on: - pull_request: - push: - branches: [main] - -jobs: - clang-format: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install clang-format - run: sudo apt-get update && sudo apt-get install -y clang-format - - - name: Run clang-format check - run: | - mapfile -t FILES < <(find include src -type f \\( -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.c' -o -name '*.h' \\)) - clang-format --version - if [ ${#FILES[@]} -eq 0 ]; then - echo "No C/C++ files found." - exit 0 - fi - - clang-format --dry-run --Werror "${FILES[@]}" -""" - -COVERAGE_YML = """name: coverage - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - coverage: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Create Build Directory - run: mkdir build - - - name: Run coverage - run: ./scripts/coverage_report.sh - - - name: Upload to Codecov - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: build/coverage/coverage.txt - flags: gcov - fail_ci_if_error: false - - - name: Upload coverage artifacts - uses: actions/upload-artifact@v4 - with: - name: coverage-gcov - path: | - build/coverage/coverage.txt - build/coverage/*.gcov -""" - -DOXYGEN_YML = """# Simple workflow for deploying static content to GitHub Pages -name: Deploy static content to Pages - -on: - # Runs on pushes targeting the default branch - push: - branches: ["main"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - # Single deploy job since we're just deploying - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Doxygen v1.13.2 - run: | - transformed_version=$(echo "1.13.2" | tr '.' '_') - wget https://github.com/doxygen/doxygen/releases/download/Release_${transformed_version}/doxygen-1.13.2.linux.bin.tar.gz - tar -xzf doxygen-1.13.2.linux.bin.tar.gz - sudo mv doxygen-1.13.2/bin/doxygen /usr/local/bin/doxygen - shell: bash - - name: Cmake configure - run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -D{{PROJECT_NAME_UPPER}}_DOCS=ON -D{{PROJECT_NAME_UPPER}}_TESTS=OFF -D{{PROJECT_NAME_UPPER}}_BENCHMARKS=OFF - - name: Build docs - run: cmake --build ${{github.workspace}}/build --target docs - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - # Upload entire repository - path: ${{github.workspace}}/build/docs/html - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 -""" - -HEADER_HPP = """#pragma once - -/** - * @file {{HEADER_NAME}} - * @brief Main header for the {{PROJECT_NAME}} library - */ - -namespace {{NAMESPACE}} { - -/** - * @brief Example function. - * - * TODO: Replace with actual library functionality. - */ -inline int example() { - return 42; -} - -} // namespace {{NAMESPACE}} -""" - -UNITTESTS_CPP = """#include - -#include "{{NAMESPACE}}/{{HEADER_NAME}}" - -TEST(ExampleTest, BasicAssertion) { - EXPECT_EQ({{NAMESPACE}}::example(), 42); -} -""" - -BENCHMARKS_CPP = """#include - -#include "{{NAMESPACE}}/{{HEADER_NAME}}" - -static void BM_Example(benchmark::State& state) { - for (auto _ : state) { - benchmark::DoNotOptimize({{NAMESPACE}}::example()); - } -} - -BENCHMARK(BM_Example); - -BENCHMARK_MAIN(); -""" - - -# --------------------------------------------------------------------------- -# Generation logic -# --------------------------------------------------------------------------- - -def generate(args: argparse.Namespace) -> None: - project_name = args.name - namespace = args.namespace or project_name.replace("-", "") - project_name_upper = to_upper(project_name) - header_name = f"{to_snake(project_name)}.hpp" - output_dir = Path(args.output_dir).resolve() / project_name - - if output_dir.exists(): - print(f"Error: output directory already exists: {output_dir}") - sys.exit(1) - - substitutions = { - "{{PROJECT_NAME}}": project_name, - "{{NAMESPACE}}": namespace, - "{{PROJECT_NAME_UPPER}}": project_name_upper, - "{{HEADER_NAME}}": header_name, - } - - def sub(text: str) -> str: - for key, value in substitutions.items(): - text = text.replace(key, value) - return text - - # Create directories - (output_dir / "include" / namespace).mkdir(parents=True) - (output_dir / "src" / "tests").mkdir(parents=True) - (output_dir / "src" / "benchmarks").mkdir(parents=True) - (output_dir / "src" / "docs").mkdir(parents=True) - (output_dir / "src" / "docs" / "images").mkdir(parents=True) - (output_dir / "scripts").mkdir(parents=True) - (output_dir / ".github" / "workflows").mkdir(parents=True) - - # Write files - files = { - output_dir / "CMakeLists.txt": sub(CMAKE_LISTS_TXT), - output_dir / "CMakePresets.json": sub(CMAKE_PRESETS_JSON), - output_dir / ".clang-format": sub(CLANG_FORMAT), - output_dir / ".gitignore": sub(GITIGNORE), - output_dir / "README.md": sub(README_MD), - output_dir / "AGENTS.md": sub(AGENTS_MD), - output_dir / "src" / "docs" / "Doxyfile.in": sub(DOXYFILE_IN), - output_dir / "scripts" / "coverage_report.sh": sub(COVERAGE_REPORT_SH), - output_dir / ".github" / "workflows" / "build-test.yml": sub(BUILD_TEST_YML), - output_dir / ".github" / "workflows" / "linter.yml": sub(LINTER_YML), - output_dir / ".github" / "workflows" / "coverage.yml": sub(COVERAGE_YML), - output_dir / ".github" / "workflows" / "doxygen.yml": sub(DOXYGEN_YML), - output_dir / "include" / namespace / header_name: sub(HEADER_HPP), - output_dir / "src" / "tests" / "unittests.cpp": sub(UNITTESTS_CPP), - output_dir / "src" / "benchmarks" / "benchmarks.cpp": sub(BENCHMARKS_CPP), - } - - for path, content in files.items(): - path.write_text(content) - print(f"Created: {path.relative_to(output_dir.parent)}") - - # Make coverage script executable - (output_dir / "scripts" / "coverage_report.sh").chmod(0o755) - - print(f"\\nProject '{project_name}' generated successfully at {output_dir}") - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Scaffold a new C++20 repository following modern C++ conventions." - ) - parser.add_argument("--name", required=True, help="Project name (e.g., my-lib)") - parser.add_argument( - "--namespace", - help="C++ namespace (defaults to project name with hyphens removed)", - ) - parser.add_argument( - "--output-dir", - default=".", - help="Output directory (default: current directory)", - ) - args = parser.parse_args() - generate(args) - - -if __name__ == "__main__": - main() From 6372bd006075b7972da7629d8bf0ae340c7a4993 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 30 May 2026 22:51:26 +0300 Subject: [PATCH 05/27] chore: add agentic/cpp as a git submodule --- .gitmodules | 3 +++ agentic/cpp | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 agentic/cpp diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..6f6274b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "agentic/cpp"] + path = agentic/cpp + url = git@github.com:Malkovsky/ai_for_cpp.git diff --git a/agentic/cpp b/agentic/cpp new file mode 160000 index 0000000..e1e5aeb --- /dev/null +++ b/agentic/cpp @@ -0,0 +1 @@ +Subproject commit e1e5aeb54ff729b22245d09d6d9cdc4e14b0a60d From 0ea73463904a84940fd347c21b249391f42b8cc7 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 30 May 2026 23:28:00 +0300 Subject: [PATCH 06/27] Updated agentic --- agentic/cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agentic/cpp b/agentic/cpp index e1e5aeb..4fa94cd 160000 --- a/agentic/cpp +++ b/agentic/cpp @@ -1 +1 @@ -Subproject commit e1e5aeb54ff729b22245d09d6d9cdc4e14b0a60d +Subproject commit 4fa94cdcfbd0ddbcd77492324b45fcb82e4f85bd From 8a9b6754b3237414fd8725316a584698ea571215 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sun, 31 May 2026 21:44:15 +0300 Subject: [PATCH 07/27] Improved implementation --- include/pixie/bits.h | 280 +++++++++- include/pixie/rmq/bp_plus_minus_one_rmq.h | 383 ++++++++++++-- include/pixie/rmq/cartesian_tree_rmq.h | 116 +++++ include/pixie/rmq/rmq_base.h | 21 + include/pixie/rmq/segment_tree.h | 54 ++ include/pixie/rmq/sparse_table.h | 52 ++ src/benchmarks/bench_rmq.cpp | 492 +++++++++++++++++- .../excess_positions_benchmarks.cpp | 64 +++ src/tests/excess_positions_tests.cpp | 81 +++ src/tests/rmq_tests.cpp | 21 + 10 files changed, 1510 insertions(+), 54 deletions(-) diff --git a/include/pixie/bits.h b/include/pixie/bits.h index b103600..57e5c16 100644 --- a/include/pixie/bits.h +++ b/include/pixie/bits.h @@ -880,6 +880,17 @@ struct ExcessResult { size_t offset = 128; }; +/** + * @brief Pair of boundary minimum results for adjacent BP query blocks. + * + * @details `suffix` is local to the left/suffix block and `prefix` is local to + * the right/prefix block. + */ +struct ExcessBoundaryPairResult { + ExcessResult suffix; + ExcessResult prefix; +}; + constexpr int8_t excess_byte_delta_value(uint8_t x) { return static_cast(2 * std::popcount(x) - 8); } @@ -1106,15 +1117,16 @@ static inline ExcessResult excess_min_128(const uint64_t* s, left = std::min(left, 128); right = std::min(right, 128); + if (right - left <= 32 && (left & 7u) == 0 && (right & 7u) == 0) + [[unlikely]] { + return excess_min_128_byte_lut_short(s, left, right); + } + int best = prefix_excess_128(s, left); size_t best_offset = left; if (left == right) { return {best, best_offset}; } - if (right - left <= 32 && (left & 7u) == 0 && (right & 7u) == 0) - [[unlikely]] { - return excess_min_128_byte_lut_short(s, left, right); - } #ifdef PIXIE_AVX2_SUPPORT int current = best; @@ -1130,7 +1142,10 @@ static inline ExcessResult excess_min_128(const uint64_t* s, const size_t first_full_nibble = bit >> 2; const size_t last_full_nibble = right >> 2; - if (first_full_nibble < last_full_nibble) { + const size_t right_partial_width = bit < right ? (right & 3u) : 0; + const size_t end_nibble = + last_full_nibble + (right_partial_width == 0 ? 0 : 1); + if (first_full_nibble < end_nibble) { const __m256i nibbles = excess_nibbles_128_avx2(s); __m256i ps = _mm256_shuffle_epi8(excess_lut_delta, nibbles); @@ -1148,19 +1163,34 @@ static inline ExcessResult excess_min_128(const uint64_t* s, __m256i b = _mm256_permute2x128_si256(ps, ps, 0x08); const __m256i excl_ps = _mm256_alignr_epi8(ps, b, 15); - const __m256i candidates = - _mm256_add_epi8(excl_ps, _mm256_shuffle_epi8(excess_lut_min, nibbles)); + __m256i local_min = _mm256_shuffle_epi8(excess_lut_min, nibbles); + if (right_partial_width != 0) { + __m256i partial_min = _mm256_shuffle_epi8(excess_lut_pos0, nibbles); + if (right_partial_width >= 2) { + partial_min = _mm256_min_epi8( + partial_min, _mm256_shuffle_epi8(excess_lut_pos1, nibbles)); + } + if (right_partial_width >= 3) { + partial_min = _mm256_min_epi8( + partial_min, _mm256_shuffle_epi8(excess_lut_pos2, nibbles)); + } + local_min = _mm256_blendv_epi8( + local_min, partial_min, + _mm256_cmpeq_epi8( + excess_lut_nibble_index, + _mm256_set1_epi8(static_cast(last_full_nibble)))); + } + const __m256i partial_candidates = _mm256_add_epi8(excl_ps, local_min); const __m256i idx = excess_lut_nibble_index; const int first_minus_one_value = static_cast(first_full_nibble) - 1; const __m256i first_minus_one = _mm256_set1_epi8(static_cast(first_minus_one_value)); - const __m256i last = - _mm256_set1_epi8(static_cast(last_full_nibble)); + const __m256i last = _mm256_set1_epi8(static_cast(end_nibble)); const __m256i active = _mm256_and_si256( _mm256_cmpgt_epi8(idx, first_minus_one), _mm256_cmpgt_epi8(last, idx)); const __m256i masked_candidates = - _mm256_blendv_epi8(_mm256_set1_epi8(127), candidates, active); + _mm256_blendv_epi8(_mm256_set1_epi8(127), partial_candidates, active); __m128i min128 = _mm_min_epi8(_mm256_castsi256_si128(masked_candidates), @@ -1183,12 +1213,25 @@ static inline ExcessResult excess_min_128(const uint64_t* s, const uint8_t nibble = static_cast((word >> ((nibble_index & 15u) * 4u)) & 0xFu); best = candidate_min; - best_offset = static_cast(nibble_index) * 4u + - static_cast(excess_lut_min_offset[nibble]); + if (right_partial_width != 0 && nibble_index == last_full_nibble) { + int local = 0; + int local_best = 0; + size_t local_offset = 1; + for (size_t i = 0; i < right_partial_width; ++i) { + local += ((nibble >> i) & 1u) != 0 ? 1 : -1; + if (i == 0 || local < local_best) { + local_best = local; + local_offset = i + 1; + } + } + best_offset = static_cast(nibble_index) * 4u + local_offset; + } else { + best_offset = static_cast(nibble_index) * 4u + + static_cast(excess_lut_min_offset[nibble]); + } } - bit = last_full_nibble * 4; - current = prefix_excess_128(s, bit); + bit = end_nibble * 4; } for (; bit < right; ++bit) { @@ -1214,6 +1257,215 @@ static inline ExcessResult excess_min_128(const uint64_t* s, return {best, best_offset}; } +/** + * @brief Compute disjoint suffix/prefix boundary minima for two 128-bit blocks. + * + * @details Computes `excess_min_128(suffix_s, suffix_left, 127)` and + * `excess_min_128(prefix_s, 0, prefix_right)`. When `suffix_left > + * prefix_right`, the AVX2 path packs the two disjoint local ranges into one + * nibble stream and shares the prefix-sum/reduction work. Non-disjoint or + * out-of-shape inputs fall back to the two independent production calls. + * + * @param suffix_s Left boundary block. + * @param suffix_left First local prefix offset in the suffix range. + * @param prefix_s Right boundary block. + * @param prefix_right Last local prefix offset in the prefix range. + * @return Pair of local minimum results. + */ +static inline ExcessBoundaryPairResult excess_min_128_disjoint_suffix_prefix( + const uint64_t* suffix_s, + size_t suffix_left, + const uint64_t* prefix_s, + size_t prefix_right) noexcept { + suffix_left = std::min(suffix_left, 128); + prefix_right = std::min(prefix_right, 128); + if (suffix_left <= prefix_right || suffix_left > 127 || prefix_right > 127) { + return {excess_min_128(suffix_s, suffix_left, 127), + excess_min_128(prefix_s, 0, prefix_right)}; + } + +#ifdef PIXIE_AVX2_SUPPORT + ExcessResult prefix{0, 0}; + + int suffix_best = prefix_excess_128(suffix_s, suffix_left); + ExcessResult suffix{suffix_best, suffix_left}; + size_t suffix_bit = suffix_left; + int suffix_current = suffix_best; + for (; suffix_bit < 127 && (suffix_bit & 3u) != 0; ++suffix_bit) { + suffix_current += + ((suffix_s[suffix_bit >> 6] >> (suffix_bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = suffix_bit + 1; + if (suffix_current < suffix.min_excess) { + suffix = {suffix_current, offset}; + } + } + + alignas(32) int8_t lane_nibbles[32] = {}; + alignas(32) int8_t lane_indices[32] = {}; + alignas(32) int8_t prefix_active[32] = {}; + alignas(32) int8_t suffix_active[32] = {}; + alignas(32) int8_t partial_widths[32] = {}; + alignas(32) int8_t prefix_partial[32] = {}; + alignas(32) int8_t suffix_partial[32] = {}; + + size_t lane = 0; + int prefix_artificial_delta = 0; + const size_t prefix_last_nibble = prefix_right >> 2; + const size_t prefix_partial_width = prefix_right & 3u; + const size_t prefix_end_nibble = + prefix_last_nibble + (prefix_partial_width == 0 ? 0 : 1); + for (size_t nibble_index = 0; nibble_index < prefix_end_nibble; + ++nibble_index) { + const uint64_t word = prefix_s[nibble_index >> 4]; + const uint8_t nibble = + static_cast((word >> ((nibble_index & 15u) * 4u)) & 0xFu); + lane_nibbles[lane] = static_cast(nibble); + lane_indices[lane] = static_cast(nibble_index); + prefix_active[lane] = static_cast(0xFFu); + if (prefix_partial_width != 0 && nibble_index == prefix_last_nibble) { + partial_widths[lane] = static_cast(prefix_partial_width); + prefix_partial[lane] = static_cast(0xFFu); + } + prefix_artificial_delta += 2 * std::popcount(nibble) - 4; + ++lane; + } + + if (suffix_bit < 127) { + const size_t suffix_first_nibble = suffix_bit >> 2; + for (size_t nibble_index = suffix_first_nibble; nibble_index < 32; + ++nibble_index) { + const uint64_t word = suffix_s[nibble_index >> 4]; + const uint8_t nibble = + static_cast((word >> ((nibble_index & 15u) * 4u)) & 0xFu); + lane_nibbles[lane] = static_cast(nibble); + lane_indices[lane] = static_cast(nibble_index); + suffix_active[lane] = static_cast(0xFFu); + if (nibble_index == 31) { + partial_widths[lane] = 3; + suffix_partial[lane] = static_cast(0xFFu); + } + ++lane; + } + } + + if (lane > 32) { + return {excess_min_128(suffix_s, suffix_left, 127), + excess_min_128(prefix_s, 0, prefix_right)}; + } + + __m256i nibbles = + _mm256_load_si256(reinterpret_cast(lane_nibbles)); + __m256i ps = _mm256_shuffle_epi8(excess_lut_delta, nibbles); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 1)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 2)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 4)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 8)); + + __m128i ps_lo = _mm256_castsi256_si128(ps); + __m128i ps_hi = _mm256_extracti128_si256(ps, 1); + __m128i carry = + _mm_set1_epi8(static_cast(_mm_extract_epi8(ps_lo, 15))); + ps_hi = _mm_add_epi8(ps_hi, carry); + ps = _mm256_inserti128_si256(_mm256_castsi128_si256(ps_lo), ps_hi, 1); + + __m256i b = _mm256_permute2x128_si256(ps, ps, 0x08); + const __m256i excl_ps = _mm256_alignr_epi8(ps, b, 15); + + __m256i local_min = _mm256_shuffle_epi8(excess_lut_min, nibbles); + const __m256i width = + _mm256_load_si256(reinterpret_cast(partial_widths)); + __m256i partial_min = _mm256_shuffle_epi8(excess_lut_pos0, nibbles); + partial_min = _mm256_min_epi8( + partial_min, + _mm256_blendv_epi8(_mm256_set1_epi8(127), + _mm256_shuffle_epi8(excess_lut_pos1, nibbles), + _mm256_cmpgt_epi8(width, _mm256_set1_epi8(1)))); + partial_min = _mm256_min_epi8( + partial_min, + _mm256_blendv_epi8(_mm256_set1_epi8(127), + _mm256_shuffle_epi8(excess_lut_pos2, nibbles), + _mm256_cmpgt_epi8(width, _mm256_set1_epi8(2)))); + const __m256i any_partial = _mm256_or_si256( + _mm256_load_si256(reinterpret_cast(prefix_partial)), + _mm256_load_si256(reinterpret_cast(suffix_partial))); + local_min = _mm256_blendv_epi8(local_min, partial_min, any_partial); + + const __m256i base_candidates = _mm256_add_epi8(excl_ps, local_min); + const __m256i prefix_mask = + _mm256_load_si256(reinterpret_cast(prefix_active)); + const __m256i suffix_mask = + _mm256_load_si256(reinterpret_cast(suffix_active)); + const __m256i sentinel = _mm256_set1_epi8(127); + + const __m256i prefix_candidates = + _mm256_blendv_epi8(sentinel, base_candidates, prefix_mask); + const __m256i suffix_candidates = _mm256_blendv_epi8( + sentinel, + _mm256_add_epi8(base_candidates, + _mm256_set1_epi8(static_cast( + suffix_current - prefix_artificial_delta))), + suffix_mask); + + auto reduce_min = [](__m256i values) { + __m128i min128 = _mm_min_epi8(_mm256_castsi256_si128(values), + _mm256_extracti128_si256(values, 1)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 8)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 4)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 2)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 1)); + return static_cast(static_cast(_mm_extract_epi8(min128, 0))); + }; + + auto local_offset = [](uint8_t nibble, size_t width_value) { + if (width_value == 0 || width_value == 4) { + return static_cast(excess_lut_min_offset[nibble]); + } + int current = 0; + int best = 0; + size_t best_offset = 1; + for (size_t i = 0; i < width_value; ++i) { + current += ((nibble >> i) & 1u) != 0 ? 1 : -1; + if (i == 0 || current < best) { + best = current; + best_offset = i + 1; + } + } + return best_offset; + }; + + const int prefix_min = reduce_min(prefix_candidates); + if (prefix_min < prefix.min_excess) { + const uint32_t mask = static_cast(_mm256_movemask_epi8( + _mm256_cmpeq_epi8(prefix_candidates, + _mm256_set1_epi8(static_cast(prefix_min))))); + const uint32_t prefix_lane = std::countr_zero(mask); + const uint8_t nibble = static_cast(lane_nibbles[prefix_lane]); + prefix.min_excess = prefix_min; + prefix.offset = + static_cast(lane_indices[prefix_lane]) * 4u + + local_offset(nibble, static_cast(partial_widths[prefix_lane])); + } + + const int suffix_min = reduce_min(suffix_candidates); + if (suffix_min < suffix.min_excess) { + const uint32_t mask = static_cast(_mm256_movemask_epi8( + _mm256_cmpeq_epi8(suffix_candidates, + _mm256_set1_epi8(static_cast(suffix_min))))); + const uint32_t suffix_lane = std::countr_zero(mask); + const uint8_t nibble = static_cast(lane_nibbles[suffix_lane]); + suffix.min_excess = suffix_min; + suffix.offset = + static_cast(lane_indices[suffix_lane]) * 4u + + local_offset(nibble, static_cast(partial_widths[suffix_lane])); + } + + return {suffix, prefix}; +#else + return {excess_min_128(suffix_s, suffix_left, 127), + excess_min_128(prefix_s, 0, prefix_right)}; +#endif +} + /** * @brief Find the first prefix reaching target_x in a 128-bit bitstring. * diff --git a/include/pixie/rmq/bp_plus_minus_one_rmq.h b/include/pixie/rmq/bp_plus_minus_one_rmq.h index e796b93..1038ef7 100644 --- a/include/pixie/rmq/bp_plus_minus_one_rmq.h +++ b/include/pixie/rmq/bp_plus_minus_one_rmq.h @@ -23,9 +23,9 @@ namespace pixie::rmq { * @details The indexed depth sequence is represented by BP deltas: bit 1 means * the next depth is current + 1, and bit 0 means current - 1. A sequence with * @p depth_count depth positions has @p depth_count - 1 delta bits. Blocks - * match the 128-bit excess primitives in `bits.h`; only the absolute minimum - * value of each block is stored, and positions are recovered by rescanning the - * selected block. + * match the 128-bit excess primitives in `bits.h`; each block stores a compact + * 16-byte summary with its base depth, absolute minimum value, and first local + * offset attaining that minimum. * * @tparam Index Unsigned integer type used for stored positions. * @tparam BlockSize Number of depth positions per microblock. @@ -38,54 +38,112 @@ class BpPlusMinusOneRmq { static constexpr std::size_t npos = std::numeric_limits::max(); static constexpr Index invalid_index = std::numeric_limits::max(); + /** + * @brief Construct an empty ±1 RMQ index. + */ BpPlusMinusOneRmq() = default; + /** + * @brief Build a ±1 RMQ index over a BP-encoded depth-delta sequence. + * + * @details The bit span is not copied and must outlive this object. Bit `1` + * encodes a +1 step between adjacent depths, and bit `0` encodes a -1 step. + * A sequence with @p depth_count positions requires @p depth_count - 1 bits. + * + * @param bits Packed delta bits in little-endian bit order within each word. + * @param depth_count Number of depth positions indexed by the RMQ. + * @throws std::length_error if @p Index cannot represent all positions or a + * packed 48-bit block summary overflows. + * @throws std::invalid_argument if @p bits does not contain enough words. + */ BpPlusMinusOneRmq(std::span bits, std::size_t depth_count) : input_bits_(bits), depth_count_(depth_count) { build(); } + /** + * @brief Copy a ±1 RMQ index and rebuild internal non-owning views. + * + * @param other Source index. + */ BpPlusMinusOneRmq(const BpPlusMinusOneRmq& other) : input_bits_(other.input_bits_), depth_count_(other.depth_count_), - block_min_values_(other.block_min_values_) { + block_summaries_(other.block_summaries_) { reset_macro_rmq(); } + /** + * @brief Copy-assign a ±1 RMQ index and rebuild internal non-owning views. + * + * @param other Source index. + * @return Reference to this object. + */ BpPlusMinusOneRmq& operator=(const BpPlusMinusOneRmq& other) { if (this == &other) { return *this; } input_bits_ = other.input_bits_; depth_count_ = other.depth_count_; - block_min_values_ = other.block_min_values_; + block_summaries_ = other.block_summaries_; reset_macro_rmq(); return *this; } + /** + * @brief Move a ±1 RMQ index and rebuild internal non-owning views. + * + * @param other Source index. + */ BpPlusMinusOneRmq(BpPlusMinusOneRmq&& other) noexcept : input_bits_(other.input_bits_), depth_count_(other.depth_count_), - block_min_values_(std::move(other.block_min_values_)) { + block_summaries_(std::move(other.block_summaries_)) { reset_macro_rmq(); } + /** + * @brief Move-assign a ±1 RMQ index and rebuild internal non-owning views. + * + * @param other Source index. + * @return Reference to this object. + */ BpPlusMinusOneRmq& operator=(BpPlusMinusOneRmq&& other) noexcept { if (this == &other) { return *this; } input_bits_ = other.input_bits_; depth_count_ = other.depth_count_; - block_min_values_ = std::move(other.block_min_values_); + block_summaries_ = std::move(other.block_summaries_); reset_macro_rmq(); return *this; } + /** + * @brief Return the number of indexed depth positions. + * + * @return The @p depth_count passed to construction. + */ std::size_t size() const { return depth_count_; } + /** + * @brief Whether the indexed depth sequence is empty. + * + * @return `true` when `size() == 0`. + */ bool empty() const { return depth_count_ == 0; } + /** + * @brief Return the first minimum depth position in [@p left, @p right]. + * + * @details The query range is inclusive over depth positions, not delta-bit + * positions. Ties return the smallest position attaining the minimum depth. + * + * @param left First depth position in the query range. + * @param right Last depth position in the query range. + * @return Zero-based position of the first range minimum, or `npos`. + */ std::size_t arg_min(std::size_t left, std::size_t right) const { if (left > right || right >= depth_count_) { return npos; @@ -94,20 +152,20 @@ class BpPlusMinusOneRmq { const std::size_t left_block = left / BlockSize; const std::size_t right_block = right / BlockSize; if (left_block == right_block) { - return scan_block_range(left_block, left % BlockSize, right % BlockSize) - .position; + return scan_block_range_position(left_block, left % BlockSize, + right % BlockSize); } - Candidate answer = scan_block_range(left_block, left % BlockSize, - block_size(left_block) - 1); - answer = - better(answer, scan_block_range(right_block, 0, right % BlockSize)); + const std::size_t left_offset = left % BlockSize; + const std::size_t right_offset = right % BlockSize; + Candidate answer = + scan_block_range(left_block, left_offset, block_size(left_block) - 1); + answer = better(answer, scan_block_range(right_block, 0, right_offset)); if (left_block + 1 < right_block) { const std::size_t block_position = macro_rmq_.arg_min(left_block + 1, right_block - 1); - if (block_position != - SparseTable, Index>::npos) { + if (block_position != MacroRmq::npos) { answer = better(answer, scan_full_block(block_position)); } } @@ -121,11 +179,182 @@ class BpPlusMinusOneRmq { std::int64_t value = std::numeric_limits::max(); }; + /** + * @brief Packed summary for one 128-position depth block. + * + * @details Stores signed 48-bit base depth, signed 48-bit absolute block + * minimum, and 16-bit first local offset of that minimum in two 64-bit words. + * The hot macro-RMQ comparison keeps the sign-biased minimum in the low + * 48 bits of `word0`, so signed ordering is available as one masked unsigned + * comparison. The high 16 bits of `word0` store the local minimum offset, and + * `word1` stores the base depth. + */ + struct alignas(16) BlockSummary { + static constexpr std::uint64_t kSigned48Mask = (std::uint64_t{1} << 48) - 1; + static constexpr std::uint64_t kSigned48SignBit = std::uint64_t{1} << 47; + static constexpr std::int64_t kSigned48Min = -(std::int64_t{1} << 47); + static constexpr std::int64_t kSigned48Max = (std::int64_t{1} << 47) - 1; + + std::uint64_t word0 = 0; + std::uint64_t word1 = 0; + + /** + * @brief Pack block metadata into a 16-byte summary. + * + * @param base_depth Absolute depth at local offset 0. + * @param min_value Absolute minimum depth in the block. + * @param min_offset First local offset attaining @p min_value. + * @return Packed block summary. + * @throws std::length_error if a signed 48-bit field or 16-bit offset + * overflows. + */ + static BlockSummary make(std::int64_t base_depth, + std::int64_t min_value, + std::size_t min_offset) { + if (!fits_signed48(base_depth) || !fits_signed48(min_value)) { + throw std::length_error("RMQ +/-1 block summary depth overflow"); + } + if (min_offset > std::numeric_limits::max()) { + throw std::length_error("RMQ +/-1 block summary offset overflow"); + } + + const std::uint64_t packed_base = pack_signed48(base_depth); + const std::uint64_t ordered_min = pack_ordered48(min_value); + return {ordered_min | (static_cast(min_offset) << 48), + packed_base}; + } + + /** + * @brief Decode the absolute depth at local offset 0. + * + * @return Signed base depth for this block. + */ + std::int64_t base_depth() const { + return unpack_signed48(word1 & kSigned48Mask); + } + + /** + * @brief Decode the absolute minimum depth in this block. + * + * @return Signed block minimum value. + */ + std::int64_t min_value() const { + return unpack_ordered48(ordered_min_value()); + } + + /** + * @brief Return the sign-biased minimum payload for fast comparisons. + * + * @details The returned low-48-bit value preserves signed order under + * unsigned comparison: smaller signed depths have smaller payloads. + * + * @return Sign-biased 48-bit minimum depth payload. + */ + std::uint64_t ordered_min_value() const { return word0 & kSigned48Mask; } + + /** + * @brief Decode the first local minimum offset in this block. + * + * @return Local depth-position offset in [0, BlockSize). + */ + std::size_t min_offset() const { + return static_cast(word0 >> 48); + } + + private: + /** + * @brief Test whether a value fits in the signed 48-bit packed field. + * + * @param value Value to test. + * @return `true` if @p value is representable. + */ + static bool fits_signed48(std::int64_t value) { + return value >= kSigned48Min && value <= kSigned48Max; + } + + /** + * @brief Truncate a signed value to its 48-bit two's-complement payload. + * + * @param value Signed 48-bit value. + * @return Low 48 payload bits. + */ + static std::uint64_t pack_signed48(std::int64_t value) { + return static_cast(value) & kSigned48Mask; + } + + /** + * @brief Encode a signed 48-bit value for unsigned ordered comparison. + * + * @param value Signed 48-bit value. + * @return Low 48 payload bits with the sign bit flipped. + */ + static std::uint64_t pack_ordered48(std::int64_t value) { + return pack_signed48(value) ^ kSigned48SignBit; + } + + /** + * @brief Sign-extend a 48-bit two's-complement payload. + * + * @param value Low 48 payload bits. + * @return Decoded signed value. + */ + static std::int64_t unpack_signed48(std::uint64_t value) { + if ((value & kSigned48SignBit) != 0) { + value |= ~kSigned48Mask; + } + return static_cast(value); + } + + /** + * @brief Decode a sign-biased 48-bit ordered payload. + * + * @param value Low 48 payload bits with the sign bit flipped. + * @return Decoded signed value. + */ + static std::int64_t unpack_ordered48(std::uint64_t value) { + return unpack_signed48(value ^ kSigned48SignBit); + } + }; + + static_assert(sizeof(BlockSummary) == 16); + static_assert(alignof(BlockSummary) == 16); + + struct BlockSummaryMinLess { + /** + * @brief Compare two block summaries by absolute minimum value. + * + * @param left First block summary. + * @param right Second block summary. + * @return `true` if @p left has a strictly smaller block minimum. + */ + bool operator()(const BlockSummary& left, const BlockSummary& right) const { + return left.ordered_min_value() < right.ordered_min_value(); + } + }; + + using MacroRmq = SparseTable; + + /** + * @brief Return the number of depth positions in a block. + * + * @param block Zero-based block index. + * @return `BlockSize` for full blocks, or the tail size for the last block. + */ std::size_t block_size(std::size_t block) const { const std::size_t begin = block * BlockSize; return std::min(BlockSize, depth_count_ - begin); } + /** + * @brief Choose the better of two candidate minima. + * + * @details Missing candidates use `npos`. Smaller values win; equal values + * return the smaller global position. + * + * @param left First candidate. + * @param right Second candidate. + * @return Selected candidate. + */ Candidate better(Candidate left, Candidate right) const { if (left.position == npos) { return right; @@ -142,9 +371,18 @@ class BpPlusMinusOneRmq { return right.position < left.position ? right : left; } + /** + * @brief Build block summaries and the macro sparse table. + * + * @details Computes the absolute base depth, absolute block minimum, and + * first local minimum offset for each 128-position block. + * + * @throws std::length_error if @p Index or a packed block summary overflows. + * @throws std::invalid_argument if the input bit span is too small. + */ void build() { - block_min_values_.clear(); - macro_rmq_ = SparseTable, Index>(); + block_summaries_.clear(); + macro_rmq_ = MacroRmq(); if (depth_count_ == 0) { return; @@ -157,12 +395,13 @@ class BpPlusMinusOneRmq { } const std::size_t block_count = (depth_count_ + BlockSize - 1) / BlockSize; - block_min_values_.reserve(block_count); + block_summaries_.reserve(block_count); std::int64_t base_depth = 0; for (std::size_t block = 0; block < block_count; ++block) { const std::size_t begin = block * BlockSize; const std::size_t size = std::min(BlockSize, depth_count_ - begin); + std::size_t min_offset = 0; std::int64_t min_depth = base_depth; std::int64_t current_depth = base_depth; for (std::size_t offset = 1; offset < size; ++offset) { @@ -171,10 +410,12 @@ class BpPlusMinusOneRmq { current_depth += up ? 1 : -1; if (current_depth < min_depth) { min_depth = current_depth; + min_offset = offset; } } - block_min_values_.push_back(min_depth); + block_summaries_.push_back( + BlockSummary::make(base_depth, min_depth, min_offset)); if (block + 1 < block_count) { base_depth += block_excess(begin, next_block_delta_count(begin)); } @@ -183,24 +424,56 @@ class BpPlusMinusOneRmq { reset_macro_rmq(); } + /** + * @brief Read one BP delta bit. + * + * @param position Zero-based delta-bit position. + * @return `true` for a +1 step and `false` for a -1 step. + */ bool bit(std::size_t position) const { return ((input_bits_[position >> 6] >> (position & 63)) & 1u) != 0; } + /** + * @brief Read an input word or return zero past the available span. + * + * @param word Zero-based 64-bit word index. + * @return Input word value, or zero when @p word is out of range. + */ std::uint64_t word_or_zero(std::size_t word) const { return word < input_bits_.size() ? input_bits_[word] : 0; } + /** + * @brief Load the two 64-bit words backing a 128-position block. + * + * @param block Zero-based block index. + * @return Pair of words suitable for `excess_min_128`. + */ std::array block_bits(std::size_t block) const { const std::size_t first_word = block * (BlockSize / 64); return {word_or_zero(first_word), word_or_zero(first_word + 1)}; } + /** + * @brief Count delta bits from a block start to the next block boundary. + * + * @param begin Global depth-position offset of the block start. + * @return Number of delta bits contributing to the transition to the next + * block base depth. + */ std::size_t next_block_delta_count(std::size_t begin) const { const std::size_t next_begin = begin + BlockSize; return std::min(next_begin, depth_count_ - 1) - begin; } + /** + * @brief Compute total excess over a contiguous delta-bit range. + * + * @param begin First delta-bit position. + * @param delta_count Number of delta bits to scan. + * @return Sum of +1/-1 deltas in the range. + */ std::int64_t block_excess(std::size_t begin, std::size_t delta_count) const { std::int64_t excess = 0; for (std::size_t i = 0; i < delta_count; ++i) { @@ -209,13 +482,43 @@ class BpPlusMinusOneRmq { return excess; } - std::int64_t block_base_depth(std::size_t block, - const std::array& bits, - std::size_t size) const { - const ExcessResult full_min = excess_min_128(bits.data(), 0, size - 1); - return block_min_values_[block] - full_min.min_excess; + /** + * @brief Return only the minimum position inside one block range. + * + * @details Used for same-block queries where the absolute minimum value is + * not needed. The range is inclusive in local block offsets. + * + * @param block Zero-based block index. + * @param left_offset First local depth-position offset. + * @param right_offset Last local depth-position offset. + * @return Global position of the first local minimum, or `npos`. + */ + std::size_t scan_block_range_position(std::size_t block, + std::size_t left_offset, + std::size_t right_offset) const { + const std::size_t begin = block * BlockSize; + const std::size_t size = block_size(block); + right_offset = std::min(right_offset, size - 1); + const auto bits = block_bits(block); + const ExcessResult result = + excess_min_128(bits.data(), left_offset, right_offset); + if (result.offset == npos || result.offset >= size) { + return npos; + } + return begin + result.offset; } + /** + * @brief Scan an inclusive local range inside one block. + * + * @details Uses `excess_min_128` for the relative minimum and combines it + * with the stored block base depth to return an absolute-depth candidate. + * + * @param block Zero-based block index. + * @param left_offset First local depth-position offset. + * @param right_offset Last local depth-position offset. + * @return Candidate containing global position and absolute depth. + */ Candidate scan_block_range(std::size_t block, std::size_t left_offset, std::size_t right_offset) const { @@ -223,35 +526,41 @@ class BpPlusMinusOneRmq { const std::size_t size = block_size(block); right_offset = std::min(right_offset, size - 1); const auto bits = block_bits(block); - const std::int64_t base_depth = block_base_depth(block, bits, size); const ExcessResult result = excess_min_128(bits.data(), left_offset, right_offset); if (result.offset == npos || result.offset >= size) { return {}; } - return {begin + result.offset, base_depth + result.min_excess}; + return {begin + result.offset, + block_summaries_[block].base_depth() + result.min_excess}; } + /** + * @brief Return the precomputed minimum candidate for a full block. + * + * @param block Zero-based block index. + * @return Candidate containing global position and absolute depth. + */ Candidate scan_full_block(std::size_t block) const { const std::size_t begin = block * BlockSize; - const std::size_t size = block_size(block); - const auto bits = block_bits(block); - const ExcessResult result = excess_min_128(bits.data(), 0, size - 1); - if (result.offset == npos || result.offset >= size) { - return {}; - } - return {begin + result.offset, block_min_values_[block]}; + return {begin + block_summaries_[block].min_offset(), + block_summaries_[block].min_value()}; } + /** + * @brief Rebuild the macro sparse table over block summaries. + * + * @details Called after build, copy, and move operations because the sparse + * table stores a non-owning span into this object's block-summary vector. + */ void reset_macro_rmq() { - macro_rmq_ = SparseTable, Index>( - std::span(block_min_values_)); + macro_rmq_ = MacroRmq(std::span(block_summaries_)); } std::span input_bits_; std::size_t depth_count_ = 0; - std::vector block_min_values_; - SparseTable, Index> macro_rmq_; + std::vector block_summaries_; + MacroRmq macro_rmq_; }; } // namespace pixie::rmq diff --git a/include/pixie/rmq/cartesian_tree_rmq.h b/include/pixie/rmq/cartesian_tree_rmq.h index db7d315..6048996 100644 --- a/include/pixie/rmq/cartesian_tree_rmq.h +++ b/include/pixie/rmq/cartesian_tree_rmq.h @@ -35,14 +35,32 @@ class CartesianTreeRmq RmqBase, T>::npos; static constexpr Index invalid_index = std::numeric_limits::max(); + /** + * @brief Construct an empty Cartesian-tree RMQ index. + */ CartesianTreeRmq() = default; + /** + * @brief Build a Cartesian-tree RMQ index over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values stay stable: the smaller index remains the first minimum. + * + * @param values Values to index. + * @param compare Ordering used to choose minima. + * @throws std::length_error if @p Index cannot represent all positions. + */ explicit CartesianTreeRmq(std::span values, Compare compare = Compare()) : values_(values), compare_(compare) { build(); } + /** + * @brief Copy an RMQ index and rebuild internal non-owning views. + * + * @param other Source index. + */ CartesianTreeRmq(const CartesianTreeRmq& other) : values_(other.values_), compare_(other.compare_), @@ -55,6 +73,12 @@ class CartesianTreeRmq reset_depth_rmq(); } + /** + * @brief Copy-assign an RMQ index and rebuild internal non-owning views. + * + * @param other Source index. + * @return Reference to this object. + */ CartesianTreeRmq& operator=(const CartesianTreeRmq& other) { if (this == &other) { return *this; @@ -71,6 +95,11 @@ class CartesianTreeRmq return *this; } + /** + * @brief Move an RMQ index and rebuild internal non-owning views. + * + * @param other Source index. + */ CartesianTreeRmq(CartesianTreeRmq&& other) noexcept : values_(other.values_), compare_(std::move(other.compare_)), @@ -83,6 +112,12 @@ class CartesianTreeRmq reset_depth_rmq(); } + /** + * @brief Move-assign an RMQ index and rebuild internal non-owning views. + * + * @param other Source index. + * @return Reference to this object. + */ CartesianTreeRmq& operator=(CartesianTreeRmq&& other) noexcept { if (this == &other) { return *this; @@ -99,10 +134,32 @@ class CartesianTreeRmq return *this; } + /** + * @brief Return the number of indexed values. + * + * @return `values.size()` from construction. + */ std::size_t size_impl() const { return values_.size(); } + /** + * @brief Return the value at an indexed position. + * + * @param position Zero-based position in the indexed values. + * @return Copy of the value at @p position. + */ T value_at_impl(std::size_t position) const { return values_[position]; } + /** + * @brief Return the first minimum position in [@p left, @p right]. + * + * @details Converts the query to an LCA query over the Cartesian-tree Euler + * tour and returns the corresponding original array position. Ties return the + * smaller original position because the Cartesian tree is stable. + * + * @param left First position in the query range. + * @param right Last position in the query range. + * @return Zero-based position of the first range minimum, or `npos`. + */ std::size_t arg_min_impl(std::size_t left, std::size_t right) const { if (left > right || right >= values_.size()) { return npos; @@ -119,11 +176,30 @@ class CartesianTreeRmq return euler_nodes_[euler_position]; } + /** + * @brief Return the Euler-tour node sequence used by the reduction. + * + * @return Non-owning span of original array positions in Euler-tour order. + */ std::span euler_nodes() const { return euler_nodes_; } + /** + * @brief Return the Euler-tour depth sequence used by the reduction. + * + * @return Non-owning span of depths corresponding to `euler_nodes()`. + */ std::span euler_depths() const { return depths_; } private: + /** + * @brief Rebuild all Cartesian-tree and Euler-tour auxiliary data. + * + * @details Clears previous state, builds a stable Cartesian tree, records its + * Euler tour, converts adjacent Euler-depth deltas to bits, and rebuilds the + * ±1 RMQ backend. + * + * @throws std::length_error if @p Index cannot represent all positions. + */ void build() { left_child_.clear(); right_child_.clear(); @@ -152,6 +228,15 @@ class CartesianTreeRmq reset_depth_rmq(); } + /** + * @brief Build the stable min Cartesian tree. + * + * @details Uses the standard monotone-stack construction. Strictly smaller + * values become ancestors; equal values are not popped, preserving first + * minimum tie-breaking. + * + * @return Root node position in the original value array. + */ std::size_t build_cartesian_tree() { std::vector stack; stack.reserve(values_.size()); @@ -174,6 +259,15 @@ class CartesianTreeRmq return stack.front(); } + /** + * @brief Append the Euler tour of a Cartesian-tree subtree. + * + * @details Visits @p node, recurses into each existing child, and appends + * @p node again after returning from that child. + * + * @param node Current Cartesian-tree node. + * @param depth Depth of @p node in the Cartesian tree. + */ void euler_tour(std::size_t node, std::int64_t depth) { append_euler(node, depth); if (left_child_[node] != invalid_index) { @@ -186,6 +280,15 @@ class CartesianTreeRmq } } + /** + * @brief Append one node/depth pair to the Euler-tour arrays. + * + * @details Records the first Euler occurrence of @p node if this is the first + * time the node is appended. + * + * @param node Cartesian-tree node, also an original value position. + * @param depth Depth of @p node in the Cartesian tree. + */ void append_euler(std::size_t node, std::int64_t depth) { if (first_occurrence_[node] == invalid_index) { first_occurrence_[node] = static_cast(euler_nodes_.size()); @@ -194,11 +297,24 @@ class CartesianTreeRmq depths_.push_back(depth); } + /** + * @brief Rebuild the ±1 RMQ backend over the current Euler-depth deltas. + * + * @details Called after build, copy, and move operations because the backend + * stores non-owning spans into this object's `euler_delta_bits_` storage. + */ void reset_depth_rmq() { depth_rmq_ = BpPlusMinusOneRmq( std::span(euler_delta_bits_), depths_.size()); } + /** + * @brief Pack adjacent Euler-depth changes into BP-style delta bits. + * + * @details Bit `1` means the next Euler depth is current depth + 1; bit `0` + * means current depth - 1. Cartesian-tree Euler tours have only ±1 adjacent + * depth changes. + */ void build_euler_delta_bits() { euler_delta_bits_.assign((depths_.size() - 1 + 63) / 64, 0); for (std::size_t i = 1; i < depths_.size(); ++i) { diff --git a/include/pixie/rmq/rmq_base.h b/include/pixie/rmq/rmq_base.h index cc919da..a188924 100644 --- a/include/pixie/rmq/rmq_base.h +++ b/include/pixie/rmq/rmq_base.h @@ -22,16 +22,27 @@ class RmqBase { /** * @brief Number of indexed values. + * + * @return The number of values covered by the underlying RMQ index. */ std::size_t size() const { return impl().size_impl(); } /** * @brief Whether the indexed array is empty. + * + * @return `true` when `size() == 0`. */ bool empty() const { return size() == 0; } /** * @brief Return the first minimum position in [@p left, @p right]. + * + * @details The query range is inclusive. Ties are resolved by returning the + * smallest position attaining the minimum. Invalid ranges return `npos`. + * + * @param left First position in the query range. + * @param right Last position in the query range. + * @return Zero-based position of the first range minimum, or `npos`. */ std::size_t arg_min(std::size_t left, std::size_t right) const { return impl().arg_min_impl(left, right); @@ -39,7 +50,12 @@ class RmqBase { /** * @brief Return the minimum value in [@p left, @p right]. + * * @details Invalid ranges return a default-constructed value. + * + * @param left First position in the query range. + * @param right Last position in the query range. + * @return The minimum value in the inclusive range, or `Value{}`. */ Value range_min(std::size_t left, std::size_t right) const { const std::size_t position = arg_min(left, right); @@ -50,6 +66,11 @@ class RmqBase { } private: + /** + * @brief Return this object as its concrete CRTP implementation. + * + * @return Reference to the derived RMQ implementation. + */ const Impl& impl() const { return static_cast(*this); } }; diff --git a/include/pixie/rmq/segment_tree.h b/include/pixie/rmq/segment_tree.h index 5e734a8..3276c78 100644 --- a/include/pixie/rmq/segment_tree.h +++ b/include/pixie/rmq/segment_tree.h @@ -31,17 +31,51 @@ class SegmentTree : public RmqBase, T> { RmqBase, T>::npos; static constexpr Index invalid_index = std::numeric_limits::max(); + /** + * @brief Construct an empty segment tree. + */ SegmentTree() = default; + /** + * @brief Build an iterative segment tree over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values keep the smaller index as the RMQ answer. + * + * @param values Values to index. + * @param compare Ordering used to choose minima. + * @throws std::length_error if @p Index cannot represent all positions. + */ explicit SegmentTree(std::span values, Compare compare = Compare()) : values_(values), compare_(compare) { build(); } + /** + * @brief Return the number of indexed values. + * + * @return `values.size()` from construction. + */ std::size_t size_impl() const { return values_.size(); } + /** + * @brief Return the value at an indexed position. + * + * @param position Zero-based position in the indexed values. + * @return Copy of the value at @p position. + */ T value_at_impl(std::size_t position) const { return values_[position]; } + /** + * @brief Return the first minimum position in [@p left, @p right]. + * + * @details Answers in O(log n) by walking the flat iterative segment tree. + * Ties return the smaller position. + * + * @param left First position in the query range. + * @param right Last position in the query range. + * @return Zero-based position of the first range minimum, or `npos`. + */ std::size_t arg_min_impl(std::size_t left, std::size_t right) const { if (left > right || right >= values_.size()) { return npos; @@ -69,6 +103,17 @@ class SegmentTree : public RmqBase, T> { } private: + /** + * @brief Choose the better of two candidate positions. + * + * @details `npos` and `invalid_index` are treated as missing. If both values + * compare equal, the smaller position wins to preserve first-minimum + * semantics. + * + * @param left First candidate position, `npos`, or `invalid_index`. + * @param right Second candidate position, `npos`, or `invalid_index`. + * @return Position of the selected candidate. + */ std::size_t better(std::size_t left, std::size_t right) const { if (left == npos || left == invalid_index) { return right; @@ -85,6 +130,15 @@ class SegmentTree : public RmqBase, T> { return std::min(left, right); } + /** + * @brief Build the flat iterative segment tree. + * + * @details Leaves start at `leaf_base_`, which is the next power of two. + * Unused leaves contain `invalid_index`, and internal nodes store the first + * minimum position of their covered segment. + * + * @throws std::length_error if @p Index cannot represent all positions. + */ void build() { tree_.clear(); leaf_base_ = 0; diff --git a/include/pixie/rmq/sparse_table.h b/include/pixie/rmq/sparse_table.h index d2ea6ab..4e963a7 100644 --- a/include/pixie/rmq/sparse_table.h +++ b/include/pixie/rmq/sparse_table.h @@ -31,17 +31,51 @@ class SparseTable : public RmqBase, T> { RmqBase, T>::npos; static constexpr Index invalid_index = std::numeric_limits::max(); + /** + * @brief Construct an empty sparse table. + */ SparseTable() = default; + /** + * @brief Build a sparse table over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values keep the smaller index as the RMQ answer. + * + * @param values Values to index. + * @param compare Ordering used to choose minima. + * @throws std::length_error if @p Index cannot represent all positions. + */ explicit SparseTable(std::span values, Compare compare = Compare()) : values_(values), compare_(compare) { build(); } + /** + * @brief Return the number of indexed values. + * + * @return `values.size()` from construction. + */ std::size_t size_impl() const { return values_.size(); } + /** + * @brief Return the value at an indexed position. + * + * @param position Zero-based position in the indexed values. + * @return Copy of the value at @p position. + */ T value_at_impl(std::size_t position) const { return values_[position]; } + /** + * @brief Return the first minimum position in [@p left, @p right]. + * + * @details Answers in O(1) by comparing the two power-of-two ranges covering + * the inclusive query interval. Ties return the smaller position. + * + * @param left First position in the query range. + * @param right Last position in the query range. + * @return Zero-based position of the first range minimum, or `npos`. + */ std::size_t arg_min_impl(std::size_t left, std::size_t right) const { if (left > right || right >= values_.size()) { return npos; @@ -55,6 +89,16 @@ class SparseTable : public RmqBase, T> { } private: + /** + * @brief Choose the better of two candidate positions. + * + * @details `npos` is treated as missing. If both values compare equal, the + * smaller position wins to preserve first-minimum semantics. + * + * @param left First candidate position, or `npos`. + * @param right Second candidate position, or `npos`. + * @return Position of the selected candidate. + */ std::size_t better(std::size_t left, std::size_t right) const { if (left == npos) { return right; @@ -71,6 +115,14 @@ class SparseTable : public RmqBase, T> { return std::min(left, right); } + /** + * @brief Build all sparse-table levels over the indexed values. + * + * @details Level 0 stores singleton positions. Each higher level stores the + * first minimum of two adjacent half ranges from the previous level. + * + * @throws std::length_error if @p Index cannot represent all positions. + */ void build() { table_.clear(); if (values_.empty()) { diff --git a/src/benchmarks/bench_rmq.cpp b/src/benchmarks/bench_rmq.cpp index 4067e7e..abd4b46 100644 --- a/src/benchmarks/bench_rmq.cpp +++ b/src/benchmarks/bench_rmq.cpp @@ -1,7 +1,9 @@ #include +#include #include #include +#include #include #include #include @@ -12,6 +14,7 @@ namespace { constexpr std::uint64_t kSeed = 42; constexpr std::size_t kQueryCount = 32768; +constexpr std::size_t kBpBlockSize = 128; using Index = std::size_t; struct Dataset { @@ -29,6 +32,82 @@ struct DepthDataset { std::vector> ranges; }; +struct BpQueryShape { + std::size_t same_block = 0; + std::size_t cross_block = 0; + std::size_t middle_block = 0; + std::size_t disjoint_boundary = 0; + std::size_t fused_boundary = 0; + std::size_t excess_calls = 0; + std::size_t left_boundary_width = 0; + std::size_t right_boundary_width = 0; + std::size_t same_block_width = 0; +}; + +struct BlockRange { + std::size_t block = 0; + std::size_t left = 0; + std::size_t right = 0; +}; + +struct MacroRange { + std::size_t left = 0; + std::size_t right = 0; +}; + +struct alignas(16) BenchBlockSummary { + static constexpr std::uint64_t kSigned48Mask = (std::uint64_t{1} << 48) - 1; + static constexpr std::uint64_t kSigned48SignBit = std::uint64_t{1} << 47; + + std::uint64_t word0 = 0; + std::uint64_t word1 = 0; + + static BenchBlockSummary make(std::int64_t base_depth, + std::int64_t min_value, + std::size_t min_offset) { + const std::uint64_t packed_base = pack_signed48(base_depth); + const std::uint64_t ordered_min = pack_ordered48(min_value); + return {ordered_min | (static_cast(min_offset) << 48), + packed_base}; + } + + std::int64_t min_value() const { + return unpack_ordered48(ordered_min_value()); + } + + std::uint64_t ordered_min_value() const { return word0 & kSigned48Mask; } + + private: + static std::uint64_t pack_signed48(std::int64_t value) { + return static_cast(value) & kSigned48Mask; + } + + static std::uint64_t pack_ordered48(std::int64_t value) { + return pack_signed48(value) ^ kSigned48SignBit; + } + + static std::int64_t unpack_signed48(std::uint64_t value) { + if ((value & kSigned48SignBit) != 0) { + value |= ~kSigned48Mask; + } + return static_cast(value); + } + + static std::int64_t unpack_ordered48(std::uint64_t value) { + return unpack_signed48(value ^ kSigned48SignBit); + } +}; + +static_assert(sizeof(BenchBlockSummary) == 16); +static_assert(alignof(BenchBlockSummary) == 16); + +struct BenchBlockSummaryMinLess { + bool operator()(const BenchBlockSummary& left, + const BenchBlockSummary& right) const { + return left.ordered_min_value() < right.ordered_min_value(); + } +}; + Dataset make_dataset(std::size_t size, std::size_t max_width) { Dataset dataset; dataset.size = size; @@ -83,6 +162,207 @@ DepthDataset make_depth_dataset(std::size_t size, std::size_t max_width) { return dataset; } +std::size_t bp_block_size(const DepthDataset& dataset, std::size_t block) { + const std::size_t begin = block * kBpBlockSize; + return std::min(kBpBlockSize, dataset.depths.size() - begin); +} + +std::array bp_block_bits_stack(const DepthDataset& dataset, + std::size_t block) { + const std::size_t first_word = block * (kBpBlockSize / 64); + const std::uint64_t lo = + first_word < dataset.bits.size() ? dataset.bits[first_word] : 0; + const std::uint64_t hi = + first_word + 1 < dataset.bits.size() ? dataset.bits[first_word + 1] : 0; + return {lo, hi}; +} + +const std::uint64_t* bp_block_bits_direct(const DepthDataset& dataset, + std::size_t block) { + return dataset.bits.data() + block * (kBpBlockSize / 64); +} + +BpQueryShape compute_bp_query_shape(const DepthDataset& dataset) { + BpQueryShape shape; + for (const auto [left, right] : dataset.ranges) { + const std::size_t left_block = left / kBpBlockSize; + const std::size_t right_block = right / kBpBlockSize; + if (left_block == right_block) { + ++shape.same_block; + ++shape.excess_calls; + shape.same_block_width += right - left + 1; + continue; + } + + ++shape.cross_block; + const std::size_t left_offset = left % kBpBlockSize; + const std::size_t right_offset = right % kBpBlockSize; + const std::size_t left_width = kBpBlockSize - left_offset; + const std::size_t right_width = right_offset + 1; + if (left_offset > right_offset) { + ++shape.disjoint_boundary; + if ((kBpBlockSize - 1 - left_offset) + right_offset >= 32) { + ++shape.fused_boundary; + } + } + shape.excess_calls += 2; + shape.left_boundary_width += left_width; + shape.right_boundary_width += right_width; + if (left_block + 1 < right_block) { + ++shape.middle_block; + } + } + return shape; +} + +void set_depth_counters(benchmark::State& state, + const DepthDataset& dataset, + bool include_shape) { + state.counters["N"] = static_cast(dataset.size); + state.counters["max_width"] = static_cast(dataset.max_width); + state.counters["index_bytes"] = static_cast(sizeof(Index)); + + if (!include_shape) { + return; + } + + const BpQueryShape shape = compute_bp_query_shape(dataset); + const double query_count = static_cast(dataset.ranges.size()); + const double cross_count = static_cast(shape.cross_block); + const double same_count = static_cast(shape.same_block); + state.counters["same_block_ratio"] = + static_cast(shape.same_block) / query_count; + state.counters["cross_block_ratio"] = + static_cast(shape.cross_block) / query_count; + state.counters["middle_block_ratio"] = + static_cast(shape.middle_block) / query_count; + state.counters["disjoint_boundary_ratio"] = + static_cast(shape.disjoint_boundary) / query_count; + state.counters["fused_boundary_eligible_ratio"] = + static_cast(shape.fused_boundary) / query_count; + state.counters["excess_calls_per_query"] = + static_cast(shape.excess_calls) / query_count; + state.counters["avg_same_width"] = + same_count == 0 + ? 0.0 + : static_cast(shape.same_block_width) / same_count; + state.counters["avg_left_boundary_width"] = + cross_count == 0 + ? 0.0 + : static_cast(shape.left_boundary_width) / cross_count; + state.counters["avg_right_boundary_width"] = + cross_count == 0 + ? 0.0 + : static_cast(shape.right_boundary_width) / cross_count; +} + +std::vector make_same_block_ranges(const DepthDataset& dataset) { + std::vector out; + out.reserve(dataset.ranges.size()); + for (const auto [left, right] : dataset.ranges) { + const std::size_t left_block = left / kBpBlockSize; + const std::size_t right_block = right / kBpBlockSize; + if (left_block == right_block) { + out.push_back({left_block, left % kBpBlockSize, right % kBpBlockSize}); + } + } + return out; +} + +std::vector make_left_boundary_ranges(const DepthDataset& dataset) { + std::vector out; + out.reserve(dataset.ranges.size()); + for (const auto [left, right] : dataset.ranges) { + const std::size_t left_block = left / kBpBlockSize; + const std::size_t right_block = right / kBpBlockSize; + if (left_block != right_block) { + out.push_back({left_block, left % kBpBlockSize, + bp_block_size(dataset, left_block) - 1}); + } + } + return out; +} + +std::vector make_right_boundary_ranges( + const DepthDataset& dataset) { + std::vector out; + out.reserve(dataset.ranges.size()); + for (const auto [left, right] : dataset.ranges) { + const std::size_t left_block = left / kBpBlockSize; + const std::size_t right_block = right / kBpBlockSize; + if (left_block != right_block) { + out.push_back({right_block, 0, right % kBpBlockSize}); + } + } + return out; +} + +std::vector> make_boundary_pairs( + const DepthDataset& dataset) { + std::vector> out; + out.reserve(dataset.ranges.size()); + for (const auto [left, right] : dataset.ranges) { + const std::size_t left_block = left / kBpBlockSize; + const std::size_t right_block = right / kBpBlockSize; + if (left_block != right_block) { + out.push_back({{left_block, left % kBpBlockSize, + bp_block_size(dataset, left_block) - 1}, + {right_block, 0, right % kBpBlockSize}}); + } + } + return out; +} + +std::vector> make_disjoint_boundary_pairs( + const DepthDataset& dataset) { + std::vector> out; + out.reserve(dataset.ranges.size()); + for (const auto [left, right] : dataset.ranges) { + const std::size_t left_block = left / kBpBlockSize; + const std::size_t right_block = right / kBpBlockSize; + const std::size_t left_offset = left % kBpBlockSize; + const std::size_t right_offset = right % kBpBlockSize; + if (left_block != right_block && left_offset > right_offset) { + out.push_back( + {{left_block, left_offset, bp_block_size(dataset, left_block) - 1}, + {right_block, 0, right_offset}}); + } + } + return out; +} + +std::vector make_macro_ranges(const DepthDataset& dataset) { + std::vector out; + out.reserve(dataset.ranges.size()); + for (const auto [left, right] : dataset.ranges) { + const std::size_t left_block = left / kBpBlockSize; + const std::size_t right_block = right / kBpBlockSize; + if (left_block + 1 < right_block) { + out.push_back({left_block + 1, right_block - 1}); + } + } + return out; +} + +std::vector make_block_summaries( + const DepthDataset& dataset) { + const std::size_t block_count = + (dataset.depths.size() + kBpBlockSize - 1) / kBpBlockSize; + std::vector summaries; + summaries.reserve(block_count); + for (std::size_t block = 0; block < block_count; ++block) { + const std::size_t begin = block * kBpBlockSize; + const std::size_t end = + std::min(begin + kBpBlockSize, dataset.depths.size()); + auto min_it = std::min_element(dataset.depths.begin() + begin, + dataset.depths.begin() + end); + summaries.push_back(BenchBlockSummary::make( + dataset.depths[begin], *min_it, + static_cast(min_it - dataset.depths.begin() - begin))); + } + return summaries; +} + template void run_queries(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); @@ -103,6 +383,166 @@ void run_queries(benchmark::State& state) { state.counters["index_bytes"] = static_cast(sizeof(Index)); } +void run_bp_boundary_stack( + benchmark::State& state, + std::vector (*make_ranges)(const DepthDataset&)) { + const std::size_t size = static_cast(state.range(0)); + const std::size_t max_width = static_cast(state.range(1)); + const DepthDataset dataset = make_depth_dataset(size, max_width); + const std::vector ranges = make_ranges(dataset); + if (ranges.empty()) { + state.SkipWithError("no diagnostic ranges for this size/width"); + return; + } + + std::size_t query_index = 0; + for (auto _ : state) { + const BlockRange range = ranges[query_index++ % ranges.size()]; + const auto bits = bp_block_bits_stack(dataset, range.block); + ExcessResult result = excess_min_128(bits.data(), range.left, range.right); + benchmark::DoNotOptimize(result.min_excess); + benchmark::DoNotOptimize(result.offset); + } + + set_depth_counters(state, dataset, false); + state.counters["diagnostic_ranges"] = static_cast(ranges.size()); +} + +void run_bp_boundary_pair_stack(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const std::size_t max_width = static_cast(state.range(1)); + const DepthDataset dataset = make_depth_dataset(size, max_width); + const auto ranges = make_boundary_pairs(dataset); + if (ranges.empty()) { + state.SkipWithError("no cross-block boundary pairs for this size/width"); + return; + } + + std::size_t query_index = 0; + for (auto _ : state) { + const auto [left, right] = ranges[query_index++ % ranges.size()]; + const auto left_bits = bp_block_bits_stack(dataset, left.block); + const auto right_bits = bp_block_bits_stack(dataset, right.block); + ExcessResult left_result = + excess_min_128(left_bits.data(), left.left, left.right); + ExcessResult right_result = + excess_min_128(right_bits.data(), right.left, right.right); + benchmark::DoNotOptimize(left_result.min_excess); + benchmark::DoNotOptimize(left_result.offset); + benchmark::DoNotOptimize(right_result.min_excess); + benchmark::DoNotOptimize(right_result.offset); + } + + set_depth_counters(state, dataset, false); + state.counters["diagnostic_ranges"] = static_cast(ranges.size()); +} + +void run_bp_boundary_pair_direct(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const std::size_t max_width = static_cast(state.range(1)); + const DepthDataset dataset = make_depth_dataset(size, max_width); + const auto ranges = make_boundary_pairs(dataset); + if (ranges.empty()) { + state.SkipWithError("no cross-block boundary pairs for this size/width"); + return; + } + + std::size_t query_index = 0; + for (auto _ : state) { + const auto [left, right] = ranges[query_index++ % ranges.size()]; + ExcessResult left_result = excess_min_128( + bp_block_bits_direct(dataset, left.block), left.left, left.right); + ExcessResult right_result = excess_min_128( + bp_block_bits_direct(dataset, right.block), right.left, right.right); + benchmark::DoNotOptimize(left_result.min_excess); + benchmark::DoNotOptimize(left_result.offset); + benchmark::DoNotOptimize(right_result.min_excess); + benchmark::DoNotOptimize(right_result.offset); + } + + set_depth_counters(state, dataset, false); + state.counters["diagnostic_ranges"] = static_cast(ranges.size()); +} + +void run_bp_boundary_pair_disjoint_direct(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const std::size_t max_width = static_cast(state.range(1)); + const DepthDataset dataset = make_depth_dataset(size, max_width); + const auto ranges = make_disjoint_boundary_pairs(dataset); + if (ranges.empty()) { + state.SkipWithError("no disjoint cross-block boundary pairs"); + return; + } + + std::size_t query_index = 0; + for (auto _ : state) { + const auto [left, right] = ranges[query_index++ % ranges.size()]; + ExcessResult left_result = excess_min_128( + bp_block_bits_direct(dataset, left.block), left.left, left.right); + ExcessResult right_result = excess_min_128( + bp_block_bits_direct(dataset, right.block), right.left, right.right); + benchmark::DoNotOptimize(left_result.min_excess); + benchmark::DoNotOptimize(left_result.offset); + benchmark::DoNotOptimize(right_result.min_excess); + benchmark::DoNotOptimize(right_result.offset); + } + + set_depth_counters(state, dataset, false); + state.counters["diagnostic_ranges"] = static_cast(ranges.size()); +} + +void run_bp_boundary_pair_fused(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const std::size_t max_width = static_cast(state.range(1)); + const DepthDataset dataset = make_depth_dataset(size, max_width); + const auto ranges = make_disjoint_boundary_pairs(dataset); + if (ranges.empty()) { + state.SkipWithError("no disjoint cross-block boundary pairs"); + return; + } + + std::size_t query_index = 0; + for (auto _ : state) { + const auto [left, right] = ranges[query_index++ % ranges.size()]; + ExcessBoundaryPairResult result = excess_min_128_disjoint_suffix_prefix( + bp_block_bits_direct(dataset, left.block), left.left, + bp_block_bits_direct(dataset, right.block), right.right); + benchmark::DoNotOptimize(result.suffix.min_excess); + benchmark::DoNotOptimize(result.suffix.offset); + benchmark::DoNotOptimize(result.prefix.min_excess); + benchmark::DoNotOptimize(result.prefix.offset); + } + + set_depth_counters(state, dataset, false); + state.counters["diagnostic_ranges"] = static_cast(ranges.size()); +} + +void run_bp_macro_only(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const std::size_t max_width = static_cast(state.range(1)); + const DepthDataset dataset = make_depth_dataset(size, max_width); + const std::vector block_summaries = + make_block_summaries(dataset); + const pixie::rmq::SparseTable + macro_rmq{std::span(block_summaries)}; + const std::vector ranges = make_macro_ranges(dataset); + if (ranges.empty()) { + state.SkipWithError("no middle-block macro ranges for this size/width"); + return; + } + + std::size_t query_index = 0; + for (auto _ : state) { + const MacroRange range = ranges[query_index++ % ranges.size()]; + std::size_t result = macro_rmq.arg_min(range.left, range.right); + benchmark::DoNotOptimize(result); + } + + set_depth_counters(state, dataset, false); + state.counters["diagnostic_ranges"] = static_cast(ranges.size()); +} + template void run_depth_queries(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); @@ -119,9 +559,7 @@ void run_depth_queries(benchmark::State& state) { benchmark::DoNotOptimize(result); } - state.counters["N"] = static_cast(size); - state.counters["max_width"] = static_cast(max_width); - state.counters["index_bytes"] = static_cast(sizeof(Index)); + set_depth_counters(state, dataset, true); } void register_benchmarks() { @@ -172,6 +610,54 @@ void register_benchmarks() { ->Args({static_cast(size), static_cast(width)}) ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark("rmq_bp_diag_same_block_boundary", + [](benchmark::State& state) { + run_bp_boundary_stack( + state, make_same_block_ranges); + }) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark("rmq_bp_diag_left_boundary", + [](benchmark::State& state) { + run_bp_boundary_stack( + state, make_left_boundary_ranges); + }) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark("rmq_bp_diag_right_boundary", + [](benchmark::State& state) { + run_bp_boundary_stack( + state, make_right_boundary_ranges); + }) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark("rmq_bp_diag_boundary_pair_stack", + run_bp_boundary_pair_stack) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark("rmq_bp_diag_boundary_pair_direct", + run_bp_boundary_pair_direct) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark("rmq_bp_diag_boundary_pair_disjoint_direct", + run_bp_boundary_pair_disjoint_direct) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark("rmq_bp_diag_boundary_pair_fused", + run_bp_boundary_pair_fused) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark("rmq_bp_diag_macro_only", run_bp_macro_only) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); } } } diff --git a/src/benchmarks/excess_positions_benchmarks.cpp b/src/benchmarks/excess_positions_benchmarks.cpp index 0e0850f..148ebe6 100644 --- a/src/benchmarks/excess_positions_benchmarks.cpp +++ b/src/benchmarks/excess_positions_benchmarks.cpp @@ -63,6 +63,19 @@ static std::vector> make_128_ranges( return ranges; } +static std::vector> make_disjoint_boundary_ranges( + size_t num_ranges = 4096) { + std::mt19937_64 rng(45); + std::uniform_int_distribution prefix_dist(0, 126); + std::vector> ranges(num_ranges); + for (auto& [suffix_left, prefix_right] : ranges) { + prefix_right = prefix_dist(rng); + std::uniform_int_distribution suffix_dist(prefix_right + 1, 127); + suffix_left = suffix_dist(rng); + } + return ranges; +} + static std::vector make_512_targets(size_t num_targets = 4096) { std::mt19937 rng(44); std::uniform_int_distribution target_dist(-128, 128); @@ -113,6 +126,57 @@ BENCHMARK(BM_ExcessMin128) ->Args({63, 64}) ->Args({17, 17}); +static void BM_ExcessMin128BoundaryPairIndependent(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const auto ranges = make_disjoint_boundary_ranges(); + const size_t num_blocks = blocks.size(); + const size_t num_ranges = ranges.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& suffix = blocks[idx % num_blocks]; + const auto& prefix = blocks[(idx + 1) % num_blocks]; + const auto [suffix_left, prefix_right] = ranges[idx % num_ranges]; + ExcessResult suffix_result = + excess_min_128(suffix.data(), suffix_left, 127); + ExcessResult prefix_result = excess_min_128(prefix.data(), 0, prefix_right); + benchmark::DoNotOptimize(suffix_result.min_excess); + benchmark::DoNotOptimize(suffix_result.offset); + benchmark::DoNotOptimize(prefix_result.min_excess); + benchmark::DoNotOptimize(prefix_result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin128BoundaryPairIndependent); + +static void BM_ExcessMin128BoundaryPairFused(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const auto ranges = make_disjoint_boundary_ranges(); + const size_t num_blocks = blocks.size(); + const size_t num_ranges = ranges.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& suffix = blocks[idx % num_blocks]; + const auto& prefix = blocks[(idx + 1) % num_blocks]; + const auto [suffix_left, prefix_right] = ranges[idx % num_ranges]; + ExcessBoundaryPairResult result = excess_min_128_disjoint_suffix_prefix( + suffix.data(), suffix_left, prefix.data(), prefix_right); + benchmark::DoNotOptimize(result.suffix.min_excess); + benchmark::DoNotOptimize(result.suffix.offset); + benchmark::DoNotOptimize(result.prefix.min_excess); + benchmark::DoNotOptimize(result.prefix.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin128BoundaryPairFused); + template static void BM_ExcessMin128Variant(benchmark::State& state) { const size_t left = static_cast(state.range(0)); diff --git a/src/tests/excess_positions_tests.cpp b/src/tests/excess_positions_tests.cpp index 62cdf3a..c5c43f7 100644 --- a/src/tests/excess_positions_tests.cpp +++ b/src/tests/excess_positions_tests.cpp @@ -162,6 +162,28 @@ static size_t count_matches(const uint64_t* out) { return cnt; } +static void check_boundary_pair_matches_independent( + const std::array& suffix, + size_t suffix_left, + const std::array& prefix, + size_t prefix_right) { + const ExcessBoundaryPairResult result = excess_min_128_disjoint_suffix_prefix( + suffix.data(), suffix_left, prefix.data(), prefix_right); + const ExcessResult expected_suffix = + excess_min_128(suffix.data(), suffix_left, 127); + const ExcessResult expected_prefix = + excess_min_128(prefix.data(), 0, prefix_right); + + ASSERT_EQ(result.suffix.min_excess, expected_suffix.min_excess) + << "suffix_left=" << suffix_left << " prefix_right=" << prefix_right; + ASSERT_EQ(result.suffix.offset, expected_suffix.offset) + << "suffix_left=" << suffix_left << " prefix_right=" << prefix_right; + ASSERT_EQ(result.prefix.min_excess, expected_prefix.min_excess) + << "suffix_left=" << suffix_left << " prefix_right=" << prefix_right; + ASSERT_EQ(result.prefix.offset, expected_prefix.offset) + << "suffix_left=" << suffix_left << " prefix_right=" << prefix_right; +} + template static void check_matches_naive(Fn fn, const char* fn_name, @@ -345,6 +367,65 @@ TEST(ExcessPositions128, MinMatchesNaiveRandom) { } } +TEST(ExcessPositions128, DisjointBoundaryPairMatchesIndependentFixedCases) { + const std::array, 5> cases = {{ + {0, 0}, + {UINT64_MAX, UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull}, + {0x0000FFFF0000FFFFull, 0xFFFF0000FFFF0000ull}, + }}; + const std::array, 10> ranges = {{ + {1, 0}, + {4, 3}, + {17, 16}, + {32, 31}, + {63, 62}, + {64, 32}, + {65, 63}, + {96, 31}, + {127, 0}, + {120, 119}, + }}; + + for (const auto& suffix : cases) { + for (const auto& prefix : cases) { + for (const auto [suffix_left, prefix_right] : ranges) { + check_boundary_pair_matches_independent(suffix, suffix_left, prefix, + prefix_right); + } + } + } +} + +TEST(ExcessPositions128, DisjointBoundaryPairMatchesIndependentRandom) { + std::mt19937_64 rng(45); + std::uniform_int_distribution prefix_dist(0, 126); + + for (int t = 0; t < 1000; ++t) { + const std::array suffix = {rng(), rng()}; + const std::array prefix = {rng(), rng()}; + for (int q = 0; q < 16; ++q) { + const size_t prefix_right = prefix_dist(rng); + std::uniform_int_distribution suffix_dist(prefix_right + 1, 127); + const size_t suffix_left = suffix_dist(rng); + check_boundary_pair_matches_independent(suffix, suffix_left, prefix, + prefix_right); + } + } +} + +TEST(ExcessPositions128, BoundaryPairFallbackMatchesIndependent) { + const std::array suffix = {0x0123456789ABCDEFull, + 0xFEDCBA9876543210ull}; + const std::array prefix = {0x0000FFFF0000FFFFull, + 0xFFFF0000FFFF0000ull}; + + check_boundary_pair_matches_independent(suffix, 32, prefix, 32); + check_boundary_pair_matches_independent(suffix, 0, prefix, 127); + check_boundary_pair_matches_independent(suffix, 128, prefix, 0); +} + TEST(ExcessPositions128Experimental, MinVariantsMatchNaive) { const std::array, 6> cases = {{ {0, 0}, diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index e727353..5f13d8b 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -166,6 +166,27 @@ TEST(RmqBpPlusMinusOne, CrossBlockTieKeepsFirstPosition) { std::less())); } +TEST(RmqBpPlusMinusOne, DisjointBoundaryRangeMatchesNaive) { + std::vector depths(384); + for (std::size_t i = 1; i < depths.size(); ++i) { + const bool up = (i % 9 == 0) || (i % 9 == 3) || (i % 11 == 0); + depths[i] = depths[i - 1] + (up ? 1 : -1); + } + + const std::vector bits = pack_depth_deltas(depths); + const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); + const std::vector> ranges = { + {96, 160}, {120, 140}, {127, 129}, {190, 258}, {250, 260}, + }; + + for (const auto [left, right] : ranges) { + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(depths), left, right, + std::less())) + << "range=[" << left << "," << right << "]"; + } +} + TEST(RmqBpPlusMinusOne, RejectsTooSmallBitSpan) { const std::vector bits; EXPECT_THROW((pixie::rmq::BpPlusMinusOneRmq<>(bits, 2)), From 353af4392430da514cde137243c559b9d1d79956 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Mon, 8 Jun 2026 10:50:57 +0300 Subject: [PATCH 08/27] Huge work on rmq btree --- AGENTS.md | 5 + CMakeLists.txt | 9 + include/pixie/bits.h | 687 ++++++++-- include/pixie/cache_line.h | 88 +- include/pixie/rmq.h | 2 + include/pixie/rmq/bp_plus_minus_one_rmq.h | 160 ++- include/pixie/rmq/cartesian_tree_rmq.h | 259 ++-- .../rmq/experimental/node_euler_btree_rmq.h | 1217 +++++++++++++++++ include/pixie/rmq/node_euler_btree_rmq.h | 1016 ++++++++++++++ include/pixie/rmq/rmq_base.h | 21 +- include/pixie/rmq/rmq_one_interval_btree.h | 833 +++++++++++ include/pixie/rmq/segment_tree.h | 43 +- include/pixie/rmq/sparse_table.h | 112 +- src/benchmarks/bench_rmq.cpp | 811 ++++++++++- src/benchmarks/benchmarks.cpp | 26 +- .../excess_positions_benchmarks.cpp | 298 ++++ src/tests/excess_positions_tests.cpp | 182 +++ src/tests/excess_record_lows_tests.cpp | 293 ++++ src/tests/rmq_tests.cpp | 1139 ++++++++++++--- 19 files changed, 6601 insertions(+), 600 deletions(-) create mode 100644 include/pixie/rmq/experimental/node_euler_btree_rmq.h create mode 100644 include/pixie/rmq/node_euler_btree_rmq.h create mode 100644 include/pixie/rmq/rmq_one_interval_btree.h create mode 100644 src/tests/excess_record_lows_tests.cpp diff --git a/AGENTS.md b/AGENTS.md index 4078e17..997d599 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,11 @@ When a task matches a skill, read: 3. **SIMD conditional compilation**: Uses `#ifdef PIXIE_AVX512_SUPPORT` / `PIXIE_AVX2_SUPPORT` with scalar fallbacks. 4. **Target domain**: Optimized for data sizes up to 2^64 bits. 5. **Platform**: Linux/Unix is the primary target platform. +6. **Interval convention**: Public APIs, tests, and benchmarks use C++-style + half-open ranges `[left, right)`: `left` is included and `right` is excluded. + Empty ranges (`left == right`) are invalid unless an API explicitly documents + otherwise. Low-level primitives may still use their own documented interval + conventions. ### Why Header-Only? diff --git a/CMakeLists.txt b/CMakeLists.txt index 4bc2e0d..f1f11fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -184,6 +184,15 @@ if(PIXIE_TESTS) gtest_main ${PIXIE_DIAGNOSTICS_LIBS}) + add_executable(excess_record_lows_tests + src/tests/excess_record_lows_tests.cpp) + target_include_directories(excess_record_lows_tests + PUBLIC include) + target_link_libraries(excess_record_lows_tests + gtest + gtest_main + ${PIXIE_DIAGNOSTICS_LIBS}) + add_executable(rmq_tests src/tests/rmq_tests.cpp) target_include_directories(rmq_tests diff --git a/include/pixie/bits.h b/include/pixie/bits.h index 57e5c16..f930842 100644 --- a/include/pixie/bits.h +++ b/include/pixie/bits.h @@ -47,6 +47,57 @@ static inline const __m256i mask_first_half = _mm256_setr_epi8( // clang-format on #endif +static inline constexpr int8_t excess_nibble_min_offset[16] = { + 4, 4, 4, 4, 2, 2, 1, 1, 3, 3, 1, 1, 2, 2, 1, 1}; + +#if defined(__SSSE3__) && defined(__SSE4_1__) +#define PIXIE_SSE41_SUPPORT +// clang-format off +static inline const __m128i excess_lut_delta_sse = _mm_setr_epi8( + -4, -2, -2, 0, + -2, 0, 0, 2, + -2, 0, 0, 2, + 0, 2, 2, 4); +static inline const __m128i excess_lut_pos0_sse = _mm_setr_epi8( + -1, 1, -1, 1, + -1, 1, -1, 1, + -1, 1, -1, 1, + -1, 1, -1, 1); +static inline const __m128i excess_lut_pos1_sse = _mm_setr_epi8( + -2, 0, 0, 2, + -2, 0, 0, 2, + -2, 0, 0, 2, + -2, 0, 0, 2); +static inline const __m128i excess_lut_pos2_sse = _mm_setr_epi8( + -3, -1, -1, 1, + -1, 1, 1, 3, + -3, -1, -1, 1, + -1, 1, 1, 3); +static inline const __m128i excess_lut_min_sse = _mm_setr_epi8( + -4, -2, -2, 0, + -2, 0, -1, 1, + -3, -1, -1, 1, + -2, 0, -1, 1); +static inline const __m128i excess_lut_nibble_index_sse = _mm_setr_epi8( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15); +static inline const __m128i excess_lut_nibble_mask_sse = _mm_set1_epi8(0x0F); +// clang-format on +#endif + +#ifdef PIXIE_SSE41_SUPPORT +static inline __m128i excess_nibbles_64_sse(const uint64_t* s) noexcept { + const __m128i word_vec = _mm_loadl_epi64(reinterpret_cast(s)); + const __m128i lo_nibbles = + _mm_and_si128(word_vec, excess_lut_nibble_mask_sse); + const __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(word_vec, 4), excess_lut_nibble_mask_sse); + return _mm_unpacklo_epi8(lo_nibbles, hi_nibbles); +} +#endif + /** * @brief Test 16 int16 RmM btree child ranges for a node-local target. * @details Each lane represents one child summary. The function checks whether @@ -939,6 +990,45 @@ static inline constexpr std::array excess_byte_min_offset_lut = excess_make_byte_lut( [](uint8_t x) { return excess_byte_min_prefix_offset_value(x); }); +/** + * @brief Compute record-low mask for a single byte relative to a threshold. + * + * @details For each bit position in the byte (0..7), computes the local + * excess starting from 0. Sets the corresponding bit in the result mask + * iff the local excess is strictly less than @p threshold. + */ +constexpr uint8_t excess_byte_record_lows_mask(uint8_t byte, int threshold) { + int cur = 0; + uint8_t mask = 0; + for (int bit = 0; bit < 8; ++bit) { + cur += ((byte >> bit) & 1u) ? 1 : -1; + if (cur < threshold) { + mask |= static_cast(1u << bit); + } + } + return mask; +} + +/** + * @brief LUT for record-low masks within a single byte. + * + * @details For each byte value (256) and each possible gap g in [0, 7], + * stores a bitmask of positions whose local excess is strictly less than + * -g. Gap g = start_excess - best_excess; when g >= 8 no new record low + * is possible because the byte's minimum local excess is -8. + */ +static inline constexpr std::array, 256> + excess_byte_record_lows_lut = [] { + std::array, 256> out{}; + for (size_t byte = 0; byte < 256; ++byte) { + for (int g = 0; g < 8; ++g) { + out[byte][g] = + excess_byte_record_lows_mask(static_cast(byte), -g); + } + } + return out; + }(); + /** * @brief Find every prefix whose excess equals target_x in a 128-bit bitstring. * @@ -1058,6 +1148,26 @@ static inline int prefix_excess_128(const uint64_t* s, return 2 * ones - static_cast(end_offset); } +/** + * @brief Prefix excess in a 64-bit bitstring. + * + * @details `excess(i) = 2 * popcount(bits[0..i)) - i` for `i` in `[0, 64]`. + * + * @param s One little-endian 64-bit word (bit 0 of `s[0]` is first). + * @param end_offset Exclusive prefix boundary, clamped to `[0, 64]`. + * @return Prefix excess on `[0, end_offset)`. + */ +static inline int prefix_excess_64(const uint64_t* s, + size_t end_offset) noexcept { + end_offset = end_offset > 64 ? 64 : end_offset; + if (end_offset == 0) { + return 0; + } + const int ones = static_cast( + std::popcount(s[0] & first_bits_mask(static_cast(end_offset)))); + return 2 * ones - static_cast(end_offset); +} + static inline ExcessResult excess_min_128_byte_lut_short( const uint64_t* s, size_t left, @@ -1102,6 +1212,151 @@ static inline ExcessResult excess_min_128_byte_lut_short( return {best, best_offset}; } +/** + * @brief Return the minimum prefix excess and first attaining offset. + * + * @details Prefix positions are offsets in `[0, 64]`; position 0 is the empty + * prefix and position `k` is the excess after consuming the first `k` bits. + * The query range `[left, right]` is inclusive. Ties return the first offset + * attaining the minimum. Invalid ranges return the default `ExcessResult` + * sentinel. + * + * @param s One little-endian 64-bit word (bit 0 of `s[0]` is first). + * @param left First prefix position to consider, inclusive. + * @param right Last prefix position to consider, inclusive. + * @return Minimum excess and first local offset attaining it. + */ +static inline ExcessResult excess_min_64(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 64); + right = std::min(right, 64); + + int best = prefix_excess_64(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + +#ifdef PIXIE_SSE41_SUPPORT + int current = best; + size_t bit = left; + for (; bit < right && (bit & 3u) != 0; ++bit) { + current += ((s[0] >> bit) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } + + const size_t first_full_nibble = bit >> 2; + const size_t last_full_nibble = right >> 2; + const size_t right_partial_width = bit < right ? (right & 3u) : 0; + const size_t end_nibble = + last_full_nibble + (right_partial_width == 0 ? 0 : 1); + if (first_full_nibble < end_nibble) { + const __m128i nibbles = excess_nibbles_64_sse(s); + + __m128i ps = _mm_shuffle_epi8(excess_lut_delta_sse, nibbles); + ps = _mm_add_epi8(ps, _mm_slli_si128(ps, 1)); + ps = _mm_add_epi8(ps, _mm_slli_si128(ps, 2)); + ps = _mm_add_epi8(ps, _mm_slli_si128(ps, 4)); + ps = _mm_add_epi8(ps, _mm_slli_si128(ps, 8)); + + const __m128i excl_ps = _mm_slli_si128(ps, 1); + __m128i local_min = _mm_shuffle_epi8(excess_lut_min_sse, nibbles); + if (right_partial_width != 0) { + __m128i partial_min = _mm_shuffle_epi8(excess_lut_pos0_sse, nibbles); + if (right_partial_width >= 2) { + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos1_sse, nibbles)); + } + if (right_partial_width >= 3) { + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos2_sse, nibbles)); + } + local_min = _mm_blendv_epi8( + local_min, partial_min, + _mm_cmpeq_epi8(excess_lut_nibble_index_sse, + _mm_set1_epi8(static_cast(last_full_nibble)))); + } + const __m128i partial_candidates = _mm_add_epi8(excl_ps, local_min); + + const __m128i idx = excess_lut_nibble_index_sse; + const int first_minus_one_value = static_cast(first_full_nibble) - 1; + const __m128i first_minus_one = + _mm_set1_epi8(static_cast(first_minus_one_value)); + const __m128i last = _mm_set1_epi8(static_cast(end_nibble)); + const __m128i active = _mm_and_si128(_mm_cmpgt_epi8(idx, first_minus_one), + _mm_cmpgt_epi8(last, idx)); + const __m128i masked_candidates = + _mm_blendv_epi8(_mm_set1_epi8(127), partial_candidates, active); + + __m128i min128 = masked_candidates; + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 8)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 4)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 2)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 1)); + + const int candidate_min = + static_cast(static_cast(_mm_extract_epi8(min128, 0))); + if (candidate_min < best) { + const __m128i equal_min = _mm_cmpeq_epi8( + masked_candidates, _mm_set1_epi8(static_cast(candidate_min))); + const uint32_t equal_mask = + static_cast(_mm_movemask_epi8(equal_min)); + const uint32_t nibble_index = std::countr_zero(equal_mask); + const uint8_t nibble = + static_cast((s[0] >> (nibble_index * 4u)) & 0xFu); + best = candidate_min; + if (right_partial_width != 0 && nibble_index == last_full_nibble) { + int local = 0; + int local_best = 0; + size_t local_offset = 1; + for (size_t i = 0; i < right_partial_width; ++i) { + local += ((nibble >> i) & 1u) != 0 ? 1 : -1; + if (i == 0 || local < local_best) { + local_best = local; + local_offset = i + 1; + } + } + best_offset = static_cast(nibble_index) * 4u + local_offset; + } else { + best_offset = static_cast(nibble_index) * 4u + + static_cast(excess_nibble_min_offset[nibble]); + } + } + + bit = end_nibble * 4; + } + + for (; bit < right; ++bit) { + current += ((s[0] >> bit) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } +#else + int current = best; + for (size_t bit = left; bit < right; ++bit) { + current += ((s[0] >> bit) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } +#endif + + return {best, best_offset}; +} + /** * @brief Return the minimum prefix excess and first attaining offset. * @param s 2 little-endian uint64_t words (bit 0 of s[0] is the first bit). @@ -1257,14 +1512,41 @@ static inline ExcessResult excess_min_128(const uint64_t* s, return {best, best_offset}; } +/** + * @brief Compute disjoint suffix/prefix boundary minima for two 64-bit blocks. + * + * @details Computes `excess_min_64(suffix_s, suffix_left, 63)` and + * `excess_min_64(prefix_s, 0, prefix_right)`. The individual calls use the + * SSE nibble-LUT path when available. + * + * @param suffix_s Left boundary block. + * @param suffix_left First local prefix offset in the suffix range. + * @param prefix_s Right boundary block. + * @param prefix_right Last local prefix offset in the prefix range. + * @return Pair of local minimum results. + */ +static inline ExcessBoundaryPairResult excess_min_64_disjoint_suffix_prefix( + const uint64_t* suffix_s, + size_t suffix_left, + const uint64_t* prefix_s, + size_t prefix_right) noexcept { + suffix_left = std::min(suffix_left, 64); + prefix_right = std::min(prefix_right, 64); + return {excess_min_64(suffix_s, suffix_left, 63), + excess_min_64(prefix_s, 0, prefix_right)}; +} + /** * @brief Compute disjoint suffix/prefix boundary minima for two 128-bit blocks. * * @details Computes `excess_min_128(suffix_s, suffix_left, 127)` and * `excess_min_128(prefix_s, 0, prefix_right)`. When `suffix_left > - * prefix_right`, the AVX2 path packs the two disjoint local ranges into one - * nibble stream and shares the prefix-sum/reduction work. Non-disjoint or - * out-of-shape inputs fall back to the two independent production calls. + * prefix_right`, the AVX2 path blends active whole nibbles from both blocks + * into one synthetic lane vector, fills the gap with neutral zero-delta + * nibbles, and shares the prefix-sum/reduction work. Boundary fragments that + * do not cover a whole nibble are handled by scalar or partial-nibble logic. + * Non-disjoint or out-of-shape inputs fall back to the two independent + * production calls. * * @param suffix_s Left boundary block. * @param suffix_left First local prefix offset in the suffix range. @@ -1300,61 +1582,26 @@ static inline ExcessBoundaryPairResult excess_min_128_disjoint_suffix_prefix( } } - alignas(32) int8_t lane_nibbles[32] = {}; - alignas(32) int8_t lane_indices[32] = {}; - alignas(32) int8_t prefix_active[32] = {}; - alignas(32) int8_t suffix_active[32] = {}; - alignas(32) int8_t partial_widths[32] = {}; - alignas(32) int8_t prefix_partial[32] = {}; - alignas(32) int8_t suffix_partial[32] = {}; - - size_t lane = 0; - int prefix_artificial_delta = 0; const size_t prefix_last_nibble = prefix_right >> 2; const size_t prefix_partial_width = prefix_right & 3u; const size_t prefix_end_nibble = prefix_last_nibble + (prefix_partial_width == 0 ? 0 : 1); - for (size_t nibble_index = 0; nibble_index < prefix_end_nibble; - ++nibble_index) { - const uint64_t word = prefix_s[nibble_index >> 4]; - const uint8_t nibble = - static_cast((word >> ((nibble_index & 15u) * 4u)) & 0xFu); - lane_nibbles[lane] = static_cast(nibble); - lane_indices[lane] = static_cast(nibble_index); - prefix_active[lane] = static_cast(0xFFu); - if (prefix_partial_width != 0 && nibble_index == prefix_last_nibble) { - partial_widths[lane] = static_cast(prefix_partial_width); - prefix_partial[lane] = static_cast(0xFFu); - } - prefix_artificial_delta += 2 * std::popcount(nibble) - 4; - ++lane; - } - - if (suffix_bit < 127) { - const size_t suffix_first_nibble = suffix_bit >> 2; - for (size_t nibble_index = suffix_first_nibble; nibble_index < 32; - ++nibble_index) { - const uint64_t word = suffix_s[nibble_index >> 4]; - const uint8_t nibble = - static_cast((word >> ((nibble_index & 15u) * 4u)) & 0xFu); - lane_nibbles[lane] = static_cast(nibble); - lane_indices[lane] = static_cast(nibble_index); - suffix_active[lane] = static_cast(0xFFu); - if (nibble_index == 31) { - partial_widths[lane] = 3; - suffix_partial[lane] = static_cast(0xFFu); - } - ++lane; - } - } - - if (lane > 32) { - return {excess_min_128(suffix_s, suffix_left, 127), - excess_min_128(prefix_s, 0, prefix_right)}; - } - + const size_t suffix_first_nibble = suffix_bit < 127 ? suffix_bit >> 2 : 32; + const int prefix_artificial_delta = + prefix_excess_128(prefix_s, prefix_end_nibble * 4); + + const __m256i idx = excess_lut_nibble_index; + const __m256i prefix_active = _mm256_cmpgt_epi8( + _mm256_set1_epi8(static_cast(prefix_end_nibble)), idx); + const __m256i suffix_active = _mm256_cmpgt_epi8( + idx, _mm256_set1_epi8(static_cast(suffix_first_nibble) - 1)); + + const __m256i prefix_nibbles = excess_nibbles_128_avx2(prefix_s); + const __m256i suffix_nibbles = excess_nibbles_128_avx2(suffix_s); __m256i nibbles = - _mm256_load_si256(reinterpret_cast(lane_nibbles)); + _mm256_blendv_epi8(_mm256_set1_epi8(3), prefix_nibbles, prefix_active); + nibbles = _mm256_blendv_epi8(nibbles, suffix_nibbles, suffix_active); + __m256i ps = _mm256_shuffle_epi8(excess_lut_delta, nibbles); ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 1)); ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 2)); @@ -1372,39 +1619,41 @@ static inline ExcessBoundaryPairResult excess_min_128_disjoint_suffix_prefix( const __m256i excl_ps = _mm256_alignr_epi8(ps, b, 15); __m256i local_min = _mm256_shuffle_epi8(excess_lut_min, nibbles); - const __m256i width = - _mm256_load_si256(reinterpret_cast(partial_widths)); - __m256i partial_min = _mm256_shuffle_epi8(excess_lut_pos0, nibbles); - partial_min = _mm256_min_epi8( - partial_min, - _mm256_blendv_epi8(_mm256_set1_epi8(127), - _mm256_shuffle_epi8(excess_lut_pos1, nibbles), - _mm256_cmpgt_epi8(width, _mm256_set1_epi8(1)))); - partial_min = _mm256_min_epi8( - partial_min, - _mm256_blendv_epi8(_mm256_set1_epi8(127), - _mm256_shuffle_epi8(excess_lut_pos2, nibbles), - _mm256_cmpgt_epi8(width, _mm256_set1_epi8(2)))); - const __m256i any_partial = _mm256_or_si256( - _mm256_load_si256(reinterpret_cast(prefix_partial)), - _mm256_load_si256(reinterpret_cast(suffix_partial))); - local_min = _mm256_blendv_epi8(local_min, partial_min, any_partial); + if (prefix_partial_width != 0) { + __m256i partial_min = _mm256_shuffle_epi8(excess_lut_pos0, nibbles); + if (prefix_partial_width >= 2) { + partial_min = _mm256_min_epi8( + partial_min, _mm256_shuffle_epi8(excess_lut_pos1, nibbles)); + } + if (prefix_partial_width >= 3) { + partial_min = _mm256_min_epi8( + partial_min, _mm256_shuffle_epi8(excess_lut_pos2, nibbles)); + } + local_min = _mm256_blendv_epi8( + local_min, partial_min, + _mm256_cmpeq_epi8( + idx, _mm256_set1_epi8(static_cast(prefix_last_nibble)))); + } + __m256i suffix_partial_min = _mm256_min_epi8( + _mm256_shuffle_epi8(excess_lut_pos0, nibbles), + _mm256_min_epi8(_mm256_shuffle_epi8(excess_lut_pos1, nibbles), + _mm256_shuffle_epi8(excess_lut_pos2, nibbles))); + const __m256i suffix_tail_active = _mm256_and_si256( + suffix_active, _mm256_cmpeq_epi8(idx, _mm256_set1_epi8(31))); + local_min = + _mm256_blendv_epi8(local_min, suffix_partial_min, suffix_tail_active); const __m256i base_candidates = _mm256_add_epi8(excl_ps, local_min); - const __m256i prefix_mask = - _mm256_load_si256(reinterpret_cast(prefix_active)); - const __m256i suffix_mask = - _mm256_load_si256(reinterpret_cast(suffix_active)); const __m256i sentinel = _mm256_set1_epi8(127); const __m256i prefix_candidates = - _mm256_blendv_epi8(sentinel, base_candidates, prefix_mask); + _mm256_blendv_epi8(sentinel, base_candidates, prefix_active); const __m256i suffix_candidates = _mm256_blendv_epi8( sentinel, _mm256_add_epi8(base_candidates, _mm256_set1_epi8(static_cast( suffix_current - prefix_artificial_delta))), - suffix_mask); + suffix_active); auto reduce_min = [](__m256i values) { __m128i min128 = _mm_min_epi8(_mm256_castsi256_si128(values), @@ -1439,11 +1688,16 @@ static inline ExcessBoundaryPairResult excess_min_128_disjoint_suffix_prefix( _mm256_cmpeq_epi8(prefix_candidates, _mm256_set1_epi8(static_cast(prefix_min))))); const uint32_t prefix_lane = std::countr_zero(mask); - const uint8_t nibble = static_cast(lane_nibbles[prefix_lane]); + const uint64_t word = prefix_s[prefix_lane >> 4]; + const uint8_t nibble = + static_cast((word >> ((prefix_lane & 15u) * 4u)) & 0xFu); prefix.min_excess = prefix_min; + const size_t width = + prefix_partial_width != 0 && prefix_lane == prefix_last_nibble + ? prefix_partial_width + : 4; prefix.offset = - static_cast(lane_indices[prefix_lane]) * 4u + - local_offset(nibble, static_cast(partial_widths[prefix_lane])); + static_cast(prefix_lane) * 4u + local_offset(nibble, width); } const int suffix_min = reduce_min(suffix_candidates); @@ -1452,11 +1706,13 @@ static inline ExcessBoundaryPairResult excess_min_128_disjoint_suffix_prefix( _mm256_cmpeq_epi8(suffix_candidates, _mm256_set1_epi8(static_cast(suffix_min))))); const uint32_t suffix_lane = std::countr_zero(mask); - const uint8_t nibble = static_cast(lane_nibbles[suffix_lane]); + const uint64_t word = suffix_s[suffix_lane >> 4]; + const uint8_t nibble = + static_cast((word >> ((suffix_lane & 15u) * 4u)) & 0xFu); suffix.min_excess = suffix_min; + const size_t width = suffix_lane == 31 ? 3 : 4; suffix.offset = - static_cast(lane_indices[suffix_lane]) * 4u + - local_offset(nibble, static_cast(partial_widths[suffix_lane])); + static_cast(suffix_lane) * 4u + local_offset(nibble, width); } return {suffix, prefix}; @@ -1584,6 +1840,273 @@ static inline void excess_positions_512(const uint64_t* s, } } +/** + * @brief Find every strict record-low prefix excess position in a 128-bit + * bitstring. + * + * @details A position i (1 <= i <= 128) is a strict record low if its prefix + * excess is strictly smaller than every prefix excess at positions j < i. + * Position 0 (empty prefix, excess 0) is always a record low vacuously but is + * not represented in the output bitmask. Bit b of out[w] is set iff position + * (w*64 + b + 1) is a strict record low. + * + * @param s 2 little-endian uint64_t words (bit 0 of s[0] is the first bit). + * @param out 2 uint64_t words receiving the result bitmask. + */ +static inline void excess_record_lows_128(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + for (size_t i = 0; i < 128; ++i) { + const uint64_t w = s[i >> 6]; + const int bit = static_cast((w >> (i & 63)) & 1ull); + cur += bit ? +1 : -1; + if (cur < best) { + best = cur; + out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } +} + +/** + * @brief Byte-LUT accelerated record-low scan for 128-bit blocks. + * + * @details Uses precomputed per-byte minimum and delta LUTs to skip whole + * bytes that cannot contain a record low. When a byte's local minimum is + * below the running global best, the byte is scanned bit-by-bit; otherwise + * the byte is skipped and only the running excess is updated. + */ +static inline void excess_record_lows_128_byte_lut(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + for (size_t byte_idx = 0; byte_idx < 16; ++byte_idx) { + const size_t bit_base = byte_idx * 8; + const uint8_t byte = static_cast((s[bit_base >> 6] >> (bit_base & 63)) & 0xFFu); + const int byte_min = excess_byte_min_lut[byte]; + if (cur + byte_min < best) { + for (size_t i = 0; i < 8; ++i) { + const int bit = static_cast((byte >> i) & 1u); + cur += bit ? +1 : -1; + if (cur < best) { + best = cur; + const size_t pos = bit_base + i; + out[pos >> 6] |= (uint64_t{1} << (pos & 63)); + } + } + } else { + cur += excess_byte_delta_lut[byte]; + } + } +} + +/** + * @brief LUT-based record-low scan for 128-bit blocks (branchless per byte). + * + * @details Precomputed per-byte LUT: for each byte value and each possible + * threshold (cur - best, clamped to 0..7), returns a bitmask of positions + * within the byte whose excess is strictly below the threshold. The LUT + * eliminates all bit-by-bit scanning and branching inside the hot loop. + */ +static inline void excess_record_lows_128_lut(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + for (size_t byte_idx = 0; byte_idx < 16; ++byte_idx) { + const size_t bit_base = byte_idx * 8; + const uint8_t byte = + static_cast((s[bit_base >> 6] >> (bit_base & 63)) & 0xFFu); + const int gap = cur - best; + const int idx = gap > 7 ? 7 : (gap < 0 ? 0 : gap); + const uint8_t mask = excess_byte_record_lows_lut[byte][static_cast(idx)]; + if (mask != 0) { + // Recompute absolute excesses for masked positions to update cur/best. + int local = 0; + int local_best = 0; + uint8_t local_mask = 0; + for (int bit = 0; bit < 8; ++bit) { + local += ((byte >> bit) & 1u) ? 1 : -1; + if (local < local_best) { + local_best = local; + local_mask |= static_cast(1u << bit); + } + } + // Only output positions whose absolute excess is < best. + uint8_t out_mask = 0; + local = 0; + for (int bit = 0; bit < 8; ++bit) { + local += ((byte >> bit) & 1u) ? 1 : -1; + if (cur + local < best) { + out_mask |= static_cast(1u << bit); + best = cur + local; + } + } + if (out_mask != 0) { + const uint64_t word = static_cast(out_mask) << (bit_base & 63); + out[bit_base >> 6] |= word; + } + } + cur += excess_byte_delta_lut[byte]; + } +} + +#ifdef PIXIE_AVX2_SUPPORT +/** + * @brief AVX2 record-low scan for 128-bit blocks. + * + * @details Expands the 128-bit block into eight 16-bit chunks, computes + * vector prefix sums, then extracts record lows with a running scalar + * minimum. This avoids the unpredictable branch in the scalar loop. + */ +static inline void excess_record_lows_128_avx2(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + + const __m256i masks = excess_bit_masks_16x_i16(); + const __m256i zero = _mm256_setzero_si256(); + const __m256i pos = _mm256_set1_epi16(1); + const __m256i neg = _mm256_set1_epi16(-1); + + for (size_t chunk = 0; chunk < 8; ++chunk) { + const size_t chunk_bit = chunk * 16; + const uint16_t bits = + chunk < 4 + ? static_cast((s[0] >> (chunk * 16)) & 0xFFFFu) + : static_cast((s[1] >> ((chunk - 4) * 16)) & 0xFFFFu); + + const __m256i vb = _mm256_set1_epi16(static_cast(bits)); + const __m256i m = _mm256_and_si256(vb, masks); + const __m256i is_zero = _mm256_cmpeq_epi16(m, zero); + const __m256i steps = _mm256_blendv_epi8(pos, neg, is_zero); + const __m256i pref_rel = excess_prefix_sum_16x_i16(steps); + const __m256i pref_abs = + _mm256_add_epi16(pref_rel, _mm256_set1_epi16(static_cast(cur))); + + alignas(32) int16_t vals[16]; + _mm256_store_si256(reinterpret_cast<__m256i*>(vals), pref_abs); + + for (size_t lane = 0; lane < 16; ++lane) { + const int val = vals[lane]; + if (val < best) { + best = val; + const size_t pos_idx = chunk_bit + lane; + out[pos_idx >> 6] |= (uint64_t{1} << (pos_idx & 63)); + } + } + + cur += 2 * static_cast(std::popcount(bits)) - 16; + } +} + +/** + * @brief AVX2 nibble-LUT record-low scan for 128-bit blocks. + * + * @details Loads 128 bits as 32 nibbles in one AVX2 register. + * Uses precomputed LUTs to get per-nibble excess values, prefix-sums + * them with 32-lane i8 operations, then compares against the running + * scalar minimum. Record-low positions are written into the output + * bitmask bit-by-bit. + */ +static inline void excess_record_lows_128_nibble_lut(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + + const __m256i vdelta = excess_lut_delta; + const __m256i vmin = excess_lut_min; + + __m256i nibbles = excess_nibbles_128_avx2(s); + + // Compute inclusive prefix sums of per-nibble total excess changes. + __m256i ps = _mm256_shuffle_epi8(vdelta, nibbles); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 1)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 2)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 4)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 8)); + + __m128i ps_lo = _mm256_castsi256_si128(ps); + __m128i ps_hi = _mm256_extracti128_si256(ps, 1); + __m128i carry = + _mm_set1_epi8(static_cast(_mm_extract_epi8(ps_lo, 15))); + ps_hi = _mm_add_epi8(ps_hi, carry); + ps = _mm256_inserti128_si256(_mm256_castsi128_si256(ps_lo), ps_hi, 1); + + // Exclusive prefix: shift in zero at start. + __m256i b = _mm256_permute2x128_si256(ps, ps, 0x08); + __m256i excl_ps = _mm256_alignr_epi8(ps, b, 15); + + // Local minima relative to each nibble start. + __m256i local_min = _mm256_shuffle_epi8(vmin, nibbles); + + alignas(32) int8_t excl[32]; + _mm256_store_si256(reinterpret_cast<__m256i*>(excl), excl_ps); + + alignas(32) int8_t nibble_min[32]; + _mm256_store_si256(reinterpret_cast<__m256i*>(nibble_min), local_min); + + alignas(32) int8_t nibble_vals[32]; + _mm256_store_si256(reinterpret_cast<__m256i*>(nibble_vals), nibbles); + + for (int n = 0; n < 32; ++n) { + const int nibble_base = cur + excl[n]; + const int nibble_best = nibble_base + nibble_min[n]; + if (nibble_best < best) { + // This nibble contains at least one record low — scan bit-by-bit. + const uint8_t nibble = static_cast(nibble_vals[n]); + int local = 0; + for (int bit = 0; bit < 4; ++bit) { + local += ((nibble >> bit) & 1u) ? 1 : -1; + const int val = nibble_base + local; + if (val < best) { + best = val; + const size_t pos = static_cast(n) * 4 + bit; + out[pos >> 6] |= (uint64_t{1} << (pos & 63)); + } + } + } + } +} +#endif + +/** + * @brief Find every strict record-low prefix excess position in a 512-bit + * bitstring. + * + * @details A position i (1 <= i <= 512) is a strict record low if its prefix + * excess is strictly smaller than every prefix excess at positions j < i. + * Position 0 is not represented in the output bitmask. Bit b of out[w] is set + * iff position (w*64 + b + 1) is a strict record low. + * + * @param s 8 little-endian uint64_t words (bit 0 of s[0] is the first bit). + * @param out 8 uint64_t words receiving the result bitmask. + */ +static inline void excess_record_lows_512(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = out[2] = out[3] = 0; + out[4] = out[5] = out[6] = out[7] = 0; + int best = 0; + int global = 0; + for (int k = 0; k < 4; ++k) { + const uint64_t* block = s + 2 * k; + uint64_t* block_out = out + 2 * k; + for (size_t i = 0; i < 128; ++i) { + const uint64_t w = block[i >> 6]; + const int bit = static_cast((w >> (i & 63)) & 1ull); + global += bit ? +1 : -1; + if (global < best) { + best = global; + block_out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } + } +} + /** * @brief Calculates 32 bit ranks of every 8th bit, result is stored as 32 diff --git a/include/pixie/cache_line.h b/include/pixie/cache_line.h index 94b3ce2..82f5f23 100644 --- a/include/pixie/cache_line.h +++ b/include/pixie/cache_line.h @@ -6,36 +6,65 @@ #include #include +namespace pixie { + +inline constexpr std::size_t kAlignedStorageLineBytes = 64; +inline constexpr std::size_t kAlignedStorageLineBits = + kAlignedStorageLineBytes * 8; +inline constexpr std::size_t kAlignedStorageLineWords64 = + kAlignedStorageLineBytes / sizeof(std::uint64_t); +inline constexpr std::size_t kAlignedStorageLineWords16 = + kAlignedStorageLineBytes / sizeof(std::uint16_t); + /** - * @brief A simple struct to represent a aligned storage for a cache line. + * @brief A 64-byte aligned storage block. */ -struct alignas(64) CacheLine { - std::array data; +struct alignas(kAlignedStorageLineBytes) CacheLine { + std::array data{}; }; +static_assert(alignof(CacheLine) == kAlignedStorageLineBytes); +static_assert(sizeof(CacheLine) == kAlignedStorageLineBytes); +static_assert(kAlignedStorageLineBytes % sizeof(std::uint64_t) == 0); +static_assert(kAlignedStorageLineBytes % sizeof(std::uint16_t) == 0); + /** - * @brief A simple aligned storage for cache-line sized blocks. + * @brief Aligned storage for 64-byte blocks. * - * @details Provides typed views over the same underlying storage as cache - * lines, 64-bit words, or bytes. All spans are contiguous and sized to the - * total storage capacity. + * @details The constructor and resize accept a logical size in bits. Storage is + * rounded up to a full 64-byte block, and all views expose the padded capacity. */ class AlignedStorage { private: std::vector data_; + static constexpr std::size_t LinesForBits(std::size_t bits) { + return bits / kAlignedStorageLineBits + + (bits % kAlignedStorageLineBits != 0); + } + public: AlignedStorage() = default; /** - * @brief Construct storage for at least @p bits bytes, rounded up to 512 - * bits. + * @brief Construct storage for at least @p bits bits. */ - AlignedStorage(size_t bits) : data_((bits + 511) / 512) {} + explicit AlignedStorage(std::size_t bits) : data_(LinesForBits(bits)) {} /** - * @brief Resize storage to hold at least @p bits bits, rounded up to 512 + * @brief Resize storage to hold at least @p bits bits. */ - void resize(size_t bits) { data_.resize((bits + 511) / 512); } + void resize(std::size_t bits) { data_.resize(LinesForBits(bits)); } + + /** @brief Padded storage capacity in bits. */ + std::size_t capacity_bits() const { + return data_.size() * kAlignedStorageLineBits; + } + + /** @brief Padded storage capacity in bytes. */ + std::size_t capacity_bytes() const { + return data_.size() * kAlignedStorageLineBytes; + } + /** @brief Mutable view as cache lines. */ std::span AsLines() { return data_; } /** @brief Const view as cache lines. */ @@ -44,45 +73,44 @@ class AlignedStorage { /** * @brief Mutable view as 64-bit words. */ - std::span As64BitInts() { - return std::span(reinterpret_cast(data_.data()), - data_.size() * 8); + std::span As64BitInts() { + return std::span( + reinterpret_cast(data_.data()), + data_.size() * kAlignedStorageLineWords64); } /** @brief Const view as 64-bit words. */ - std::span AsConst64BitInts() const { - return std::span( - reinterpret_cast(data_.data()), data_.size() * 8); + std::span AsConst64BitInts() const { + return std::span( + reinterpret_cast(data_.data()), + data_.size() * kAlignedStorageLineWords64); } /** * @brief Mutable view as bytes. - * @note Uses a byte pointer to the underlying storage. */ - std::span AsBytes() { - return std::span(reinterpret_cast(data_.data()), - data_.size() * 64); - } + std::span AsBytes() { return std::as_writable_bytes(AsLines()); } /** @brief Const view as bytes. */ std::span AsConstBytes() const { - return std::span( - reinterpret_cast(data_.data()), data_.size() * 64); + return std::as_bytes(AsConstLines()); } /** - * @brief Mutable view as bytes. - * @note Uses a byte pointer to the underlying storage. + * @brief Mutable view as 16-bit words. */ std::span As16BitInts() { return std::span( - reinterpret_cast(data_.data()), data_.size() * 32); + reinterpret_cast(data_.data()), + data_.size() * kAlignedStorageLineWords16); } - /** @brief Const view as bytes. */ + /** @brief Const view as 16-bit words. */ std::span AsConst16BitInts() const { return std::span( reinterpret_cast(data_.data()), - data_.size() * 32); + data_.size() * kAlignedStorageLineWords16); } }; + +} // namespace pixie diff --git a/include/pixie/rmq.h b/include/pixie/rmq.h index 452dfbe..2d79c70 100644 --- a/include/pixie/rmq.h +++ b/include/pixie/rmq.h @@ -2,6 +2,8 @@ #include #include +#include #include +#include #include #include diff --git a/include/pixie/rmq/bp_plus_minus_one_rmq.h b/include/pixie/rmq/bp_plus_minus_one_rmq.h index 1038ef7..e14e28f 100644 --- a/include/pixie/rmq/bp_plus_minus_one_rmq.h +++ b/include/pixie/rmq/bp_plus_minus_one_rmq.h @@ -23,9 +23,9 @@ namespace pixie::rmq { * @details The indexed depth sequence is represented by BP deltas: bit 1 means * the next depth is current + 1, and bit 0 means current - 1. A sequence with * @p depth_count depth positions has @p depth_count - 1 delta bits. Blocks - * match the 128-bit excess primitives in `bits.h`; each block stores a compact - * 16-byte summary with its base depth, absolute minimum value, and first local - * offset attaining that minimum. + * match the 64-bit or 128-bit excess primitives in `bits.h`; each block stores + * a compact 16-byte summary with its base depth, absolute minimum value, and + * first local offset attaining that minimum. * * @tparam Index Unsigned integer type used for stored positions. * @tparam BlockSize Number of depth positions per microblock. @@ -33,7 +33,7 @@ namespace pixie::rmq { template class BpPlusMinusOneRmq { public: - static_assert(BlockSize == 128); + static_assert(BlockSize == 64 || BlockSize == 128); static constexpr std::size_t npos = std::numeric_limits::max(); static constexpr Index invalid_index = std::numeric_limits::max(); @@ -135,36 +135,50 @@ class BpPlusMinusOneRmq { bool empty() const { return depth_count_ == 0; } /** - * @brief Return the first minimum depth position in [@p left, @p right]. + * @brief Return the first minimum depth position in [@p left, @p right). * - * @details The query range is inclusive over depth positions, not delta-bit + * @details The query range is half-open over depth positions, not delta-bit * positions. Ties return the smallest position attaining the minimum depth. + * Empty or invalid ranges return `npos`. * * @param left First depth position in the query range. - * @param right Last depth position in the query range. + * @param right One past the last depth position in the query range. * @return Zero-based position of the first range minimum, or `npos`. */ std::size_t arg_min(std::size_t left, std::size_t right) const { - if (left > right || right >= depth_count_) { + if (left >= right || right > depth_count_) { return npos; } + const std::size_t last = right - 1; const std::size_t left_block = left / BlockSize; - const std::size_t right_block = right / BlockSize; + const std::size_t right_block = last / BlockSize; if (left_block == right_block) { return scan_block_range_position(left_block, left % BlockSize, - right % BlockSize); + last % BlockSize); } const std::size_t left_offset = left % BlockSize; - const std::size_t right_offset = right % BlockSize; - Candidate answer = - scan_block_range(left_block, left_offset, block_size(left_block) - 1); - answer = better(answer, scan_block_range(right_block, 0, right_offset)); + const std::size_t right_offset = last % BlockSize; + Candidate answer; + if (left_offset > right_offset && left_offset < BlockSize - 1 && + right_offset > 0) { + const auto left_bits = block_bits(left_block); + const auto right_bits = block_bits(right_block); + const ExcessBoundaryPairResult result = block_disjoint_boundary_min( + left_bits.data(), left_offset, right_bits.data(), right_offset); + answer = local_excess_result_candidate(left_block, result.suffix); + answer = better( + answer, local_excess_result_candidate(right_block, result.prefix)); + } else { + answer = + scan_block_range(left_block, left_offset, block_size(left_block) - 1); + answer = better(answer, scan_block_range(right_block, 0, right_offset)); + } if (left_block + 1 < right_block) { const std::size_t block_position = - macro_rmq_.arg_min(left_block + 1, right_block - 1); + macro_rmq_.arg_min(left_block + 1, right_block); if (block_position != MacroRmq::npos) { answer = better(answer, scan_full_block(block_position)); } @@ -180,7 +194,7 @@ class BpPlusMinusOneRmq { }; /** - * @brief Packed summary for one 128-position depth block. + * @brief Packed summary for one depth block. * * @details Stores signed 48-bit base depth, signed 48-bit absolute block * minimum, and 16-bit first local offset of that minimum in two 64-bit words. @@ -375,7 +389,7 @@ class BpPlusMinusOneRmq { * @brief Build block summaries and the macro sparse table. * * @details Computes the absolute base depth, absolute block minimum, and - * first local minimum offset for each 128-position block. + * first local minimum offset for each block. * * @throws std::length_error if @p Index or a packed block summary overflows. * @throws std::invalid_argument if the input bit span is too small. @@ -401,39 +415,19 @@ class BpPlusMinusOneRmq { for (std::size_t block = 0; block < block_count; ++block) { const std::size_t begin = block * BlockSize; const std::size_t size = std::min(BlockSize, depth_count_ - begin); - std::size_t min_offset = 0; - std::int64_t min_depth = base_depth; - std::int64_t current_depth = base_depth; - for (std::size_t offset = 1; offset < size; ++offset) { - const std::size_t delta_position = begin + offset - 1; - const bool up = bit(delta_position); - current_depth += up ? 1 : -1; - if (current_depth < min_depth) { - min_depth = current_depth; - min_offset = offset; - } - } + const auto bits = block_bits(block); + const ExcessResult block_min = block_excess_min(bits.data(), 0, size - 1); - block_summaries_.push_back( - BlockSummary::make(base_depth, min_depth, min_offset)); + block_summaries_.push_back(BlockSummary::make( + base_depth, base_depth + block_min.min_excess, block_min.offset)); if (block + 1 < block_count) { - base_depth += block_excess(begin, next_block_delta_count(begin)); + base_depth += block_excess(bits, next_block_delta_count(begin)); } } reset_macro_rmq(); } - /** - * @brief Read one BP delta bit. - * - * @param position Zero-based delta-bit position. - * @return `true` for a +1 step and `false` for a -1 step. - */ - bool bit(std::size_t position) const { - return ((input_bits_[position >> 6] >> (position & 63)) & 1u) != 0; - } - /** * @brief Read an input word or return zero past the available span. * @@ -445,13 +439,16 @@ class BpPlusMinusOneRmq { } /** - * @brief Load the two 64-bit words backing a 128-position block. + * @brief Load the words backing one block. * * @param block Zero-based block index. - * @return Pair of words suitable for `excess_min_128`. + * @return Pair of words; 64-position blocks use only the first word. */ std::array block_bits(std::size_t block) const { const std::size_t first_word = block * (BlockSize / 64); + if constexpr (BlockSize == 64) { + return {word_or_zero(first_word), 0}; + } return {word_or_zero(first_word), word_or_zero(first_word + 1)}; } @@ -470,16 +467,55 @@ class BpPlusMinusOneRmq { /** * @brief Compute total excess over a contiguous delta-bit range. * - * @param begin First delta-bit position. + * @param bits Two words loaded from a block boundary. * @param delta_count Number of delta bits to scan. * @return Sum of +1/-1 deltas in the range. */ - std::int64_t block_excess(std::size_t begin, std::size_t delta_count) const { - std::int64_t excess = 0; - for (std::size_t i = 0; i < delta_count; ++i) { - excess += bit(begin + i) ? 1 : -1; + std::int64_t block_excess(const std::array& bits, + std::size_t delta_count) const { + if constexpr (BlockSize == 64) { + return prefix_excess_64(bits.data(), delta_count); } - return excess; + return prefix_excess_128(bits.data(), delta_count); + } + + /** + * @brief Find the local minimum in a block-width bit range. + * + * @param bits Words loaded from a block boundary. + * @param left First local prefix offset, inclusive. + * @param right Last local prefix offset, inclusive. + * @return Local minimum excess result. + */ + ExcessResult block_excess_min(const std::uint64_t* bits, + std::size_t left, + std::size_t right) const { + if constexpr (BlockSize == 64) { + return excess_min_64(bits, left, right); + } + return excess_min_128(bits, left, right); + } + + /** + * @brief Find local minima for disjoint left-suffix and right-prefix ranges. + * + * @param suffix_bits Words loaded from the left boundary block. + * @param suffix_left First local prefix offset in the suffix range. + * @param prefix_bits Words loaded from the right boundary block. + * @param prefix_right Last local prefix offset in the prefix range. + * @return Pair of local minimum results. + */ + ExcessBoundaryPairResult block_disjoint_boundary_min( + const std::uint64_t* suffix_bits, + std::size_t suffix_left, + const std::uint64_t* prefix_bits, + std::size_t prefix_right) const { + if constexpr (BlockSize == 64) { + return excess_min_64_disjoint_suffix_prefix(suffix_bits, suffix_left, + prefix_bits, prefix_right); + } + return excess_min_128_disjoint_suffix_prefix(suffix_bits, suffix_left, + prefix_bits, prefix_right); } /** @@ -501,7 +537,7 @@ class BpPlusMinusOneRmq { right_offset = std::min(right_offset, size - 1); const auto bits = block_bits(block); const ExcessResult result = - excess_min_128(bits.data(), left_offset, right_offset); + block_excess_min(bits.data(), left_offset, right_offset); if (result.offset == npos || result.offset >= size) { return npos; } @@ -511,7 +547,8 @@ class BpPlusMinusOneRmq { /** * @brief Scan an inclusive local range inside one block. * - * @details Uses `excess_min_128` for the relative minimum and combines it + * @details Uses the block-width excess-min primitive for the relative minimum + * and combines it * with the stored block base depth to return an absolute-depth candidate. * * @param block Zero-based block index. @@ -527,7 +564,7 @@ class BpPlusMinusOneRmq { right_offset = std::min(right_offset, size - 1); const auto bits = block_bits(block); const ExcessResult result = - excess_min_128(bits.data(), left_offset, right_offset); + block_excess_min(bits.data(), left_offset, right_offset); if (result.offset == npos || result.offset >= size) { return {}; } @@ -535,6 +572,23 @@ class BpPlusMinusOneRmq { block_summaries_[block].base_depth() + result.min_excess}; } + /** + * @brief Convert a local excess-min result into an absolute-depth candidate. + * + * @param block Zero-based block index owning @p result. + * @param result Local minimum result from a block-width excess primitive. + * @return Candidate containing global position and absolute depth. + */ + Candidate local_excess_result_candidate(std::size_t block, + ExcessResult result) const { + const std::size_t size = block_size(block); + if (result.offset == npos || result.offset >= size) { + return {}; + } + return {block * BlockSize + result.offset, + block_summaries_[block].base_depth() + result.min_excess}; + } + /** * @brief Return the precomputed minimum candidate for a full block. * diff --git a/include/pixie/rmq/cartesian_tree_rmq.h b/include/pixie/rmq/cartesian_tree_rmq.h index 6048996..6961ddf 100644 --- a/include/pixie/rmq/cartesian_tree_rmq.h +++ b/include/pixie/rmq/cartesian_tree_rmq.h @@ -1,27 +1,30 @@ #pragma once +#include #include #include -#include #include #include #include #include +#include +#include #include #include +#include #include #include namespace pixie::rmq { /** - * @brief General RMQ via Cartesian-tree reduction to ±1 RMQ. + * @brief General RMQ via the Ferrada-Navarro BP Cartesian-tree encoding. * - * @details Builds a stable min Cartesian tree over the indexed values, takes - * its Euler tour, and answers original RMQ queries as LCA queries implemented - * by RMQ over the Euler depth sequence. The indexed values are not owned and - * must outlive this object. + * @details Builds the balanced-parentheses representation of the Cartesian-tree + * RMQ information directly from the indexed values. Queries use select/rank on + * BP positions plus a ±1 RMQ over BP prefix excess. The indexed values are not + * owned and must outlive this object. * * @tparam T Value type in the indexed array. * @tparam Compare Strict weak ordering used to choose minima. @@ -31,6 +34,9 @@ template , class Index = std::size_t> class CartesianTreeRmq : public RmqBase, T> { public: + static_assert(std::is_unsigned_v, + "CartesianTreeRmq index type must be unsigned"); + static constexpr std::size_t npos = RmqBase, T>::npos; static constexpr Index invalid_index = std::numeric_limits::max(); @@ -64,13 +70,9 @@ class CartesianTreeRmq CartesianTreeRmq(const CartesianTreeRmq& other) : values_(other.values_), compare_(other.compare_), - left_child_(other.left_child_), - right_child_(other.right_child_), - first_occurrence_(other.first_occurrence_), - euler_nodes_(other.euler_nodes_), - depths_(other.depths_), - euler_delta_bits_(other.euler_delta_bits_) { - reset_depth_rmq(); + bp_bits_(other.bp_bits_), + bp_bit_count_(other.bp_bit_count_) { + reset_bp_indexes(); } /** @@ -85,13 +87,9 @@ class CartesianTreeRmq } values_ = other.values_; compare_ = other.compare_; - left_child_ = other.left_child_; - right_child_ = other.right_child_; - first_occurrence_ = other.first_occurrence_; - euler_nodes_ = other.euler_nodes_; - depths_ = other.depths_; - euler_delta_bits_ = other.euler_delta_bits_; - reset_depth_rmq(); + bp_bits_ = other.bp_bits_; + bp_bit_count_ = other.bp_bit_count_; + reset_bp_indexes(); return *this; } @@ -103,13 +101,11 @@ class CartesianTreeRmq CartesianTreeRmq(CartesianTreeRmq&& other) noexcept : values_(other.values_), compare_(std::move(other.compare_)), - left_child_(std::move(other.left_child_)), - right_child_(std::move(other.right_child_)), - first_occurrence_(std::move(other.first_occurrence_)), - euler_nodes_(std::move(other.euler_nodes_)), - depths_(std::move(other.depths_)), - euler_delta_bits_(std::move(other.euler_delta_bits_)) { - reset_depth_rmq(); + bp_bits_(std::move(other.bp_bits_)), + bp_bit_count_(other.bp_bit_count_) { + other.values_ = std::span(); + other.bp_bit_count_ = 0; + reset_bp_indexes(); } /** @@ -124,13 +120,11 @@ class CartesianTreeRmq } values_ = other.values_; compare_ = std::move(other.compare_); - left_child_ = std::move(other.left_child_); - right_child_ = std::move(other.right_child_); - first_occurrence_ = std::move(other.first_occurrence_); - euler_nodes_ = std::move(other.euler_nodes_); - depths_ = std::move(other.depths_); - euler_delta_bits_ = std::move(other.euler_delta_bits_); - reset_depth_rmq(); + bp_bits_ = std::move(other.bp_bits_); + bp_bit_count_ = other.bp_bit_count_; + other.values_ = std::span(); + other.bp_bit_count_ = 0; + reset_bp_indexes(); return *this; } @@ -150,190 +144,141 @@ class CartesianTreeRmq T value_at_impl(std::size_t position) const { return values_[position]; } /** - * @brief Return the first minimum position in [@p left, @p right]. + * @brief Return the first minimum position in [@p left, @p right). * - * @details Converts the query to an LCA query over the Cartesian-tree Euler - * tour and returns the corresponding original array position. Ties return the - * smaller original position because the Cartesian tree is stable. + * @details Adapts the Ferrada-Navarro BP formula to Pixie's zero-based + * half-open ranges. Ties return the smaller original position. * * @param left First position in the query range. - * @param right Last position in the query range. + * @param right One past the last position in the query range. * @return Zero-based position of the first range minimum, or `npos`. */ std::size_t arg_min_impl(std::size_t left, std::size_t right) const { - if (left > right || right >= values_.size()) { + if (left >= right || right > values_.size()) { return npos; } - std::size_t first = first_occurrence_[left]; - std::size_t second = first_occurrence_[right]; - if (first > second) { - std::swap(first, second); + const BitVector& bp_index = *bp_index_; + const std::size_t first_close = bp_index.select0(left + 1); + const std::size_t last_close = bp_index.select0(right); + if (first_close == bp_bit_count_ || last_close == bp_bit_count_ || + first_close > last_close) { + return npos; } - const std::size_t euler_position = depth_rmq_.arg_min(first, second); - if (euler_position == npos) { + + const std::size_t shifted_min = + bp_depth_rmq_.arg_min(first_close + 1, last_close + 2); + if (shifted_min == npos || shifted_min == 0) { return npos; } - return euler_nodes_[euler_position]; + const std::size_t answer = bp_index.rank0(shifted_min) - 1; + return answer < values_.size() ? answer : npos; } /** - * @brief Return the Euler-tour node sequence used by the reduction. + * @brief Return the number of BP bits in the Cartesian-tree RMQ encoding. * - * @return Non-owning span of original array positions in Euler-tour order. + * @return Logical BP bit count, equal to `2 * size()`. */ - std::span euler_nodes() const { return euler_nodes_; } + std::size_t bp_bit_count() const { return bp_bit_count_; } /** - * @brief Return the Euler-tour depth sequence used by the reduction. + * @brief Return the packed BP words used by the RMQ encoding. * - * @return Non-owning span of depths corresponding to `euler_nodes()`. + * @return Non-owning span of little-endian packed BP words. */ - std::span euler_depths() const { return depths_; } + std::span bp_words() const { return bp_bits_; } private: /** - * @brief Rebuild all Cartesian-tree and Euler-tour auxiliary data. + * @brief Rebuild the direct BP Cartesian-tree RMQ representation. * - * @details Clears previous state, builds a stable Cartesian tree, records its - * Euler tour, converts adjacent Euler-depth deltas to bits, and rebuilds the - * ±1 RMQ backend. + * @details Constructs the Ferrada-Navarro BP sequence with a right-to-left + * monotone stack, then rebuilds select/rank and excess-min indexes over the + * packed BP words. * * @throws std::length_error if @p Index cannot represent all positions. */ void build() { - left_child_.clear(); - right_child_.clear(); - first_occurrence_.clear(); - euler_nodes_.clear(); - depths_.clear(); - euler_delta_bits_.clear(); - depth_rmq_ = BpPlusMinusOneRmq(); + bp_bits_.clear(); + bp_bit_count_ = 0; + reset_bp_indexes(); if (values_.empty()) { return; } - if (values_.size() > static_cast(invalid_index)) { + if (values_.size() > (static_cast(invalid_index) - 1) / 2) { throw std::length_error("Cartesian RMQ index type is too small"); } - left_child_.assign(values_.size(), invalid_index); - right_child_.assign(values_.size(), invalid_index); - - const std::size_t root = build_cartesian_tree(); - first_occurrence_.assign(values_.size(), invalid_index); - euler_nodes_.reserve(2 * values_.size() - 1); - depths_.reserve(2 * values_.size() - 1); - euler_tour(root, 0); - build_euler_delta_bits(); - reset_depth_rmq(); + bp_bit_count_ = 2 * values_.size(); + bp_bits_.assign((bp_bit_count_ + 63) / 64, 0); + build_bp_bits(); + reset_bp_indexes(); } /** - * @brief Build the stable min Cartesian tree. - * - * @details Uses the standard monotone-stack construction. Strictly smaller - * values become ancestors; equal values are not popped, preserving first - * minimum tie-breaking. + * @brief Build the BP bits with the Ferrada-Navarro stack construction. * - * @return Root node position in the original value array. + * @details The paper describes prepending bits while scanning right-to-left. + * This implementation fills the destination bitvector from right to left. */ - std::size_t build_cartesian_tree() { - std::vector stack; - stack.reserve(values_.size()); + void build_bp_bits() { + const auto stack = std::make_unique_for_overwrite(values_.size()); + std::size_t stack_size = 0; + std::size_t write_position = bp_bit_count_; - for (std::size_t i = 0; i < values_.size(); ++i) { - Index last = invalid_index; - while (!stack.empty() && compare_(values_[i], values_[stack.back()])) { - last = stack.back(); - stack.pop_back(); - } - if (last != invalid_index) { - left_child_[i] = last; + for (std::size_t i = values_.size(); i-- > 0;) { + while (stack_size != 0 && + !compare_(values_[stack[stack_size - 1]], values_[i])) { + --stack_size; + prepend_bp_bit(write_position, true); } - if (!stack.empty()) { - right_child_[stack.back()] = static_cast(i); - } - stack.push_back(static_cast(i)); + stack[stack_size++] = static_cast(i); + prepend_bp_bit(write_position, false); } - return stack.front(); - } - - /** - * @brief Append the Euler tour of a Cartesian-tree subtree. - * - * @details Visits @p node, recurses into each existing child, and appends - * @p node again after returning from that child. - * - * @param node Current Cartesian-tree node. - * @param depth Depth of @p node in the Cartesian tree. - */ - void euler_tour(std::size_t node, std::int64_t depth) { - append_euler(node, depth); - if (left_child_[node] != invalid_index) { - euler_tour(left_child_[node], depth + 1); - append_euler(node, depth); - } - if (right_child_[node] != invalid_index) { - euler_tour(right_child_[node], depth + 1); - append_euler(node, depth); + while (write_position != 0) { + prepend_bp_bit(write_position, true); } } /** - * @brief Append one node/depth pair to the Euler-tour arrays. + * @brief Prepend one BP bit into the right-to-left construction buffer. * - * @details Records the first Euler occurrence of @p node if this is the first - * time the node is appended. - * - * @param node Cartesian-tree node, also an original value position. - * @param depth Depth of @p node in the Cartesian tree. + * @param write_position Current first unwritten position; decremented here. + * @param bit `true` for an opening parenthesis and `false` for closing. */ - void append_euler(std::size_t node, std::int64_t depth) { - if (first_occurrence_[node] == invalid_index) { - first_occurrence_[node] = static_cast(euler_nodes_.size()); + void prepend_bp_bit(std::size_t& write_position, bool bit) { + --write_position; + if (bit) { + bp_bits_[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); } - euler_nodes_.push_back(static_cast(node)); - depths_.push_back(depth); } /** - * @brief Rebuild the ±1 RMQ backend over the current Euler-depth deltas. + * @brief Rebuild indexes that store non-owning spans into `bp_bits_`. * - * @details Called after build, copy, and move operations because the backend - * stores non-owning spans into this object's `euler_delta_bits_` storage. + * @details Called after build, copy, and move operations because both indexes + * keep views into this object's packed BP word storage. */ - void reset_depth_rmq() { - depth_rmq_ = BpPlusMinusOneRmq( - std::span(euler_delta_bits_), depths_.size()); - } - - /** - * @brief Pack adjacent Euler-depth changes into BP-style delta bits. - * - * @details Bit `1` means the next Euler depth is current depth + 1; bit `0` - * means current depth - 1. Cartesian-tree Euler tours have only ±1 adjacent - * depth changes. - */ - void build_euler_delta_bits() { - euler_delta_bits_.assign((depths_.size() - 1 + 63) / 64, 0); - for (std::size_t i = 1; i < depths_.size(); ++i) { - const std::int64_t delta = depths_[i] - depths_[i - 1]; - if (delta == 1) { - euler_delta_bits_[(i - 1) >> 6] |= std::uint64_t{1} << ((i - 1) & 63); - } + void reset_bp_indexes() { + bp_index_.reset(); + bp_depth_rmq_ = BpPlusMinusOneRmq(); + if (bp_bit_count_ == 0) { + return; } + const std::span words(bp_bits_); + bp_index_.emplace(words, bp_bit_count_); + bp_depth_rmq_ = BpPlusMinusOneRmq(words, bp_bit_count_ + 1); } std::span values_; Compare compare_; - std::vector left_child_; - std::vector right_child_; - std::vector first_occurrence_; - std::vector euler_nodes_; - std::vector depths_; - std::vector euler_delta_bits_; - BpPlusMinusOneRmq depth_rmq_; + std::vector bp_bits_; + std::size_t bp_bit_count_ = 0; + std::optional bp_index_; + BpPlusMinusOneRmq bp_depth_rmq_; }; } // namespace pixie::rmq diff --git a/include/pixie/rmq/experimental/node_euler_btree_rmq.h b/include/pixie/rmq/experimental/node_euler_btree_rmq.h new file mode 100644 index 0000000..156e04a --- /dev/null +++ b/include/pixie/rmq/experimental/node_euler_btree_rmq.h @@ -0,0 +1,1217 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq::experimental { + +/** + * @brief Low-level selector tag for BP-based 252-value leaves. + */ +struct BpLeafSelectorTag {}; + +/** + * @brief Experimental value RMQ with compact per-level BP selectors. + * + * @details Values are split into 252-value leaves. Low nodes store 504 BP bits + * plus an 8-bit local-minimum offset in one 512-bit selector. Middle internal + * nodes use 192-way fanout and store 384 BP bits, a 64-bit absolute + * subtree-minimum position, and 64 bits of zero-rank prefix metadata in the + * selector. High nodes use 256-way fanout and are only the root and its child + * level, so there are at most 257 of them. High nodes add a sparse depth RMQ + * side table and cache each child slot's value range and subtree minimum. + * + * This backend is intentionally not included by `pixie/rmq.h`; include this + * header directly while evaluating the experiment. + * + * @tparam T Value type in the indexed array. + * @tparam Compare Strict weak ordering used to choose minima. + * @tparam Index Unsigned integer type used for stored positions. + * @tparam LeafSize Number of original values per leaf. This experimental + * backend currently supports only 252. + * @tparam Fanout Number of children per high internal node. This experimental + * backend currently supports only 256. Middle internal nodes use 192. + * @tparam LowLevelSelector Compile-time low-level selector tag. The default + * BP leaf selector is the only implemented tag in this cleanup pass. + */ +template , + class Index = std::size_t, + std::size_t LeafSize = 252, + std::size_t Fanout = 256, + class LowLevelSelector = BpLeafSelectorTag> +class NodeEulerBTreeRmq : public RmqBase, + T> { + public: + static_assert(std::is_unsigned_v, + "NodeEulerBTreeRmq index type must be unsigned"); + static_assert(LeafSize == 252, + "experimental NodeEulerBTreeRmq currently requires 252-value " + "leaves"); + static_assert(Fanout == 256, + "experimental NodeEulerBTreeRmq currently requires 256-way " + "internal nodes"); + static_assert(std::is_same_v, + "only BP low-level leaves are implemented"); + + using Self = + NodeEulerBTreeRmq; + + static constexpr std::size_t npos = RmqBase::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + static constexpr std::size_t kLeafSize = LeafSize; + static constexpr std::size_t kFanout = Fanout; + static constexpr std::size_t kMiddleFanout = 192; + + /** + * @brief Construct an empty experimental RMQ index. + */ + NodeEulerBTreeRmq() = default; + NodeEulerBTreeRmq(const NodeEulerBTreeRmq&) = default; + NodeEulerBTreeRmq(NodeEulerBTreeRmq&&) noexcept = default; + NodeEulerBTreeRmq& operator=(const NodeEulerBTreeRmq&) = default; + NodeEulerBTreeRmq& operator=(NodeEulerBTreeRmq&&) noexcept = default; + + /** + * @brief Build an experimental B-tree RMQ index over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values keep the smaller original position as the answer. + * + * @param values Values to index. + * @param compare Ordering used to choose minima. + * @throws std::length_error if @p Index cannot represent all positions. + */ + explicit NodeEulerBTreeRmq(std::span values, + Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + /** + * @brief Return the number of indexed values. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + * + * @details Empty or invalid ranges return `npos`. Ties are reduced by + * comparing `(value, position)`, so traversal order cannot change first-min + * semantics. + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size() || level_sizes_.empty()) { + return npos; + } + + const std::size_t root_level = level_count() - 1; + if (left == 0 && right == values_.size()) { + return subtree_min_position(root_level, 0); + } + + const std::size_t left_leaf = leaf_for_value(left); + const std::size_t right_leaf = leaf_for_value(right - 1); + if (left_leaf == right_leaf) { + return leaf_range_min(left_leaf, left, right); + } + + const auto [level, node_index] = covering_node(left_leaf, right_leaf); + return query_node(level, node_index, left, right); + } + + private: + static constexpr std::size_t kSelectorEntries = 256; + static constexpr std::size_t kSelectorBits = 2 * kSelectorEntries; + static constexpr std::size_t kSelectorWords = kSelectorBits / 64; + static constexpr std::size_t kEmbeddedOffsetEntries = 252; + static constexpr std::size_t kEmbeddedPositionEntries = 192; + static constexpr std::size_t kEmbeddedOffsetBit = 2 * kEmbeddedOffsetEntries; + static constexpr std::size_t kEmbeddedPositionBit = + 2 * kEmbeddedPositionEntries; + static constexpr std::size_t kMiddleZeroPrefixWord = 7; + static constexpr std::size_t kHighSparseLevels = + static_cast(std::bit_width(Fanout)); + static constexpr std::size_t kHighSparseSlotsPerNode = + kHighSparseLevels * Fanout; + static constexpr std::size_t kLeafLinearScanThreshold = 64; + static constexpr std::size_t kLeafAvx2ScanThreshold = 16; + static constexpr std::uint64_t kEmbeddedOffsetMask = + std::uint64_t{0xFF} << (kEmbeddedOffsetBit & 63); + static constexpr bool kInvalidIndexEqualsNpos = + static_cast(invalid_index) == npos; + + static_assert(kEmbeddedOffsetBit + 8 == kSelectorBits); + static_assert(kEmbeddedPositionBit + 128 == kSelectorBits); + static_assert(LeafSize <= kEmbeddedOffsetEntries); + static_assert(kMiddleFanout <= kEmbeddedPositionEntries); + + struct HighChildMetadata { + std::size_t value_begin = 0; + std::size_t value_end = 0; + Index min_position = invalid_index; + }; + + class alignas(64) Bp512Selector { + public: + Bp512Selector() = default; + + template + void build(std::size_t entry_count, EntryLess entry_less) { + if (entry_count > kSelectorEntries) { + throw std::length_error("NodeEulerBTreeRmq local selector too large"); + } + + bp_bits_.fill(0); + if (entry_count == 0) { + return; + } + + std::array stack{}; + std::size_t stack_size = 0; + std::size_t write_position = 2 * entry_count; + + for (std::size_t i = entry_count; i-- > 0;) { + while (stack_size != 0 && !entry_less(stack[stack_size - 1], i)) { + --stack_size; + prepend_bp_bit(write_position, true); + } + stack[stack_size++] = static_cast(i); + prepend_bp_bit(write_position, false); + } + + while (write_position != 0) { + prepend_bp_bit(write_position, true); + } + } + + std::size_t arg_min(std::size_t slot_left, + std::size_t slot_right, + std::size_t entry_count) const { + if (slot_left >= slot_right || slot_right > entry_count || + entry_count > kSelectorEntries) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t bit_count = 2 * entry_count; + const std::size_t first_close = close_position(slot_left); + const std::size_t last_close = close_position(slot_right - 1); + if (first_close > last_close || last_close >= bit_count) { + return npos; + } + + const std::size_t shifted_min = + depth_arg_min(first_close + 1, last_close + 2, bit_count); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + + const std::size_t zero_rank = rank0_at(shifted_min, bit_count); + if (zero_rank == 0) { + return npos; + } + const std::size_t entry = zero_rank - 1; + return entry < entry_count ? entry : npos; + } + + std::size_t arg_min_with_zero_prefix(std::size_t slot_left, + std::size_t slot_right, + std::size_t entry_count) const { + if (slot_left >= slot_right || slot_right > entry_count || + entry_count > kEmbeddedPositionEntries) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t bit_count = 2 * entry_count; + const std::size_t first_close = + close_position_with_zero_prefix(slot_left, bit_count); + const std::size_t last_close = + close_position_with_zero_prefix(slot_right - 1, bit_count); + if (first_close > last_close || last_close >= bit_count) { + return npos; + } + + const std::size_t shifted_min = depth_arg_min_with_zero_prefix( + first_close + 1, last_close + 2, bit_count); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + + const std::size_t zero_rank = + rank0_at_with_zero_prefix(shifted_min, bit_count); + if (zero_rank == 0) { + return npos; + } + const std::size_t entry = zero_rank - 1; + return entry < entry_count ? entry : npos; + } + + std::size_t close_position(std::size_t slot) const { + return select0_512(bp_bits_.data(), slot); + } + + std::size_t rank0_at(std::size_t position, std::size_t bit_count) const { + position = std::min(position, bit_count); + return position - rank_512(bp_bits_.data(), position); + } + + std::int16_t depth_at_position(std::size_t position) const { + return static_cast(prefix_excess(position)); + } + + void set_embedded_min_offset(std::size_t offset) { + bp_bits_[kEmbeddedOffsetBit >> 6] = + (bp_bits_[kEmbeddedOffsetBit >> 6] & ~kEmbeddedOffsetMask) | + ((static_cast(offset) & 0xFFu) + << (kEmbeddedOffsetBit & 63)); + } + + std::uint8_t embedded_min_offset() const { + return static_cast( + (bp_bits_[kEmbeddedOffsetBit >> 6] & kEmbeddedOffsetMask) >> + (kEmbeddedOffsetBit & 63)); + } + + void set_embedded_min_position(std::size_t position) { + bp_bits_[kEmbeddedPositionBit >> 6] = + static_cast(position); + } + + std::size_t embedded_min_position() const { + return static_cast(bp_bits_[kEmbeddedPositionBit >> 6]); + } + + void build_zero_prefix_metadata(std::size_t bit_count) { + const std::size_t word_count = (bit_count + 63) / 64; + std::uint64_t packed = 0; + std::size_t zeros = 0; + for (std::size_t word = 0; word <= word_count; ++word) { + packed |= (static_cast(zeros) & 0xFFu) << (8 * word); + if (word == word_count) { + break; + } + const std::size_t word_begin = word * 64; + const std::size_t word_bits = + std::min(64, bit_count - word_begin); + const std::uint64_t bits = bp_bits_[word] & first_bits_mask(word_bits); + zeros += word_bits - std::popcount(bits); + } + bp_bits_[kMiddleZeroPrefixWord] = packed; + } + + private: + std::size_t prepend_bp_bit(std::size_t& write_position, bool bit) { + --write_position; + if (bit) { + bp_bits_[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); + } + return write_position; + } + + int prefix_excess(std::size_t position) const { + position = std::min(position, kSelectorBits); + const std::size_t ones = rank_512(bp_bits_.data(), position); + return static_cast(2 * ones) - static_cast(position); + } + + std::size_t zero_prefix_at_word(std::size_t word) const { + return static_cast(bp_bits_[kMiddleZeroPrefixWord] >> + (8 * word)); + } + + std::size_t rank0_at_with_zero_prefix(std::size_t position, + std::size_t bit_count) const { + position = std::min(position, bit_count); + const std::size_t full_words = position >> 6; + std::size_t zeros = zero_prefix_at_word(full_words); + const std::size_t tail_bits = position & 63; + if (tail_bits != 0) { + const std::uint64_t tail = + bp_bits_[full_words] & first_bits_mask(tail_bits); + zeros += tail_bits - std::popcount(tail); + } + return zeros; + } + + int prefix_excess_with_zero_prefix(std::size_t position, + std::size_t bit_count) const { + position = std::min(position, bit_count); + return static_cast(position) - + 2 * static_cast( + rank0_at_with_zero_prefix(position, bit_count)); + } + + std::size_t close_position_with_zero_prefix(std::size_t slot, + std::size_t bit_count) const { + const std::size_t word_count = (bit_count + 63) / 64; + for (std::size_t word = 0; word < word_count; ++word) { + const std::size_t next_zero_prefix = zero_prefix_at_word(word + 1); + if (next_zero_prefix <= slot) { + continue; + } + const std::size_t local_rank = slot - zero_prefix_at_word(word); + const std::size_t word_begin = word * 64; + const std::size_t word_bits = + std::min(64, bit_count - word_begin); + const std::uint64_t zeros = + (~bp_bits_[word]) & first_bits_mask(word_bits); + return word_begin + select_64(zeros, local_rank); + } + return npos; + } + + std::size_t depth_arg_min(std::size_t left, + std::size_t right, + std::size_t bit_count) const { + const std::size_t depth_count = bit_count + 1; + if (left >= right || right > depth_count) { + return npos; + } + + std::size_t position = left; + int best_depth = prefix_excess(position); + std::size_t best_position = position; + + while (position < right) { + const std::size_t chunk_begin = (position / 128) * 128; + const std::size_t local_left = position - chunk_begin; + const std::size_t local_right = + std::min(right - 1, chunk_begin + 128) - chunk_begin; + + int candidate_depth; + std::size_t candidate_position; + if (chunk_begin >= bit_count) { + candidate_depth = prefix_excess(chunk_begin); + candidate_position = chunk_begin; + } else { + const std::size_t word = chunk_begin >> 6; + const ExcessResult result = + excess_min_128(bp_bits_.data() + word, local_left, local_right); + candidate_depth = prefix_excess(chunk_begin) + result.min_excess; + candidate_position = chunk_begin + result.offset; + } + + if (candidate_depth < best_depth) { + best_depth = candidate_depth; + best_position = candidate_position; + } + + position = chunk_begin + local_right + 1; + } + + return best_position; + } + + std::size_t depth_arg_min_with_zero_prefix(std::size_t left, + std::size_t right, + std::size_t bit_count) const { + const std::size_t depth_count = bit_count + 1; + if (left >= right || right > depth_count) { + return npos; + } + + std::size_t position = left; + int best_depth = prefix_excess_with_zero_prefix(position, bit_count); + std::size_t best_position = position; + + while (position < right) { + const std::size_t chunk_begin = (position / 128) * 128; + const std::size_t local_left = position - chunk_begin; + const std::size_t local_right = + std::min(right - 1, chunk_begin + 128) - chunk_begin; + + int candidate_depth; + std::size_t candidate_position; + if (chunk_begin >= bit_count) { + candidate_depth = + prefix_excess_with_zero_prefix(chunk_begin, bit_count); + candidate_position = chunk_begin; + } else { + const std::size_t word = chunk_begin >> 6; + const ExcessResult result = + excess_min_128(bp_bits_.data() + word, local_left, local_right); + candidate_depth = + prefix_excess_with_zero_prefix(chunk_begin, bit_count) + + result.min_excess; + candidate_position = chunk_begin + result.offset; + } + + if (candidate_depth < best_depth) { + best_depth = candidate_depth; + best_position = candidate_position; + } + + position = chunk_begin + local_right + 1; + } + + return best_position; + } + + std::array bp_bits_{}; + }; + + static_assert(sizeof(Bp512Selector) == 64); + + class TopDepthSparseSelector { + public: + TopDepthSparseSelector() = default; + + void build(const Bp512Selector& selector, std::size_t entry_count) { + const std::size_t depth_count = 2 * entry_count + 1; + depth_count_ = depth_count; + depths_.assign(depth_count, 0); + if (depth_count == 0) { + log_count_ = 0; + sparse_positions_.clear(); + return; + } + + for (std::size_t position = 0; position < depth_count; ++position) { + depths_[position] = selector.depth_at_position(position); + } + + log_count_ = std::bit_width(depth_count); + sparse_positions_.assign(log_count_ * depth_count, 0); + for (std::size_t position = 0; position < depth_count; ++position) { + sparse_positions_[position] = static_cast(position); + } + + for (std::size_t level = 1; level < log_count_; ++level) { + const std::size_t half_span = std::size_t{1} << (level - 1); + const std::size_t span = half_span << 1; + if (span > depth_count) { + break; + } + const std::size_t previous_offset = (level - 1) * depth_count; + const std::size_t current_offset = level * depth_count; + for (std::size_t position = 0; position + span <= depth_count; + ++position) { + sparse_positions_[current_offset + position] = better_depth_position( + sparse_positions_[previous_offset + position], + sparse_positions_[previous_offset + position + half_span]); + } + } + } + + std::size_t arg_min(const Bp512Selector& selector, + std::size_t slot_left, + std::size_t slot_right, + std::size_t entry_count) const { + if (slot_left >= slot_right || slot_right > entry_count || + entry_count > kSelectorEntries) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t bit_count = 2 * entry_count; + const std::size_t first_close = selector.close_position(slot_left); + const std::size_t last_close = selector.close_position(slot_right - 1); + if (first_close > last_close || last_close >= bit_count) { + return npos; + } + + const std::size_t shifted_min = + depth_arg_min(first_close + 1, last_close + 2); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + + const std::size_t zero_rank = selector.rank0_at(shifted_min, bit_count); + if (zero_rank == 0) { + return npos; + } + const std::size_t entry = zero_rank - 1; + return entry < entry_count ? entry : npos; + } + + private: + std::uint16_t better_depth_position(std::uint16_t left, + std::uint16_t right) const { + const std::int16_t left_depth = depths_[left]; + const std::int16_t right_depth = depths_[right]; + if (right_depth < left_depth) { + return right; + } + if (left_depth < right_depth) { + return left; + } + return std::min(left, right); + } + + std::size_t depth_arg_min(std::size_t left, std::size_t right) const { + if (left >= right || right > depth_count_) { + return npos; + } + const std::size_t length = right - left; + const std::size_t level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << level; + const std::size_t offset = level * depth_count_; + return better_depth_position(sparse_positions_[offset + left], + sparse_positions_[offset + right - span]); + } + + std::vector depths_; + std::vector sparse_positions_; + std::size_t depth_count_ = 0; + std::size_t log_count_ = 0; + }; + + bool missing_position(std::size_t position) const { + if constexpr (kInvalidIndexEqualsNpos) { + return position == npos; + } else { + return position == npos || + position == static_cast(invalid_index); + } + } + + bool strictly_better_position(std::size_t left, std::size_t right) const { + if (missing_position(left)) { + return false; + } + if (missing_position(right)) { + return true; + } + return compare_(values_[left], values_[right]); + } + + std::size_t better_position(std::size_t left, std::size_t right) const { + if (missing_position(left)) { + return right; + } + if (missing_position(right)) { + return left; + } + if (compare_(values_[right], values_[left])) { + return right; + } + if (compare_(values_[left], values_[right])) { + return left; + } + return std::min(left, right); + } + + void build() { + leaf_selectors_.clear(); + medium_selectors_.clear(); + high_selectors_.clear(); + high_depth_selectors_.clear(); + high_min_positions_.clear(); + high_child_metadata_.clear(); + high_sparse_min_slots_.clear(); + medium_level_offsets_.clear(); + high_level_offsets_.clear(); + level_sizes_.clear(); + level_value_spans_.clear(); + level_fanouts_.clear(); + high_level_begin_ = std::numeric_limits::max(); + if (values_.empty()) { + return; + } + if (values_.size() > static_cast(invalid_index)) { + throw std::length_error("NodeEulerBTreeRmq index type is too small"); + } + + initialize_layout((values_.size() + LeafSize - 1) / LeafSize); + for (std::size_t leaf = 0; leaf < level_sizes_[0]; ++leaf) { + build_leaf(leaf); + } + + for (std::size_t level = 1; level < level_count(); ++level) { + for (std::size_t node = 0; node < level_sizes_[level]; ++node) { + build_internal_node(level, node); + } + } + } + + void initialize_layout(std::size_t leaf_count) { + level_sizes_.push_back(leaf_count); + level_value_spans_.push_back(LeafSize); + level_fanouts_.push_back(0); + + std::size_t current_count = leaf_count; + std::size_t current_span = LeafSize; + while (current_count > Fanout * Fanout) { + level_fanouts_.push_back(kMiddleFanout); + current_count = ceil_div(current_count, kMiddleFanout); + current_span = saturating_product(current_span, kMiddleFanout); + level_sizes_.push_back(current_count); + level_value_spans_.push_back(current_span); + } + while (current_count > 1) { + level_fanouts_.push_back(Fanout); + current_count = ceil_div(current_count, Fanout); + current_span = saturating_product(current_span, Fanout); + level_sizes_.push_back(current_count); + level_value_spans_.push_back(current_span); + } + + leaf_selectors_.resize(level_sizes_[0]); + + medium_level_offsets_.assign(level_count(), 0); + high_level_offsets_.assign(level_count(), 0); + if (level_count() <= 1) { + high_level_begin_ = std::numeric_limits::max(); + return; + } + + const std::size_t root_level = level_count() - 1; + high_level_begin_ = root_level > 1 ? root_level - 1 : 1; + + std::size_t medium_node_count = 0; + for (std::size_t level = 1; level < high_level_begin_; ++level) { + medium_level_offsets_[level] = medium_node_count; + medium_node_count += level_sizes_[level]; + } + medium_selectors_.resize(medium_node_count); + + std::size_t high_node_count = 0; + for (std::size_t level = high_level_begin_; level < level_count(); + ++level) { + high_level_offsets_[level] = high_node_count; + high_node_count += level_sizes_[level]; + } + high_selectors_.resize(high_node_count); + high_depth_selectors_.resize(high_node_count); + high_min_positions_.resize(high_node_count, invalid_index); + high_child_metadata_.resize(high_node_count * Fanout); + high_sparse_min_slots_.resize(high_node_count * kHighSparseSlotsPerNode); + } + + void build_leaf(std::size_t leaf) { + Bp512Selector& selector = leaf_selectors_[leaf]; + const std::size_t begin = node_value_begin(0, leaf); + const std::size_t count = entry_count(0, leaf); + selector.build(count, [&](std::size_t left, std::size_t right) { + return compare_(values_[begin + left], values_[begin + right]); + }); + + const std::size_t slot = selector.arg_min(0, count, count); + selector.set_embedded_min_offset(slot); + } + + void build_internal_node(std::size_t level, std::size_t node) { + Bp512Selector& selector = mutable_selector_at(level, node); + const std::size_t count = entry_count(level, node); + const std::size_t first_child = node * fanout_at_level(level); + const bool high_level = is_high_level(level); + const std::size_t high_flat = high_level ? high_flat_index(level, node) : 0; + if (high_level) { + for (std::size_t slot = 0; slot < count; ++slot) { + const std::size_t child = first_child + slot; + HighChildMetadata& metadata = + mutable_high_child_metadata_at(high_flat, slot); + metadata.value_begin = node_value_begin(level - 1, child); + metadata.value_end = node_value_end(level - 1, child); + metadata.min_position = + static_cast(subtree_min_position(level - 1, child)); + } + build_high_sparse_min_slots(high_flat, count); + } + + selector.build(count, [&](std::size_t left, std::size_t right) { + if (high_level) { + return strictly_better_position( + high_child_metadata_at(high_flat, left).min_position, + high_child_metadata_at(high_flat, right).min_position); + } + return strictly_better_position( + subtree_min_position(level - 1, first_child + left), + subtree_min_position(level - 1, first_child + right)); + }); + + const std::size_t slot = selector.arg_min(0, count, count); + const std::size_t min_position = + high_level ? high_child_metadata_at(high_flat, slot).min_position + : subtree_min_position(level - 1, first_child + slot); + if (high_level) { + high_min_positions_[high_flat] = static_cast(min_position); + mutable_high_depth_selector_at(level, node).build(selector, count); + } else { + selector.set_embedded_min_position(min_position); + selector.build_zero_prefix_metadata(2 * count); + } + } + + static std::size_t saturating_product(std::size_t left, std::size_t right) { + if (left != 0 && right > std::numeric_limits::max() / left) { + return std::numeric_limits::max(); + } + return left * right; + } + + static std::size_t ceil_div(std::size_t value, std::size_t divisor) { + return (value + divisor - 1) / divisor; + } + + std::size_t leaf_range_min(std::size_t leaf, + std::size_t left, + std::size_t right) const { + if (left >= right) { + return npos; + } + + const std::size_t begin = node_value_begin(0, leaf); + const std::size_t end = node_value_end(0, leaf); + if (left <= begin && end <= right) { + return subtree_min_position(0, leaf); + } + if (right - left <= kLeafLinearScanThreshold) { + return linear_range_min(left, right); + } + + const std::size_t slot_left = left - begin; + const std::size_t slot_right = right - begin; + const std::size_t slot = leaf_selectors_[leaf].arg_min( + slot_left, slot_right, entry_count(0, leaf)); + if (slot == npos) { + return npos; + } + return begin + slot; + } + + std::size_t linear_range_min(std::size_t left, std::size_t right) const { + if (left >= right) { + return npos; + } +#ifdef PIXIE_AVX2_SUPPORT + if constexpr (std::is_same_v && + std::is_same_v>) { + if (right - left >= kLeafAvx2ScanThreshold) { + return linear_range_min_i64_avx2(left, right); + } + } +#endif + std::size_t best = left; + for (std::size_t position = left + 1; position < right; ++position) { + if (compare_(values_[position], values_[best])) { + best = position; + } + } + return best; + } + +#ifdef PIXIE_AVX2_SUPPORT + std::size_t linear_range_min_i64_avx2(std::size_t left, + std::size_t right) const { + const std::int64_t* data = values_.data(); + std::size_t position = left; + + __m256i best_values = + _mm256_loadu_si256(reinterpret_cast(data + position)); + __m256i best_positions = _mm256_set_epi64x( + static_cast(position + 3), + static_cast(position + 2), + static_cast(position + 1), static_cast(position)); + position += 4; + + for (; position + 4 <= right; position += 4) { + const __m256i values = + _mm256_loadu_si256(reinterpret_cast(data + position)); + const __m256i positions = + _mm256_set_epi64x(static_cast(position + 3), + static_cast(position + 2), + static_cast(position + 1), + static_cast(position)); + const __m256i take_new = _mm256_cmpgt_epi64(best_values, values); + best_values = _mm256_blendv_epi8(best_values, values, take_new); + best_positions = _mm256_blendv_epi8(best_positions, positions, take_new); + } + + alignas(32) std::int64_t value_lanes[4]; + alignas(32) std::uint64_t position_lanes[4]; + _mm256_store_si256(reinterpret_cast<__m256i*>(value_lanes), best_values); + _mm256_store_si256(reinterpret_cast<__m256i*>(position_lanes), + best_positions); + + std::int64_t best_value = value_lanes[0]; + std::size_t best_position = static_cast(position_lanes[0]); + for (std::size_t lane = 1; lane < 4; ++lane) { + const std::size_t lane_position = + static_cast(position_lanes[lane]); + if (value_lanes[lane] < best_value || + (value_lanes[lane] == best_value && lane_position < best_position)) { + best_value = value_lanes[lane]; + best_position = lane_position; + } + } + + for (; position < right; ++position) { + if (data[position] < best_value) { + best_value = data[position]; + best_position = position; + } + } + return best_position; + } +#endif + + std::pair covering_node( + std::size_t left_leaf, + std::size_t right_leaf) const { + std::size_t level = 0; + std::size_t left_node = left_leaf; + std::size_t right_node = right_leaf; + while (left_node != right_node) { + ++level; + const std::size_t fanout = fanout_at_level(level); + left_node /= fanout; + right_node /= fanout; + } + return {level, left_node}; + } + + std::size_t leaf_for_value(std::size_t position) const { + return position / LeafSize; + } + + std::size_t child_for_value(std::size_t child_level, + std::size_t position) const { + return position / level_value_spans_[child_level]; + } + + std::size_t query_child_slots(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t left, + std::size_t right) const { + if (slot_left >= slot_right) { + return npos; + } + + const std::size_t count = entry_count(level, node); + const std::size_t slot = + slot_left + 1 == slot_right + ? slot_left + : selector_arg_min(level, node, slot_left, slot_right, count); + if (slot == npos) { + return npos; + } + + const std::size_t child_level = level - 1; + const std::size_t first_child = node * fanout_at_level(level); + const std::size_t child = first_child + slot; + const HighChildMetadata* high_children = + is_high_level(level) ? high_child_metadata_begin(level, node) : nullptr; + const std::size_t child_begin = high_children != nullptr + ? high_children[slot].value_begin + : node_value_begin(child_level, child); + const std::size_t child_end = high_children != nullptr + ? high_children[slot].value_end + : node_value_end(child_level, child); + const std::size_t child_min = + high_children != nullptr ? high_children[slot].min_position + : subtree_min_position(child_level, child); + if ((left <= child_begin && child_end <= right) || + contains_position(left, right, child_min)) { + return child_min; + } + + const std::size_t last_slot = slot_right - 1; + const std::size_t left_child_begin = + high_children != nullptr + ? high_children[slot_left].value_begin + : node_value_begin(child_level, first_child + slot_left); + const std::size_t left_child_end = + high_children != nullptr + ? high_children[slot_left].value_end + : node_value_end(child_level, first_child + slot_left); + std::size_t answer = query_node(child_level, first_child + slot_left, + std::max(left, left_child_begin), + std::min(right, left_child_end)); + + if (slot_left != last_slot) { + const std::size_t right_child_begin = + high_children != nullptr + ? high_children[last_slot].value_begin + : node_value_begin(child_level, first_child + last_slot); + const std::size_t right_child_end = + high_children != nullptr + ? high_children[last_slot].value_end + : node_value_end(child_level, first_child + last_slot); + answer = better_position(answer, + query_node(child_level, first_child + last_slot, + std::max(left, right_child_begin), + std::min(right, right_child_end))); + } + + if (slot_left + 1 < last_slot) { + answer = better_position( + answer, + full_child_slot_range_min(level, node, slot_left + 1, last_slot)); + } + + return answer; + } + + std::size_t full_child_slot_range_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right) const { + if (slot_left >= slot_right) { + return npos; + } + + const std::size_t slot = + slot_left + 1 == slot_right + ? slot_left + : selector_arg_min(level, node, slot_left, slot_right, + entry_count(level, node)); + if (slot == npos) { + return npos; + } + + if (is_high_level(level)) { + return high_child_metadata_begin(level, node)[slot].min_position; + } + return subtree_min_position(level - 1, + node * fanout_at_level(level) + slot); + } + + std::size_t query_node(std::size_t level, + std::size_t node, + std::size_t left, + std::size_t right) const { + if (left >= right) { + return npos; + } + const std::size_t begin = node_value_begin(level, node); + const std::size_t end = node_value_end(level, node); + if (left <= begin && end <= right) { + return subtree_min_position(level, node); + } + if (level == 0) { + return leaf_range_min(node, left, right); + } + + const std::size_t child_level = level - 1; + const std::size_t left_child = child_for_value(child_level, left); + const std::size_t right_child = child_for_value(child_level, right - 1); + const std::size_t first_child = node * fanout_at_level(level); + const std::size_t left_slot = left_child - first_child; + const std::size_t right_slot = right_child - first_child + 1; + return query_child_slots(level, node, left_slot, right_slot, left, right); + } + + bool contains_position(std::size_t left, + std::size_t right, + std::size_t position) const { + return !missing_position(position) && left <= position && position < right; + } + + std::size_t level_count() const { return level_sizes_.size(); } + + std::size_t entry_count(std::size_t level, std::size_t node) const { + if (level == 0) { + const std::size_t begin = node_value_begin(0, node); + return std::min(LeafSize, values_.size() - begin); + } + const std::size_t first_child = node * fanout_at_level(level); + return std::min(fanout_at_level(level), + level_sizes_[level - 1] - first_child); + } + + std::size_t node_value_begin(std::size_t level, std::size_t node) const { + return node * level_value_spans_[level]; + } + + std::size_t node_value_end(std::size_t level, std::size_t node) const { + return std::min(values_.size(), + node_value_begin(level, node) + level_value_spans_[level]); + } + + std::size_t subtree_min_position(std::size_t level, std::size_t node) const { + if (level == 0) { + return node_value_begin(0, node) + + leaf_selectors_[node].embedded_min_offset(); + } + if (is_high_level(level)) { + return high_min_positions_[high_flat_index(level, node)]; + } + return selector_at(level, node).embedded_min_position(); + } + + const Bp512Selector& selector_at(std::size_t level, std::size_t node) const { + if (level == 0) { + return leaf_selectors_[node]; + } + if (is_high_level(level)) { + return high_selectors_[high_flat_index(level, node)]; + } + return medium_selectors_[medium_flat_index(level, node)]; + } + + Bp512Selector& mutable_selector_at(std::size_t level, std::size_t node) { + if (is_high_level(level)) { + return high_selectors_[high_flat_index(level, node)]; + } + return medium_selectors_[medium_flat_index(level, node)]; + } + + std::size_t selector_arg_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t count) const { + if (is_high_level(level)) { + return high_sparse_arg_min(high_flat_index(level, node), slot_left, + slot_right, count); + } + const Bp512Selector& selector = selector_at(level, node); + return selector.arg_min_with_zero_prefix(slot_left, slot_right, count); + } + + bool is_high_level(std::size_t level) const { + return level > 0 && level >= high_level_begin_ && level < level_count(); + } + + std::size_t medium_flat_index(std::size_t level, std::size_t node) const { + return medium_level_offsets_[level] + node; + } + + std::size_t fanout_at_level(std::size_t level) const { + return level_fanouts_[level]; + } + + std::size_t high_flat_index(std::size_t level, std::size_t node) const { + return high_level_offsets_[level] + node; + } + + const HighChildMetadata* high_child_metadata_begin(std::size_t level, + std::size_t node) const { + return high_child_metadata_.data() + high_flat_index(level, node) * Fanout; + } + + std::uint8_t* mutable_high_sparse_min_slots_begin(std::size_t high_flat) { + return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; + } + + const std::uint8_t* high_sparse_min_slots_begin(std::size_t high_flat) const { + return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; + } + + const HighChildMetadata& high_child_metadata_at(std::size_t high_flat, + std::size_t slot) const { + return high_child_metadata_[high_flat * Fanout + slot]; + } + + HighChildMetadata& mutable_high_child_metadata_at(std::size_t high_flat, + std::size_t slot) { + return high_child_metadata_[high_flat * Fanout + slot]; + } + + std::size_t better_high_child_slot(std::size_t high_flat, + std::size_t left_slot, + std::size_t right_slot) const { + const std::size_t left_position = + high_child_metadata_at(high_flat, left_slot).min_position; + const std::size_t right_position = + high_child_metadata_at(high_flat, right_slot).min_position; + return better_position(left_position, right_position) == right_position + ? right_slot + : left_slot; + } + + void build_high_sparse_min_slots(std::size_t high_flat, std::size_t count) { + std::uint8_t* table = mutable_high_sparse_min_slots_begin(high_flat); + for (std::size_t slot = 0; slot < count; ++slot) { + table[slot] = static_cast(slot); + } + + for (std::size_t table_level = 1; table_level < kHighSparseLevels; + ++table_level) { + const std::size_t span = std::size_t{1} << table_level; + if (span > count) { + break; + } + const std::size_t half_span = span >> 1; + const std::uint8_t* previous = table + (table_level - 1) * Fanout; + std::uint8_t* current = table + table_level * Fanout; + for (std::size_t slot = 0; slot + span <= count; ++slot) { + current[slot] = static_cast(better_high_child_slot( + high_flat, previous[slot], previous[slot + half_span])); + } + } + } + + std::size_t high_sparse_arg_min(std::size_t high_flat, + std::size_t slot_left, + std::size_t slot_right, + std::size_t count) const { + if (slot_left >= slot_right || slot_right > count) { + return npos; + } + const std::size_t length = slot_right - slot_left; + if (length == 1) { + return slot_left; + } + + const std::size_t table_level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << table_level; + const std::uint8_t* table = + high_sparse_min_slots_begin(high_flat) + table_level * Fanout; + return better_high_child_slot(high_flat, table[slot_left], + table[slot_right - span]); + } + + const TopDepthSparseSelector& high_depth_selector_at(std::size_t level, + std::size_t node) const { + return high_depth_selectors_[high_flat_index(level, node)]; + } + + TopDepthSparseSelector& mutable_high_depth_selector_at(std::size_t level, + std::size_t node) { + return high_depth_selectors_[high_flat_index(level, node)]; + } + + std::span values_; + Compare compare_; + std::vector leaf_selectors_; + std::vector medium_selectors_; + std::vector high_selectors_; + std::vector high_depth_selectors_; + std::vector high_min_positions_; + std::vector high_child_metadata_; + std::vector high_sparse_min_slots_; + std::vector medium_level_offsets_; + std::vector high_level_offsets_; + std::vector level_sizes_; + std::vector level_value_spans_; + std::vector level_fanouts_; + std::size_t high_level_begin_ = std::numeric_limits::max(); +}; + +} // namespace pixie::rmq::experimental diff --git a/include/pixie/rmq/node_euler_btree_rmq.h b/include/pixie/rmq/node_euler_btree_rmq.h new file mode 100644 index 0000000..cd7292a --- /dev/null +++ b/include/pixie/rmq/node_euler_btree_rmq.h @@ -0,0 +1,1016 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief Value RMQ with per-node Cartesian/Euler selectors. + * + * @details Values are split into fixed-size leaves, then grouped by a B-tree. + * Every leaf stores a local Cartesian BP selector over original values. Every + * internal node stores the same kind of selector over child subtree minima. + * Queries first enter the lowest B-tree node covering both endpoints, then ask + * that node's selector over all intersecting children, including partial border + * children. If the selected child is fully covered, or its stored subtree + * minimum is inside the query, the answer is known immediately. Otherwise the + * selected border child is corrected recursively and compared with the + * remaining child slots. + * + * This backend is available as the main per-node Euler B-tree RMQ + * implementation. + * + * @tparam T Value type in the indexed array. + * @tparam Compare Strict weak ordering used to choose minima. + * @tparam Index Unsigned integer type used for stored positions. + * @tparam LeafSize Number of original values per leaf. + * @tparam Fanout Maximum number of children per internal node. + */ +template , + class Index = std::size_t, + std::size_t LeafSize = 256, + std::size_t Fanout = 256> +class NodeEulerBTreeRmq + : public RmqBase, + T> { + public: + static_assert(std::is_unsigned_v, + "NodeEulerBTreeRmq index type must be unsigned"); + static_assert(LeafSize > 0); + static_assert(Fanout > 1); + + static constexpr std::size_t npos = + RmqBase, T>::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + static constexpr std::size_t kLeafSize = LeafSize; + static constexpr std::size_t kFanout = Fanout; + + /** + * @brief Construct an empty RMQ index. + */ + NodeEulerBTreeRmq() = default; + NodeEulerBTreeRmq(const NodeEulerBTreeRmq&) = default; + NodeEulerBTreeRmq(NodeEulerBTreeRmq&&) noexcept = default; + NodeEulerBTreeRmq& operator=(const NodeEulerBTreeRmq&) = default; + NodeEulerBTreeRmq& operator=(NodeEulerBTreeRmq&&) noexcept = default; + + /** + * @brief Build a B-tree RMQ index over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values keep the smaller original position as the answer. + * + * @param values Values to index. + * @param compare Ordering used to choose minima. + * @throws std::length_error if @p Index cannot represent all positions. + */ + explicit NodeEulerBTreeRmq(std::span values, + Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + /** + * @brief Return the number of indexed values. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + * + * @details Empty or invalid ranges return `npos`. Ties are reduced by + * comparing `(value, position)`, so traversal order cannot change first-min + * semantics. + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size() || nodes_.empty()) { + return npos; + } + + const NodeMetadata& root = nodes_.front(); + if (left == root.value_begin && right == root.value_end) { + return root.subtree_min_position; + } + + const bool allow_leaf_linear_scan = + right - left <= kLeafLinearScanThreshold; + const std::size_t left_leaf = leaf_for_value(left); + const std::size_t right_leaf = leaf_for_value(right - 1); + if (left_leaf == right_leaf) { + const NodeMetadata& leaf = node_at(0, left_leaf); + if (left == leaf.value_begin && right == leaf.value_end) { + return leaf.subtree_min_position; + } + return leaf_range_min(leaf, selector_at(0, left_leaf), left, right, + allow_leaf_linear_scan); + } + + const auto [level, node_index] = covering_node(left_leaf, right_leaf); + return query_node(level, node_index, left, right, allow_leaf_linear_scan); + } + + private: + static constexpr std::size_t kMaxLocalEntries = + LeafSize > Fanout ? LeafSize : Fanout; + static constexpr std::size_t kMaxLocalBits = 2 * kMaxLocalEntries; + static constexpr std::size_t kMaxLocalWords = (kMaxLocalBits + 63) / 64; + static constexpr std::size_t kMaxDepthBlocks = (kMaxLocalBits + 1 + 63) / 64; + static constexpr std::size_t kInternalScalarSlotThreshold = 4; + static constexpr std::size_t kLeafLinearScanThreshold = 64; + static constexpr std::size_t kLeafAvx2ScanThreshold = 16; + static constexpr bool kLeafSizePowerOfTwo = (LeafSize & (LeafSize - 1)) == 0; + static constexpr bool kFanoutPowerOfTwo = (Fanout & (Fanout - 1)) == 0; + static constexpr bool kPowerOfTwoLayout = + kLeafSizePowerOfTwo && kFanoutPowerOfTwo; + static constexpr bool kInvalidIndexEqualsNpos = + static_cast(invalid_index) == npos; + + static_assert( + kMaxLocalBits <= + static_cast(std::numeric_limits::max())); + static_assert( + kMaxLocalEntries <= + static_cast(std::numeric_limits::max())); + + struct DepthBlockSummary { + std::int16_t base_depth = 0; + std::int16_t min_depth = 0; + std::uint16_t min_offset = 0; + std::uint16_t size = 0; + }; + + class LocalSelector { + public: + LocalSelector() = default; + + template + void build(std::size_t entry_count, EntryLess entry_less) { + if (entry_count > kMaxLocalEntries) { + throw std::length_error("NodeEulerBTreeRmq local selector too large"); + } + + entry_count_ = static_cast(entry_count); + bit_count_ = static_cast(2 * entry_count); + word_count_ = static_cast((bit_count_ + 63) / 64); + depth_block_count_ = + static_cast((bit_count_ + 1 + 63) / 64); + bp_bits_.fill(0); + close_positions_.fill(0); + depth_blocks_.fill({}); + zero_rank_prefix_.fill(0); + + if (entry_count == 0) { + return; + } + + std::array stack{}; + std::size_t stack_size = 0; + std::size_t write_position = bit_count_; + + for (std::size_t i = entry_count; i-- > 0;) { + while (stack_size != 0 && !entry_less(stack[stack_size - 1], i)) { + --stack_size; + prepend_bp_bit(write_position, true); + } + stack[stack_size++] = static_cast(i); + close_positions_[i] = prepend_bp_bit(write_position, false); + } + + while (write_position != 0) { + prepend_bp_bit(write_position, true); + } + build_depth_blocks(); + build_zero_rank_prefix(); + } + + std::size_t arg_min(std::size_t slot_left, + std::size_t slot_right, + bool force_scalar = false) const { + if (slot_left >= slot_right || slot_right > entry_count_) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t first_close = close_positions_[slot_left]; + const std::size_t last_close = close_positions_[slot_right - 1]; + if (first_close > last_close) { + return npos; + } + + const std::size_t shifted_min = + depth_arg_min(first_close + 1, last_close + 2, force_scalar); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + + const std::size_t zero_rank = rank0(shifted_min); + if (zero_rank == 0) { + return npos; + } + const std::size_t entry = zero_rank - 1; + return entry < entry_count_ ? entry : npos; + } + + std::size_t entry_count() const { return entry_count_; } + + std::size_t depth_count() const { return bit_count_ + 1; } + + std::size_t close_position(std::size_t slot) const { + return close_positions_[slot]; + } + + std::int16_t depth_at_position(std::size_t position) const { + return depth_at(position); + } + + std::size_t rank0_at(std::size_t position) const { return rank0(position); } + + private: + std::size_t prepend_bp_bit(std::size_t& write_position, bool bit) { + --write_position; + if (bit) { + bp_bits_[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); + } + return write_position; + } + + bool bit(std::size_t position) const { + return ((bp_bits_[position >> 6] >> (position & 63)) & 1u) != 0; + } + + static std::uint64_t low_bits_mask(std::size_t count) { + if (count == 0) { + return 0; + } + if (count >= 64) { + return ~std::uint64_t{0}; + } + return (std::uint64_t{1} << count) - 1; + } + + std::int16_t prefix_excess_in_word(std::size_t word, + std::size_t count) const { + if (count == 0 || word >= word_count_) { + return 0; + } + const std::uint64_t bits = bp_bits_[word] & low_bits_mask(count); + const std::size_t ones = std::popcount(bits); + return static_cast(2 * static_cast(ones) - + static_cast(count)); + } + + std::int16_t depth_at(std::size_t position) const { + const std::size_t block = position >> 6; + const std::size_t offset = position & 63; + return static_cast(depth_blocks_[block].base_depth + + prefix_excess_in_word(block, offset)); + } + + void build_depth_blocks() { + const std::size_t depth_count = bit_count_ + 1; + std::int16_t current_depth = 0; + + for (std::size_t block = 0; block < depth_block_count_; ++block) { + const std::size_t block_begin = block * 64; + const std::size_t block_size = + std::min(64, depth_count - block_begin); + DepthBlockSummary summary; + summary.base_depth = current_depth; + summary.min_depth = current_depth; + summary.size = static_cast(block_size); + + for (std::size_t offset = 1; offset < block_size; ++offset) { + current_depth = static_cast( + current_depth + (bit(block_begin + offset - 1) ? 1 : -1)); + if (current_depth < summary.min_depth) { + summary.min_depth = current_depth; + summary.min_offset = static_cast(offset); + } + } + + depth_blocks_[block] = summary; + if (block_begin + block_size < depth_count) { + current_depth = static_cast( + current_depth + (bit(block_begin + block_size - 1) ? 1 : -1)); + } + } + } + + void build_zero_rank_prefix() { + zero_rank_prefix_[0] = 0; + for (std::size_t word = 0; word < word_count_; ++word) { + const std::size_t word_begin = word * 64; + const std::size_t word_bits = + std::min(64, bit_count_ - word_begin); + const std::uint64_t logical_bits = + bp_bits_[word] & low_bits_mask(word_bits); + const std::size_t zeros = word_bits - std::popcount(logical_bits); + zero_rank_prefix_[word + 1] = + static_cast(zero_rank_prefix_[word] + zeros); + } + } + + std::size_t depth_arg_min_scalar(std::size_t left, + std::size_t right) const { + const std::size_t depth_count = bit_count_ + 1; + if (left >= right || right > depth_count) { + return npos; + } + + std::size_t position = left; + std::int16_t best_depth = depth_at(position); + std::size_t best_position = position; + std::int16_t current_depth = best_depth; + ++position; + + while (position < right && (position & 63) != 0) { + current_depth = static_cast(current_depth + + (bit(position - 1) ? 1 : -1)); + if (current_depth < best_depth) { + best_depth = current_depth; + best_position = position; + } + ++position; + } + + while (position + 64 <= right) { + const DepthBlockSummary& summary = depth_blocks_[position >> 6]; + if (summary.min_depth < best_depth) { + best_depth = summary.min_depth; + best_position = position + summary.min_offset; + } + position += 64; + } + + if (position < right) { + current_depth = depth_at(position); + if (current_depth < best_depth) { + best_depth = current_depth; + best_position = position; + } + ++position; + } + while (position < right) { + current_depth = static_cast(current_depth + + (bit(position - 1) ? 1 : -1)); + if (current_depth < best_depth) { + best_depth = current_depth; + best_position = position; + } + ++position; + } + + return best_position; + } + + std::size_t depth_arg_min(std::size_t left, + std::size_t right, + bool force_scalar) const { + const std::size_t depth_count = bit_count_ + 1; + if (left >= right || right > depth_count) { + return npos; + } + if (force_scalar) { + return depth_arg_min_scalar(left, right); + } + + std::size_t position = left; + std::int16_t best_depth = depth_at(position); + std::size_t best_position = position; + + while (position < right) { + const std::size_t chunk_begin = (position / 128) * 128; + const std::size_t local_left = position - chunk_begin; + const std::size_t local_right = + std::min(right - 1, chunk_begin + 128) - chunk_begin; + + if (chunk_begin >= bit_count_) { + const std::int16_t candidate_depth = depth_at(chunk_begin); + if (candidate_depth < best_depth) { + best_depth = candidate_depth; + best_position = chunk_begin; + } + } else { + const std::size_t word = chunk_begin >> 6; + const ExcessResult candidate = + excess_min_128(bp_bits_.data() + word, local_left, local_right); + const std::int16_t candidate_depth = static_cast( + depth_at(chunk_begin) + candidate.min_excess); + const std::size_t candidate_position = chunk_begin + candidate.offset; + if (candidate_depth < best_depth) { + best_depth = candidate_depth; + best_position = candidate_position; + } + } + + position = chunk_begin + local_right + 1; + } + + return best_position; + } + + std::size_t rank0(std::size_t position) const { + position = std::min(position, bit_count_); + const std::size_t full_words = position >> 6; + std::size_t zeros = zero_rank_prefix_[full_words]; + const std::size_t tail_bits = position & 63; + if (tail_bits != 0) { + const std::uint64_t tail = + bp_bits_[full_words] & low_bits_mask(tail_bits); + zeros += tail_bits - std::popcount(tail); + } + return zeros; + } + + std::array bp_bits_{}; + std::array close_positions_{}; + std::array depth_blocks_{}; + std::array zero_rank_prefix_{}; + std::uint16_t entry_count_ = 0; + std::uint16_t bit_count_ = 0; + std::uint16_t word_count_ = 0; + std::uint16_t depth_block_count_ = 0; + }; + + class TopDepthSparseSelector { + public: + TopDepthSparseSelector() = default; + + void build(const LocalSelector& selector) { + const std::size_t depth_count = selector.depth_count(); + depth_count_ = depth_count; + depths_.assign(depth_count, 0); + if (depth_count == 0) { + log_count_ = 0; + sparse_positions_.clear(); + return; + } + + for (std::size_t position = 0; position < depth_count; ++position) { + depths_[position] = selector.depth_at_position(position); + } + + log_count_ = std::bit_width(depth_count); + sparse_positions_.assign(log_count_ * depth_count, 0); + for (std::size_t position = 0; position < depth_count; ++position) { + sparse_positions_[position] = static_cast(position); + } + + for (std::size_t level = 1; level < log_count_; ++level) { + const std::size_t half_span = std::size_t{1} << (level - 1); + const std::size_t span = half_span << 1; + if (span > depth_count) { + break; + } + const std::size_t previous_offset = (level - 1) * depth_count; + const std::size_t current_offset = level * depth_count; + for (std::size_t position = 0; position + span <= depth_count; + ++position) { + sparse_positions_[current_offset + position] = better_depth_position( + sparse_positions_[previous_offset + position], + sparse_positions_[previous_offset + position + half_span]); + } + } + } + + std::size_t arg_min(const LocalSelector& selector, + std::size_t slot_left, + std::size_t slot_right) const { + if (slot_left >= slot_right || slot_right > selector.entry_count()) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t first_close = selector.close_position(slot_left); + const std::size_t last_close = selector.close_position(slot_right - 1); + if (first_close > last_close) { + return npos; + } + + const std::size_t shifted_min = + depth_arg_min(first_close + 1, last_close + 2); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + + const std::size_t zero_rank = selector.rank0_at(shifted_min); + if (zero_rank == 0) { + return npos; + } + const std::size_t entry = zero_rank - 1; + return entry < selector.entry_count() ? entry : npos; + } + + private: + std::uint16_t better_depth_position(std::uint16_t left, + std::uint16_t right) const { + const std::int16_t left_depth = depths_[left]; + const std::int16_t right_depth = depths_[right]; + if (right_depth < left_depth) { + return right; + } + if (left_depth < right_depth) { + return left; + } + return std::min(left, right); + } + + std::size_t depth_arg_min(std::size_t left, std::size_t right) const { + if (left >= right || right > depth_count_) { + return npos; + } + const std::size_t length = right - left; + const std::size_t level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << level; + const std::size_t offset = level * depth_count_; + return better_depth_position(sparse_positions_[offset + left], + sparse_positions_[offset + right - span]); + } + + std::vector depths_; + std::vector sparse_positions_; + std::size_t depth_count_ = 0; + std::size_t log_count_ = 0; + }; + + static constexpr std::size_t log2_exact(std::size_t value) { + std::size_t shift = 0; + while (value > 1) { + value >>= 1; + ++shift; + } + return shift; + } + + static constexpr std::size_t kLeafShift = + kLeafSizePowerOfTwo ? log2_exact(LeafSize) : 0; + static constexpr std::size_t kFanoutShift = + kFanoutPowerOfTwo ? log2_exact(Fanout) : 0; + + struct alignas(64) NodeMetadata { + std::size_t value_begin = 0; + std::size_t value_end = 0; + std::size_t first_entry = 0; + Index subtree_min_position = invalid_index; + std::uint16_t entry_count = 0; + std::array + padding{}; + }; + + static_assert(sizeof(NodeMetadata) == 64); + + bool missing_position(std::size_t position) const { + if constexpr (kInvalidIndexEqualsNpos) { + return position == npos; + } else { + return position == npos || + position == static_cast(invalid_index); + } + } + + bool strictly_better_position(std::size_t left, std::size_t right) const { + if (missing_position(left)) { + return false; + } + if (missing_position(right)) { + return true; + } + return compare_(values_[left], values_[right]); + } + + std::size_t better_position(std::size_t left, std::size_t right) const { + if (missing_position(left)) { + return right; + } + if (missing_position(right)) { + return left; + } + if (compare_(values_[right], values_[left])) { + return right; + } + if (compare_(values_[left], values_[right])) { + return left; + } + return std::min(left, right); + } + + void build() { + nodes_.clear(); + selectors_.clear(); + top_depth_selectors_.clear(); + top_level_offsets_.clear(); + level_offsets_.clear(); + level_sizes_.clear(); + level_value_spans_.clear(); + top_level_begin_ = std::numeric_limits::max(); + if (values_.empty()) { + return; + } + if (values_.size() > static_cast(invalid_index)) { + throw std::length_error("NodeEulerBTreeRmq index type is too small"); + } + + initialize_layout((values_.size() + LeafSize - 1) / LeafSize); + const std::size_t leaf_count = level_sizes_[0]; + for (std::size_t leaf = 0; leaf < leaf_count; ++leaf) { + build_leaf(leaf); + } + + for (std::size_t child_level = 0; child_level + 1 < level_count(); + ++child_level) { + const std::size_t parent_count = level_sizes_[child_level + 1]; + for (std::size_t parent = 0; parent < parent_count; ++parent) { + build_internal_node(child_level, parent); + } + } + } + + void initialize_layout(std::size_t leaf_count) { + level_sizes_.push_back(leaf_count); + level_value_spans_.push_back(LeafSize); + while (level_sizes_.back() > 1) { + level_sizes_.push_back((level_sizes_.back() + Fanout - 1) / Fanout); + level_value_spans_.push_back(saturating_product( + level_value_spans_.back(), static_cast(Fanout))); + } + + level_offsets_.assign(level_sizes_.size(), 0); + std::size_t total_nodes = 0; + for (std::size_t level = level_sizes_.size(); level-- > 0;) { + level_offsets_[level] = total_nodes; + total_nodes += level_sizes_[level]; + } + + nodes_.resize(total_nodes); + selectors_.resize(total_nodes); + initialize_top_depth_selector_layout(); + } + + void initialize_top_depth_selector_layout() { + top_level_begin_ = std::numeric_limits::max(); + top_level_offsets_.assign(level_sizes_.size(), 0); + top_depth_selectors_.clear(); + if (level_sizes_.size() <= 1) { + return; + } + + const std::size_t root_level = level_sizes_.size() - 1; + top_level_begin_ = root_level > 1 ? root_level - 1 : 1; + + std::size_t top_node_count = 0; + for (std::size_t level = top_level_begin_; level < level_sizes_.size(); + ++level) { + top_level_offsets_[level] = top_node_count; + top_node_count += level_sizes_[level]; + } + top_depth_selectors_.resize(top_node_count); + } + + void build_leaf(std::size_t leaf) { + NodeMetadata& node = mutable_node_at(0, leaf); + LocalSelector& selector = mutable_selector_at(0, leaf); + node.first_entry = leaf * LeafSize; + node.value_begin = node.first_entry; + node.entry_count = static_cast( + std::min(LeafSize, values_.size() - node.first_entry)); + node.value_end = node.value_begin + node.entry_count; + selector.build(node.entry_count, [&](std::size_t left, std::size_t right) { + return compare_(values_[node.first_entry + left], + values_[node.first_entry + right]); + }); + + const std::size_t slot = selector.arg_min(0, node.entry_count); + node.subtree_min_position = static_cast(node.first_entry + slot); + } + + void build_internal_node(std::size_t child_level, std::size_t parent) { + const std::size_t parent_level = child_level + 1; + NodeMetadata& node = mutable_node_at(parent_level, parent); + LocalSelector& selector = mutable_selector_at(parent_level, parent); + node.first_entry = parent * Fanout; + node.entry_count = static_cast(std::min( + Fanout, level_sizes_[child_level] - node.first_entry)); + node.value_begin = node_at(child_level, node.first_entry).value_begin; + node.value_end = + node_at(child_level, node.first_entry + node.entry_count - 1).value_end; + selector.build(node.entry_count, [&](std::size_t left, std::size_t right) { + return strictly_better_position( + node_at(child_level, node.first_entry + left).subtree_min_position, + node_at(child_level, node.first_entry + right).subtree_min_position); + }); + + const std::size_t slot = selector.arg_min(0, node.entry_count); + node.subtree_min_position = + node_at(child_level, node.first_entry + slot).subtree_min_position; + if (is_top_internal_level(parent_level)) { + mutable_top_depth_selector_at(parent_level, parent).build(selector); + } + } + + static std::size_t saturating_product(std::size_t left, std::size_t right) { + if (left != 0 && right > std::numeric_limits::max() / left) { + return std::numeric_limits::max(); + } + return left * right; + } + + std::size_t leaf_range_min(const NodeMetadata& node, + const LocalSelector& selector, + std::size_t left, + std::size_t right, + bool allow_linear_scan) const { + if (allow_linear_scan && right - left <= kLeafLinearScanThreshold) { + return linear_range_min(left, right); + } + const std::size_t slot_left = left - node.value_begin; + const std::size_t slot_right = right - node.value_begin; + const std::size_t slot = selector.arg_min(slot_left, slot_right); + if (slot == npos) { + return npos; + } + return node.value_begin + slot; + } + + std::size_t linear_range_min(std::size_t left, std::size_t right) const { + if (left >= right) { + return npos; + } +#ifdef PIXIE_AVX2_SUPPORT + if constexpr (std::is_same_v && + std::is_same_v>) { + if (right - left >= kLeafAvx2ScanThreshold) { + return linear_range_min_i64_avx2(left, right); + } + } +#endif + std::size_t best = left; + for (std::size_t position = left + 1; position < right; ++position) { + if (compare_(values_[position], values_[best])) { + best = position; + } + } + return best; + } + +#ifdef PIXIE_AVX2_SUPPORT + std::size_t linear_range_min_i64_avx2(std::size_t left, + std::size_t right) const { + const std::int64_t* data = values_.data(); + std::size_t position = left; + + __m256i best_values = + _mm256_loadu_si256(reinterpret_cast(data + position)); + __m256i best_positions = _mm256_set_epi64x( + static_cast(position + 3), + static_cast(position + 2), + static_cast(position + 1), static_cast(position)); + position += 4; + + for (; position + 4 <= right; position += 4) { + const __m256i values = + _mm256_loadu_si256(reinterpret_cast(data + position)); + const __m256i positions = + _mm256_set_epi64x(static_cast(position + 3), + static_cast(position + 2), + static_cast(position + 1), + static_cast(position)); + const __m256i take_new = _mm256_cmpgt_epi64(best_values, values); + best_values = _mm256_blendv_epi8(best_values, values, take_new); + best_positions = _mm256_blendv_epi8(best_positions, positions, take_new); + } + + alignas(32) std::int64_t value_lanes[4]; + alignas(32) std::uint64_t position_lanes[4]; + _mm256_store_si256(reinterpret_cast<__m256i*>(value_lanes), best_values); + _mm256_store_si256(reinterpret_cast<__m256i*>(position_lanes), + best_positions); + + std::int64_t best_value = value_lanes[0]; + std::size_t best_position = static_cast(position_lanes[0]); + for (std::size_t lane = 1; lane < 4; ++lane) { + const std::size_t lane_position = + static_cast(position_lanes[lane]); + if (value_lanes[lane] < best_value || + (value_lanes[lane] == best_value && lane_position < best_position)) { + best_value = value_lanes[lane]; + best_position = lane_position; + } + } + + for (; position < right; ++position) { + if (data[position] < best_value) { + best_value = data[position]; + best_position = position; + } + } + return best_position; + } +#endif + + std::pair covering_node( + std::size_t left_leaf, + std::size_t right_leaf) const { + std::size_t level = 0; + std::size_t left_node = left_leaf; + std::size_t right_node = right_leaf; + while (left_node != right_node) { + ++level; + if constexpr (kFanoutPowerOfTwo) { + left_node >>= kFanoutShift; + right_node >>= kFanoutShift; + } else { + left_node /= Fanout; + right_node /= Fanout; + } + } + return {level, left_node}; + } + + std::size_t leaf_for_value(std::size_t position) const { + if constexpr (kLeafSizePowerOfTwo) { + return position >> kLeafShift; + } else { + return position / LeafSize; + } + } + + std::size_t child_for_value(std::size_t child_level, + std::size_t position) const { + if constexpr (kPowerOfTwoLayout) { + return position >> (kLeafShift + child_level * kFanoutShift); + } else { + return position / level_value_spans_[child_level]; + } + } + + bool contains_position(std::size_t left, + std::size_t right, + std::size_t position) const { + return !missing_position(position) && left <= position && position < right; + } + + std::size_t query_child_slots(std::size_t level, + std::size_t node_index, + std::size_t slot_left, + std::size_t slot_right, + std::size_t left, + std::size_t right, + bool allow_leaf_linear_scan) const { + if (slot_left >= slot_right) { + return npos; + } + + const NodeMetadata& node = node_at(level, node_index); + const std::size_t child_level = level - 1; + const std::size_t slot_count = slot_right - slot_left; + const std::size_t slot = + slot_count == 1 + ? slot_left + : selector_arg_min(level, node_index, slot_left, slot_right, + slot_count <= kInternalScalarSlotThreshold); + if (slot == npos) { + return npos; + } + + const std::size_t child_index = node.first_entry + slot; + const NodeMetadata& child = node_at(child_level, child_index); + if ((left <= child.value_begin && child.value_end <= right) || + contains_position(left, right, child.subtree_min_position)) { + return child.subtree_min_position; + } + + std::size_t answer = + query_node(child_level, child_index, std::max(left, child.value_begin), + std::min(right, child.value_end), allow_leaf_linear_scan); + answer = better_position( + answer, query_child_slots(level, node_index, slot_left, slot, left, + right, allow_leaf_linear_scan)); + answer = better_position( + answer, query_child_slots(level, node_index, slot + 1, slot_right, left, + right, allow_leaf_linear_scan)); + return answer; + } + + std::size_t query_node(std::size_t level, + std::size_t node_index, + std::size_t left, + std::size_t right, + bool allow_leaf_linear_scan) const { + if (left >= right) { + return npos; + } + const NodeMetadata& node = node_at(level, node_index); + if (left <= node.value_begin && node.value_end <= right) { + return node.subtree_min_position; + } + if (level == 0) { + return leaf_range_min(node, selector_at(level, node_index), left, right, + allow_leaf_linear_scan); + } + + const std::size_t child_level = level - 1; + const std::size_t left_child = child_for_value(child_level, left); + const std::size_t right_child = child_for_value(child_level, right - 1); + const std::size_t left_slot = left_child - node.first_entry; + const std::size_t right_slot = right_child - node.first_entry + 1; + return query_child_slots(level, node_index, left_slot, right_slot, left, + right, allow_leaf_linear_scan); + } + + std::size_t level_count() const { return level_offsets_.size(); } + + std::size_t flat_index(std::size_t level, std::size_t node_index) const { + return level_offsets_[level] + node_index; + } + + bool is_top_internal_level(std::size_t level) const { + return level > 0 && level >= top_level_begin_ && level < level_count(); + } + + std::size_t top_flat_index(std::size_t level, std::size_t node_index) const { + return top_level_offsets_[level] + node_index; + } + + std::size_t selector_arg_min(std::size_t level, + std::size_t node_index, + std::size_t slot_left, + std::size_t slot_right, + bool force_scalar) const { + const LocalSelector& selector = selector_at(level, node_index); + if (is_top_internal_level(level)) { + return top_depth_selector_at(level, node_index) + .arg_min(selector, slot_left, slot_right); + } + return selector.arg_min(slot_left, slot_right, force_scalar); + } + + const NodeMetadata& node_at(std::size_t level, std::size_t node_index) const { + return nodes_[flat_index(level, node_index)]; + } + + NodeMetadata& mutable_node_at(std::size_t level, std::size_t node_index) { + return nodes_[flat_index(level, node_index)]; + } + + const LocalSelector& selector_at(std::size_t level, + std::size_t node_index) const { + return selectors_[flat_index(level, node_index)]; + } + + LocalSelector& mutable_selector_at(std::size_t level, + std::size_t node_index) { + return selectors_[flat_index(level, node_index)]; + } + + const TopDepthSparseSelector& top_depth_selector_at( + std::size_t level, + std::size_t node_index) const { + return top_depth_selectors_[top_flat_index(level, node_index)]; + } + + TopDepthSparseSelector& mutable_top_depth_selector_at( + std::size_t level, + std::size_t node_index) { + return top_depth_selectors_[top_flat_index(level, node_index)]; + } + + std::span values_; + Compare compare_; + std::vector nodes_; + std::vector selectors_; + std::vector top_depth_selectors_; + std::vector level_offsets_; + std::vector top_level_offsets_; + std::vector level_sizes_; + std::vector level_value_spans_; + std::size_t top_level_begin_ = std::numeric_limits::max(); +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/rmq_base.h b/include/pixie/rmq/rmq_base.h index a188924..d0007ab 100644 --- a/include/pixie/rmq/rmq_base.h +++ b/include/pixie/rmq/rmq_base.h @@ -9,8 +9,8 @@ namespace pixie::rmq { * @brief CRTP facade for static range-minimum-query indexes. * * Implementations are non-owning indexes over an external random-access array. - * Queries use inclusive zero-based ranges and return the first position - * attaining the minimum. Invalid ranges return `npos`. + * Queries use half-open zero-based ranges `[left, right)` and return the first + * position attaining the minimum. Invalid or empty ranges return `npos`. */ template class RmqBase { @@ -35,13 +35,14 @@ class RmqBase { bool empty() const { return size() == 0; } /** - * @brief Return the first minimum position in [@p left, @p right]. + * @brief Return the first minimum position in [@p left, @p right). * - * @details The query range is inclusive. Ties are resolved by returning the - * smallest position attaining the minimum. Invalid ranges return `npos`. + * @details The query range is half-open. Ties are resolved by returning the + * smallest position attaining the minimum. Invalid or empty ranges return + * `npos`. * * @param left First position in the query range. - * @param right Last position in the query range. + * @param right One past the last position in the query range. * @return Zero-based position of the first range minimum, or `npos`. */ std::size_t arg_min(std::size_t left, std::size_t right) const { @@ -49,13 +50,13 @@ class RmqBase { } /** - * @brief Return the minimum value in [@p left, @p right]. + * @brief Return the minimum value in [@p left, @p right). * - * @details Invalid ranges return a default-constructed value. + * @details Invalid or empty ranges return a default-constructed value. * * @param left First position in the query range. - * @param right Last position in the query range. - * @return The minimum value in the inclusive range, or `Value{}`. + * @param right One past the last position in the query range. + * @return The minimum value in the half-open range, or `Value{}`. */ Value range_min(std::size_t left, std::size_t right) const { const std::size_t position = arg_min(left, right); diff --git a/include/pixie/rmq/rmq_one_interval_btree.h b/include/pixie/rmq/rmq_one_interval_btree.h new file mode 100644 index 0000000..0453b49 --- /dev/null +++ b/include/pixie/rmq/rmq_one_interval_btree.h @@ -0,0 +1,833 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief Interval B-tree backend for arrays with adjacent differences +/-1. + * + * @details The indexed depth sequence is represented by BP deltas: bit 1 means + * the next depth is current + 1, and bit 0 means current - 1. The structure + * keeps 128-depth-position lower blocks separate and builds a cache-aligned + * interval hierarchy above those blocks. Queries scan partial edge blocks with + * the existing 128-bit excess-min primitives and cover the aligned middle with + * B-tree summary nodes. + * + * @tparam Index Unsigned integer type used to validate representable + * positions. + * @tparam HighCacheLines Number of 64-byte cache lines assigned to one high + * summary node. + * @tparam LowFanout Number of lower-block summaries stored in one low node. + * @tparam UseBoundaryRecords Whether to precompute left-to-right and + * right-to-left block minimum record positions for cross-block edge queries. + */ +template +class OneIntervalBTreeRmq { + public: + static_assert(std::is_unsigned_v, + "OneIntervalBTreeRmq index type must be unsigned"); + static_assert(HighCacheLines > 0); + static_assert(LowFanout > 0); + + static constexpr std::size_t npos = std::numeric_limits::max(); + static constexpr Index invalid_index = std::numeric_limits::max(); + static constexpr std::size_t kBlockSize = 128; + static constexpr std::size_t kChunkSize = 128; + static constexpr std::size_t kBlockChunks = kBlockSize / kChunkSize; + static constexpr std::size_t kChunkWords = kChunkSize / 64; + static constexpr bool kUseBoundaryRecords = UseBoundaryRecords; + static constexpr std::size_t kCacheLineBytes = 64; + static constexpr std::size_t kLowFanout = LowFanout; + static constexpr std::size_t kHighFanout = + std::max(2, (512 * HighCacheLines) / (2 * 64)); + static constexpr std::size_t kMaxFanout = std::max(kLowFanout, kHighFanout); + + static_assert(kBlockSize % kChunkSize == 0); + static_assert(kChunkSize == 128); + static_assert(kMaxFanout <= 64); + static_assert( + kBlockSize * kLowFanout <= + static_cast(std::numeric_limits::max())); + + /** + * @brief Construct an empty +/-1 RMQ index. + */ + OneIntervalBTreeRmq() = default; + OneIntervalBTreeRmq(const OneIntervalBTreeRmq&) = default; + OneIntervalBTreeRmq(OneIntervalBTreeRmq&&) noexcept = default; + OneIntervalBTreeRmq& operator=(const OneIntervalBTreeRmq&) = default; + OneIntervalBTreeRmq& operator=(OneIntervalBTreeRmq&&) noexcept = default; + + /** + * @brief Build an interval B-tree over a BP-encoded depth-delta sequence. + * + * @details The bit span is not copied and must outlive this object. A + * sequence with @p depth_count depth positions requires + * @p depth_count - 1 delta bits. + * + * @param bits Packed delta bits in little-endian bit order within each word. + * @param depth_count Number of depth positions indexed by the RMQ. + * @throws std::length_error if @p Index cannot represent all positions. + * @throws std::invalid_argument if @p bits does not contain enough words. + */ + OneIntervalBTreeRmq(std::span bits, + std::size_t depth_count) + : input_bits_(bits), depth_count_(depth_count) { + build(); + } + + /** + * @brief Return the number of indexed depth positions. + */ + std::size_t size() const { return depth_count_; } + + /** + * @brief Whether the indexed depth sequence is empty. + */ + bool empty() const { return depth_count_ == 0; } + + /** + * @brief Return the first minimum depth position in [@p left, @p right). + * + * @details The query range is half-open over depth positions, not delta-bit + * positions. Empty or invalid ranges return `npos`. + */ + std::size_t arg_min(std::size_t left, std::size_t right) const { + if (left >= right || right > depth_count_) { + return npos; + } + + const std::size_t last = right - 1; + const std::size_t left_block = left / kBlockSize; + const std::size_t right_block = last / kBlockSize; + if (left_block == right_block) { + return scan_block_range_position(left_block, left % kBlockSize, + last % kBlockSize); + } + + Candidate best; + std::int64_t prefix = 0; + NodeRef best_node; + std::int64_t best_node_target = 0; + bool best_is_node = false; + + const auto consider_position = [&](std::size_t position, + std::int64_t value) { + if (position == npos) { + return; + } + if (value < best.value) { + best = {position, value}; + best_is_node = false; + } + }; + + const auto consider_node = [&](NodeRef node, std::int64_t target, + std::int64_t value) { + if (value < best.value) { + best = {npos, value}; + best_node = node; + best_node_target = target; + best_is_node = true; + } + }; + + const auto consider_excess_result = + [&](std::size_t block, ExcessResult result, std::int64_t value) { + if (result.offset >= block_size(block)) { + return; + } + consider_position(block * kBlockSize + result.offset, value); + }; + + const std::size_t left_offset = left % kBlockSize; + const std::size_t right_offset = last % kBlockSize; + ExcessResult fused_right_boundary; + bool has_fused_right_boundary = false; + + const std::size_t first_full_block = (left + kBlockSize - 1) / kBlockSize; + const std::size_t full_begin = + std::min(right, first_full_block * kBlockSize); + if constexpr (UseBoundaryRecords) { + if (left < full_begin) { + const ScanResult scan = + scan_suffix_boundary_record(left_block, left_offset); + consider_position(scan.position, scan.min_value); + prefix += scan.block_excess; + } + } else { + const bool use_fused_boundaries = left_offset > right_offset && + left_offset < kBlockSize - 1 && + right_offset > 0; + if (use_fused_boundaries) { + const auto left_bits = block_bits(left_block); + const auto right_bits = block_bits(right_block); + const ExcessBoundaryPairResult result = + excess_min_128_disjoint_suffix_prefix( + left_bits.data(), left_offset, right_bits.data(), right_offset); + const std::int64_t left_base = + block_prefix_excess(left_block, left_offset); + consider_excess_result(left_block, result.suffix, + result.suffix.min_excess - left_base); + prefix += block_prefix_excess(left_block, kBlockSize) - left_base; + fused_right_boundary = result.prefix; + has_fused_right_boundary = true; + } else if (left < full_begin) { + const ScanResult scan = scan_range(left, full_begin); + consider_position(scan.position, scan.min_value); + prefix += scan.block_excess; + } + } + + const std::size_t last_full_block_exclusive = right / kBlockSize; + const std::size_t middle_begin = full_begin; + const std::size_t middle_end = + std::max(middle_begin, last_full_block_exclusive * kBlockSize); + if (middle_begin < middle_end) { + Cover cover; + collect_cover(middle_begin, middle_end, cover); + for (std::size_t i = 0; i < cover.size; ++i) { + const NodeRef node = cover.nodes[i]; + const Summary summary = summary_at(node.level, node.index); + consider_node(node, summary.min_excess, prefix + summary.min_excess); + prefix += summary.block_excess; + } + } + + if (middle_end < right) { + if constexpr (UseBoundaryRecords) { + const ExcessResult result = + scan_prefix_boundary_record(right_block, right_offset); + consider_excess_result(right_block, result, prefix + result.min_excess); + } else { + if (has_fused_right_boundary) { + consider_excess_result(right_block, fused_right_boundary, + prefix + fused_right_boundary.min_excess); + } else { + const ScanResult scan = scan_range(middle_end, right); + consider_position(scan.position, prefix + scan.min_value); + } + } + } + + if (best_is_node) { + return descend_first_min(best_node.level, best_node.index, + best_node_target); + } + return best.position; + } + + private: + struct Summary { + std::uint64_t size_positions = 0; + std::int64_t block_excess = 0; + std::int64_t min_excess = 0; + }; + + struct BlockSummary { + Summary summary; + std::uint16_t min_offset = 0; + }; + + struct BoundaryRecords { + std::array prefix_records{}; + std::array suffix_records{}; + }; + + static_assert(sizeof(BoundaryRecords) == 4 * sizeof(std::uint64_t)); + + template + struct alignas(kCacheLineBytes) SummaryNode { + using ExcessType = Excess; + static constexpr std::size_t kFanout = Fanout; + + std::array prefix_excess{}; + std::array min_excess{}; + }; + + using LowNode = SummaryNode; + using HighNode = SummaryNode; + static_assert(alignof(LowNode) == kCacheLineBytes); + static_assert(alignof(HighNode) == kCacheLineBytes); + static_assert(sizeof(LowNode) % kCacheLineBytes == 0); + static_assert(sizeof(HighNode) % kCacheLineBytes == 0); + + struct Candidate { + std::size_t position = npos; + std::int64_t value = std::numeric_limits::max(); + }; + + struct ScanResult { + std::size_t position = npos; + std::int64_t min_value = std::numeric_limits::max(); + std::int64_t block_excess = 0; + }; + + struct NodeRef { + std::size_t level = 0; + std::size_t index = 0; + }; + + struct ChildSearchResult { + bool found = false; + std::size_t index = 0; + std::int64_t target = 0; + }; + + static constexpr std::size_t kMaxCoverItems = 512; + + struct Cover { + std::array nodes{}; + std::size_t size = 0; + + void push(NodeRef node) { + if (size < nodes.size()) { + nodes[size++] = node; + } + } + }; + + /** + * @brief Build lower block summaries and interval B-tree levels. + */ + void build() { + level_counts_.clear(); + low_levels_.clear(); + high_levels_.clear(); + block_summaries_.clear(); + boundary_records_.clear(); + top_summary_ = Summary{}; + block_count_ = 0; + + if (depth_count_ == 0) { + return; + } + if (depth_count_ > static_cast(invalid_index)) { + throw std::length_error("RMQ +/-1 interval B-tree index is too small"); + } + const std::size_t delta_count = depth_count_ - 1; + if (input_bits_.size() < (delta_count + 63) / 64) { + throw std::invalid_argument( + "RMQ +/-1 interval B-tree bit span is too small"); + } + + block_count_ = (depth_count_ + kBlockSize - 1) / kBlockSize; + block_summaries_.resize(block_count_); + if constexpr (UseBoundaryRecords) { + boundary_records_.resize(block_count_); + } + std::vector block_summaries(block_count_); + for (std::size_t block = 0; block < block_count_; ++block) { + block_summaries_[block] = summarize_block(block); + if constexpr (UseBoundaryRecords) { + boundary_records_[block] = build_boundary_records(block); + } + block_summaries[block] = block_summaries_[block].summary; + } + build_levels(block_summaries); + } + + /** + * @brief Build low and high summary levels from 128-position blocks. + */ + void build_levels(const std::vector& block_summaries) { + level_counts_.push_back(block_summaries.size()); + if (block_summaries.empty()) { + return; + } + + std::vector current = block_summaries; + current = build_parent_level(current, low_levels_.emplace_back()); + level_counts_.push_back(current.size()); + current = + build_parent_level(current, high_levels_.emplace_back()); + level_counts_.push_back(current.size()); + while (current.size() > 1) { + current = + build_parent_level(current, high_levels_.emplace_back()); + level_counts_.push_back(current.size()); + } + top_summary_ = current.front(); + } + + template + static std::vector build_parent_level(const std::vector& in, + std::vector& nodes) { + constexpr std::size_t fanout = Node::kFanout; + std::vector out((in.size() + fanout - 1) / fanout); + nodes.resize(out.size()); + for (std::size_t parent = 0; parent < out.size(); ++parent) { + const std::size_t begin = parent * fanout; + const std::size_t end = std::min(in.size(), begin + fanout); + Summary combined; + for (std::size_t i = begin; i < end; ++i) { + store_child_summary(nodes[parent], i - begin, combined.block_excess, + in[i]); + combined = append(combined, in[i]); + } + out[parent] = combined; + } + return out; + } + + template + static void store_child_summary(Node& node, + std::size_t slot, + std::int64_t prefix_excess, + const Summary& summary) { + node.prefix_excess[slot] = + static_cast( + prefix_excess + summary.block_excess); + node.min_excess[slot] = + static_cast( + summary.min_excess); + } + + BlockSummary summarize_block(std::size_t block) const { + const std::size_t size = block_size(block); + BlockSummary block_summary; + Summary& summary = block_summary.summary; + summary.size_positions = size; + if (size == 0) { + return block_summary; + } + const auto bits = block_bits(block); + const ExcessResult scan = excess_min_128(bits.data(), 0, size - 1); + summary.min_excess = scan.min_excess; + block_summary.min_offset = static_cast(scan.offset); + summary.block_excess = + block_prefix_excess(block, next_block_delta_count(block)); + return block_summary; + } + + BoundaryRecords build_boundary_records(std::size_t block) const { + const std::size_t size = block_size(block); + BoundaryRecords records; + if (size == 0) { + return records; + } + + const auto bits = block_bits(block); + std::array local_excess{}; + std::int16_t current = 0; + std::int16_t prefix_best = 0; + set_record_bit(records.prefix_records, 0); + + for (std::size_t offset = 1; offset < size; ++offset) { + current += local_bit(bits, offset - 1) ? 1 : -1; + local_excess[offset] = current; + if (current < prefix_best) { + prefix_best = current; + set_record_bit(records.prefix_records, offset); + } + } + + std::int16_t suffix_best = std::numeric_limits::max(); + for (std::size_t offset = size; offset-- > 0;) { + if (local_excess[offset] <= suffix_best) { + suffix_best = local_excess[offset]; + set_record_bit(records.suffix_records, offset); + } + } + return records; + } + + static Summary append(Summary left, const Summary& right) { + if (left.size_positions == 0) { + return right; + } + if (right.size_positions == 0) { + return left; + } + Summary result; + result.size_positions = left.size_positions + right.size_positions; + result.block_excess = left.block_excess + right.block_excess; + result.min_excess = + std::min(left.min_excess, left.block_excess + right.min_excess); + return result; + } + + std::size_t block_size(std::size_t block) const { + const std::size_t begin = block * kBlockSize; + return begin >= depth_count_ ? 0 + : std::min(kBlockSize, depth_count_ - begin); + } + + std::size_t next_block_delta_count(std::size_t block) const { + const std::size_t begin = block * kBlockSize; + if (begin >= depth_count_ - 1) { + return 0; + } + const std::size_t next_begin = begin + kBlockSize; + return std::min(next_begin, depth_count_ - 1) - begin; + } + + std::uint64_t word_or_zero(std::size_t word) const { + return word < input_bits_.size() ? input_bits_[word] : 0; + } + + std::array block_bits(std::size_t block) const { + const std::size_t first_word = (block * kBlockSize) / 64; + return {word_or_zero(first_word), word_or_zero(first_word + 1)}; + } + + std::array chunk_bits(std::size_t block, + std::size_t chunk) const { + const std::size_t first_word = + (block * kBlockSize + chunk * kChunkSize) / 64; + return {word_or_zero(first_word), word_or_zero(first_word + 1)}; + } + + std::int64_t block_prefix_excess(std::size_t block, + std::size_t prefix_offset) const { + prefix_offset = std::min(prefix_offset, next_block_delta_count(block)); + const auto bits = block_bits(block); + return prefix_excess_128(bits.data(), prefix_offset); + } + + static bool local_bit(const std::array& bits, + std::size_t offset) { + return ((bits[offset >> 6] >> (offset & 63)) & 1u) != 0; + } + + static void set_record_bit(std::array& records, + std::size_t offset) { + records[offset >> 6] |= std::uint64_t{1} << (offset & 63); + } + + ExcessResult scan_prefix_boundary_record(std::size_t block, + std::size_t right_offset) const { + const std::size_t size = block_size(block); + if (size == 0 || block >= boundary_records_.size()) { + return {}; + } + right_offset = std::min(right_offset, size - 1); + const BoundaryRecords& records = boundary_records_[block]; + + std::uint64_t candidates = records.prefix_records[right_offset >> 6] & + first_bits_mask((right_offset & 63) + 1); + if (candidates != 0) { + const std::size_t offset = (right_offset & ~std::size_t{63}) + + (63 - std::countl_zero(candidates)); + const auto bits = block_bits(block); + return {prefix_excess_128(bits.data(), offset), offset}; + } + if ((right_offset >> 6) != 0 && records.prefix_records[0] != 0) { + const std::size_t offset = + 63 - std::countl_zero(records.prefix_records[0]); + const auto bits = block_bits(block); + return {prefix_excess_128(bits.data(), offset), offset}; + } + return {}; + } + + ExcessResult scan_suffix_boundary_record_result( + std::size_t block, + std::size_t left_offset) const { + const std::size_t size = block_size(block); + if (size == 0 || block >= boundary_records_.size()) { + return {}; + } + left_offset = std::min(left_offset, size - 1); + const BoundaryRecords& records = boundary_records_[block]; + + const std::size_t word = left_offset >> 6; + std::uint64_t candidates = + records.suffix_records[word] & ~first_bits_mask(left_offset & 63); + if (candidates != 0) { + const std::size_t offset = + word * 64 + static_cast(std::countr_zero(candidates)); + const auto bits = block_bits(block); + return {prefix_excess_128(bits.data(), offset), offset}; + } + if (word == 0 && records.suffix_records[1] != 0) { + const std::size_t offset = + 64 + + static_cast(std::countr_zero(records.suffix_records[1])); + const auto bits = block_bits(block); + return {prefix_excess_128(bits.data(), offset), offset}; + } + return {}; + } + + ScanResult scan_suffix_boundary_record(std::size_t block, + std::size_t left_offset) const { + const std::size_t size = block_size(block); + if (size == 0) { + return {}; + } + left_offset = std::min(left_offset, size - 1); + const std::int64_t base = block_prefix_excess(block, left_offset); + + ScanResult scan; + const ExcessResult result = + scan_suffix_boundary_record_result(block, left_offset); + if (result.offset == npos || result.offset >= size) { + return scan; + } + + scan.position = block * kBlockSize + result.offset; + scan.min_value = result.min_excess - base; + scan.block_excess = block_prefix_excess(block, kBlockSize) - base; + return scan; + } + + ScanResult scan_range(std::size_t begin, std::size_t end) const { + const std::size_t block = begin / kBlockSize; + const std::size_t block_begin = block * kBlockSize; + return scan_block_range(block, begin - block_begin, end - block_begin - 1); + } + + std::size_t scan_block_range_position(std::size_t block, + std::size_t left_offset, + std::size_t right_offset) const { + const std::size_t size = block_size(block); + right_offset = std::min(right_offset, size - 1); + const auto bits = block_bits(block); + const ExcessResult result = + excess_min_128(bits.data(), left_offset, right_offset); + if (result.offset == npos || result.offset >= size) { + return npos; + } + return block * kBlockSize + result.offset; + } + + ScanResult scan_block_range(std::size_t block, + std::size_t left_offset, + std::size_t right_offset) const { + const std::size_t size = block_size(block); + right_offset = std::min(right_offset, size - 1); + const std::int64_t base = block_prefix_excess(block, left_offset); + + ScanResult scan; + const auto bits = block_bits(block); + const ExcessResult result = + excess_min_128(bits.data(), left_offset, right_offset); + if (result.offset == npos || result.offset >= size) { + return scan; + } + + scan.position = block * kBlockSize + result.offset; + scan.min_value = result.min_excess - base; + scan.block_excess = block_prefix_excess(block, right_offset + 1) - base; + return scan; + } + + std::size_t total_levels() const { return level_counts_.size(); } + + std::size_t level_count(std::size_t level) const { + return level < level_counts_.size() ? level_counts_[level] : 0; + } + + Summary summary_at(std::size_t level, std::size_t index) const { + if (level == 0) { + return block_summaries_[index].summary; + } + if (level + 1 >= total_levels()) { + return top_summary_; + } + const std::size_t parent_level = level + 1; + const std::size_t fanout = fanout_to_parent(level); + const std::size_t parent = index / fanout; + const std::size_t slot = index % fanout; + Summary summary; + summary.size_positions = node_size_positions(level, index); + if (parent_level == 1) { + const LowNode& node = low_levels_[0][parent]; + summary.block_excess = child_excess(node, slot); + summary.min_excess = node.min_excess[slot]; + } else { + const HighNode& node = high_levels_[parent_level - 2][parent]; + summary.block_excess = child_excess(node, slot); + summary.min_excess = node.min_excess[slot]; + } + return summary; + } + + static std::size_t fanout_to_parent(std::size_t level) { + return level == 0 ? kLowFanout : kHighFanout; + } + + static std::size_t mul_clamped(std::size_t lhs, std::size_t rhs) { + if (lhs != 0 && rhs > std::numeric_limits::max() / lhs) { + return std::numeric_limits::max(); + } + return lhs * rhs; + } + + static std::size_t level_span_positions(std::size_t level) { + std::size_t span = kBlockSize; + if (level >= 1) { + span = mul_clamped(span, kLowFanout); + } + if (level >= 2) { + for (std::size_t i = 2; i <= level; ++i) { + span = mul_clamped(span, kHighFanout); + } + } + return span; + } + + std::size_t node_start_position(std::size_t level, std::size_t index) const { + const std::size_t span = level_span_positions(level); + if (span != 0 && index > std::numeric_limits::max() / span) { + return depth_count_; + } + return std::min(depth_count_, index * span); + } + + std::size_t node_size_positions(std::size_t level, std::size_t index) const { + const std::size_t start = node_start_position(level, index); + if (start >= depth_count_) { + return 0; + } + return std::min(level_span_positions(level), depth_count_ - start); + } + + template + static std::int64_t prefix_excess_at(const Node& node, std::size_t slot) { + if (slot == 0) { + return 0; + } + return static_cast(node.prefix_excess[slot - 1]); + } + + template + static std::int64_t child_excess(const Node& node, std::size_t slot) { + return static_cast(node.prefix_excess[slot]) - + prefix_excess_at(node, slot); + } + + template + ChildSearchResult find_first_child_with_min(const Node& node, + std::size_t child_level, + std::size_t parent, + std::size_t child_count, + std::int64_t target) const { + for (std::size_t slot = 0; slot < child_count; ++slot) { + const std::int64_t candidate = + prefix_excess_at(node, slot) + + static_cast(node.min_excess[slot]); + if (candidate == target) { + return {true, parent * fanout_to_parent(child_level) + slot, + static_cast(node.min_excess[slot])}; + } + } + return {}; + } + + ChildSearchResult find_first_child_with_min(std::size_t child_level, + std::size_t parent, + std::size_t child_count, + std::int64_t target) const { + if (child_level == 0) { + return find_first_child_with_min(low_levels_[0][parent], child_level, + parent, child_count, target); + } + return find_first_child_with_min(high_levels_[child_level - 1][parent], + child_level, parent, child_count, target); + } + + void collect_cover(std::size_t begin, std::size_t end, Cover& out) const { + if (begin >= end || total_levels() == 0 || (begin % kBlockSize) != 0 || + (end % kBlockSize) != 0) { + return; + } + + Cover right_cover; + std::size_t level = 0; + std::size_t left = begin / kBlockSize; + std::size_t right = end / kBlockSize; + + while (left < right) { + if (!has_parent_level(level)) { + for (std::size_t index = left; index < right; ++index) { + out.push({level, index}); + } + break; + } + + const std::size_t fanout = fanout_to_parent(level); + while (left < right && (left % fanout) != 0) { + out.push({level, left}); + ++left; + } + while (left < right && (right % fanout) != 0) { + --right; + right_cover.push({level, right}); + } + left /= fanout; + right /= fanout; + ++level; + } + + while (right_cover.size > 0) { + out.push(right_cover.nodes[--right_cover.size]); + } + } + + bool has_parent_level(std::size_t level) const { + return level + 1 < total_levels() && level_count(level + 1) != 0; + } + + std::size_t descend_first_min(std::size_t level, + std::size_t index, + std::int64_t target) const { + while (level > 0) { + const std::size_t child_level = level - 1; + const std::size_t fanout = fanout_to_parent(child_level); + const std::size_t child_begin = index * fanout; + const std::size_t child_end = + std::min(level_count(child_level), child_begin + fanout); + const ChildSearchResult child = find_first_child_with_min( + child_level, index, child_end - child_begin, target); + if (!child.found) { + return npos; + } + index = child.index; + level = child_level; + target = child.target; + } + + if (index >= block_summaries_.size()) { + return npos; + } + const BlockSummary& block = block_summaries_[index]; + return block.summary.min_excess == target + ? index * kBlockSize + block.min_offset + : npos; + } + + std::span input_bits_; + std::size_t depth_count_ = 0; + std::size_t block_count_ = 0; + Summary top_summary_; + std::vector block_summaries_; + std::vector boundary_records_; + std::vector level_counts_; + std::vector> low_levels_; + std::vector> high_levels_; +}; + +template +using OneIntervalBTreeRmqBoundaryRecords = + OneIntervalBTreeRmq; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/segment_tree.h b/include/pixie/rmq/segment_tree.h index 3276c78..24ce332 100644 --- a/include/pixie/rmq/segment_tree.h +++ b/include/pixie/rmq/segment_tree.h @@ -67,34 +67,31 @@ class SegmentTree : public RmqBase, T> { T value_at_impl(std::size_t position) const { return values_[position]; } /** - * @brief Return the first minimum position in [@p left, @p right]. + * @brief Return the first minimum position in [@p left, @p right). * * @details Answers in O(log n) by walking the flat iterative segment tree. * Ties return the smaller position. * * @param left First position in the query range. - * @param right Last position in the query range. + * @param right One past the last position in the query range. * @return Zero-based position of the first range minimum, or `npos`. */ std::size_t arg_min_impl(std::size_t left, std::size_t right) const { - if (left > right || right >= values_.size()) { + if (left >= right || right > values_.size()) { return npos; } left += leaf_base_; right += leaf_base_; std::size_t answer = npos; - while (left <= right) { + while (left < right) { if ((left & 1u) != 0) { answer = better(answer, tree_[left]); ++left; } - if ((right & 1u) == 0) { - answer = better(answer, tree_[right]); - if (right == 0) { - break; - } + if ((right & 1u) != 0) { --right; + answer = better(answer, tree_[right]); } left >>= 1; right >>= 1; @@ -130,6 +127,26 @@ class SegmentTree : public RmqBase, T> { return std::min(left, right); } + /** + * @brief Choose the better child while building the tree. + * + * @details Child ranges are disjoint and ordered, so ties keep the left + * child. Missing tail leaves use `invalid_index`. + * + * @param left Candidate from the left child. + * @param right Candidate from the right child. + * @return Candidate for the parent node, or `invalid_index`. + */ + Index build_better(Index left, Index right) const { + if (left == invalid_index) { + return right; + } + if (right == invalid_index) { + return left; + } + return compare_(values_[right], values_[left]) ? right : left; + } + /** * @brief Build the flat iterative segment tree. * @@ -150,14 +167,16 @@ class SegmentTree : public RmqBase, T> { } leaf_base_ = std::bit_ceil(values_.size()); - tree_.assign(2 * leaf_base_, invalid_index); + tree_.clear(); + tree_.resize(2 * leaf_base_); for (std::size_t i = 0; i < values_.size(); ++i) { tree_[leaf_base_ + i] = static_cast(i); } + std::fill(tree_.begin() + leaf_base_ + values_.size(), tree_.end(), + invalid_index); for (std::size_t node = leaf_base_; node > 1;) { --node; - tree_[node] = - static_cast(better(tree_[node << 1], tree_[(node << 1) | 1])); + tree_[node] = build_better(tree_[node << 1], tree_[(node << 1) | 1]); } } diff --git a/include/pixie/rmq/sparse_table.h b/include/pixie/rmq/sparse_table.h index 4e963a7..90b0699 100644 --- a/include/pixie/rmq/sparse_table.h +++ b/include/pixie/rmq/sparse_table.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -7,6 +8,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -23,12 +27,18 @@ namespace pixie::rmq { * @tparam T Value type in the indexed array. * @tparam Compare Strict weak ordering used to choose minima. * @tparam Index Unsigned integer type used for stored positions. + * @tparam Alignment Byte alignment for each sparse-table level allocation. + * Use 0 to select the standard allocator. */ -template , class Index = std::size_t> -class SparseTable : public RmqBase, T> { +template , + class Index = std::size_t, + std::size_t Alignment = pixie::kAlignedStorageLineBytes> +class SparseTable + : public RmqBase, T> { public: static constexpr std::size_t npos = - RmqBase, T>::npos; + RmqBase, T>::npos; static constexpr Index invalid_index = std::numeric_limits::max(); /** @@ -67,24 +77,24 @@ class SparseTable : public RmqBase, T> { T value_at_impl(std::size_t position) const { return values_[position]; } /** - * @brief Return the first minimum position in [@p left, @p right]. + * @brief Return the first minimum position in [@p left, @p right). * * @details Answers in O(1) by comparing the two power-of-two ranges covering - * the inclusive query interval. Ties return the smaller position. + * the half-open query interval. Ties return the smaller position. * * @param left First position in the query range. - * @param right Last position in the query range. + * @param right One past the last position in the query range. * @return Zero-based position of the first range minimum, or `npos`. */ std::size_t arg_min_impl(std::size_t left, std::size_t right) const { - if (left > right || right >= values_.size()) { + if (left >= right || right > values_.size()) { return npos; } - const std::size_t length = right - left + 1; + const std::size_t length = right - left; const std::size_t level = std::bit_width(length) - 1; const std::size_t span = std::size_t{1} << level; const std::size_t first = table_[level][left]; - const std::size_t second = table_[level][right + 1 - span]; + const std::size_t second = table_[level][right - span]; return better(first, second); } @@ -115,6 +125,21 @@ class SparseTable : public RmqBase, T> { return std::min(left, right); } + /** + * @brief Choose the better candidate while building adjacent ranges. + * + * @details Build ranges are disjoint and ordered, so ties always keep the + * left candidate. This avoids the second strict comparison needed by the + * general query-time helper. + * + * @param left Candidate from the left half. + * @param right Candidate from the right half. + * @return First-minimum candidate for the combined range. + */ + Index build_better(Index left, Index right) const { + return compare_(values_[right], values_[left]) ? right : left; + } + /** * @brief Build all sparse-table levels over the indexed values. * @@ -132,10 +157,9 @@ class SparseTable : public RmqBase, T> { throw std::length_error("RMQ sparse table index type is too small"); } + table_.reserve(std::bit_width(values_.size())); table_.emplace_back(values_.size()); - for (std::size_t i = 0; i < values_.size(); ++i) { - table_[0][i] = static_cast(i); - } + std::iota(table_[0].begin(), table_[0].end(), Index{0}); for (std::size_t span = 2, half = 1; span <= values_.size(); half = span, span <<= 1) { @@ -143,14 +167,74 @@ class SparseTable : public RmqBase, T> { table_.emplace_back(values_.size() - span + 1); for (std::size_t i = 0; i < table_[level].size(); ++i) { table_[level][i] = static_cast( - better(table_[level - 1][i], table_[level - 1][i + half])); + build_better(table_[level - 1][i], table_[level - 1][i + half])); } } } + template + class AlignedAllocator { + public: + static_assert(AllocationAlignment >= alignof(Value)); + static_assert((AllocationAlignment & (AllocationAlignment - 1)) == 0); + + using value_type = Value; + + AlignedAllocator() = default; + + template + AlignedAllocator( + const AlignedAllocator&) noexcept {} + + [[nodiscard]] Value* allocate(std::size_t count) { + if (count == 0) { + return nullptr; + } + if (count > std::numeric_limits::max() / sizeof(Value)) { + throw std::bad_array_new_length(); + } + return static_cast(::operator new( + count * sizeof(Value), std::align_val_t{AllocationAlignment})); + } + + void deallocate(Value* pointer, std::size_t) noexcept { + ::operator delete(pointer, std::align_val_t{AllocationAlignment}); + } + + template + bool operator==( + const AlignedAllocator&) const noexcept { + return true; + } + + template + bool operator!=( + const AlignedAllocator&) const noexcept { + return false; + } + + template + struct rebind { + using other = AlignedAllocator; + }; + }; + + template + struct LevelAllocator { + using type = AlignedAllocator; + }; + + template + struct LevelAllocator { + using type = std::allocator; + }; + + using TableLevel = + std::vector::type>; + std::span values_; Compare compare_; - std::vector> table_; + std::vector table_; }; } // namespace pixie::rmq diff --git a/src/benchmarks/bench_rmq.cpp b/src/benchmarks/bench_rmq.cpp index abd4b46..d17e623 100644 --- a/src/benchmarks/bench_rmq.cpp +++ b/src/benchmarks/bench_rmq.cpp @@ -1,27 +1,56 @@ #include #include #include +#include +#include + +#ifdef __linux__ +#include +#include +#include +#include +#endif #include #include +#include #include +#include +#include #include #include +#include #include #include namespace { constexpr std::uint64_t kSeed = 42; -constexpr std::size_t kQueryCount = 32768; +constexpr std::size_t kQueryPoolBytes = 512 * 1024; +constexpr std::size_t kQueryCount = + kQueryPoolBytes / sizeof(std::pair); constexpr std::size_t kBpBlockSize = 128; +constexpr std::size_t kCacheLineBytes = 64; +constexpr std::size_t kSparseFootprintCycles = 16; +// Value-RMQ rows rotate between several arrays so query/build timings do not +// depend on one accidental global-minimum position. Large rows use fewer +// variants to keep multi-index query benchmarks within practical memory limits. +constexpr std::size_t kDefaultValueDatasetVariants = 4; +constexpr std::size_t kLargeValueDatasetVariants = 2; +constexpr std::size_t kValueDatasetVariantLargeSize = 1ull << 18; +constexpr std::size_t kMaxValueDatasetVariants = 16; +constexpr std::int64_t kValueDatasetGlobalMinimum = + std::numeric_limits::min() / 4; +constexpr const char* kValueDatasetVariantsEnv = "PIXIE_BENCH_VALUE_VARIANTS"; using Index = std::size_t; +static_assert(kQueryPoolBytes % sizeof(std::pair) == + 0); + struct Dataset { std::size_t size = 0; std::size_t max_width = 0; std::vector values; - std::vector> ranges; }; struct DepthDataset { @@ -32,6 +61,11 @@ struct DepthDataset { std::vector> ranges; }; +struct DeltaBitDataset { + std::size_t size = 0; + std::vector bits; +}; + struct BpQueryShape { std::size_t same_block = 0; std::size_t cross_block = 0; @@ -108,30 +142,430 @@ struct BenchBlockSummaryMinLess { } }; +class RandomQueryGenerator { + public: + RandomQueryGenerator(std::size_t size, + std::size_t max_width, + std::uint64_t seed) + : size_(size), rng_state_(seed), pool_(kQueryCount) { + std::mt19937_64 rng(seed); + const std::size_t width_limit = std::min(max_width, size_); + std::uniform_int_distribution width_dist(1, width_limit); + for (auto& [left, width] : pool_) { + width = width_dist(rng); + std::uniform_int_distribution left_dist(0, size_ - width); + left = left_dist(rng); + } + shift_ = next_shift(); + } + + std::pair get_random_query() { + if (index_ == pool_.size()) { + index_ = 0; + shift_ = next_shift(); + } + const std::size_t query_number = query_number_++; + const auto [left, width] = pool_[index_++]; + const std::size_t shifted_left = + shifted_left_position(left, width, query_number); + return {shifted_left, shifted_left + width}; + } + + private: + std::uint64_t splitmix64() { + std::uint64_t z = (rng_state_ += 0x9E3779B97F4A7C15ull); + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); + } + + std::size_t next_shift() { return static_cast(splitmix64()); } + + std::size_t shifted_left_position(std::size_t left, + std::size_t width, + std::size_t query_number) const { + const std::size_t valid_left_count = size_ - width + 1; + const std::size_t offset = + (shift_ % valid_left_count + query_number % valid_left_count) % + valid_left_count; + return (left + offset) % valid_left_count; + } + + std::size_t size_ = 0; + std::uint64_t rng_state_ = 0; + std::vector> pool_; + std::size_t index_ = 0; + std::size_t query_number_ = 0; + std::size_t shift_ = 0; +}; + +class PerfCounterGroup { + public: + struct Values { + std::uint64_t cycles = 0; + std::uint64_t instructions = 0; + std::uint64_t cache_misses = 0; + }; + + PerfCounterGroup() { +#ifdef __linux__ + if (!enabled()) { + return; + } + + leader_fd_ = open_hardware_event(PERF_COUNT_HW_CPU_CYCLES, -1, true); + if (leader_fd_ < 0) { + return; + } + instructions_fd_ = + open_hardware_event(PERF_COUNT_HW_INSTRUCTIONS, leader_fd_, false); + cache_misses_fd_ = + open_hardware_event(PERF_COUNT_HW_CACHE_MISSES, leader_fd_, false); + if (instructions_fd_ < 0 || cache_misses_fd_ < 0) { + close_all(); + return; + } + + ioctl(leader_fd_, PERF_EVENT_IOC_RESET, PERF_IOC_FLAG_GROUP); + ioctl(leader_fd_, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP); + active_ = true; +#endif + } + + PerfCounterGroup(const PerfCounterGroup&) = delete; + PerfCounterGroup& operator=(const PerfCounterGroup&) = delete; + + ~PerfCounterGroup() { close_all(); } + + bool stop(Values* values) { +#ifdef __linux__ + if (!active_) { + return false; + } + ioctl(leader_fd_, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP); + active_ = false; + + struct ReadGroup { + std::uint64_t count = 0; + std::uint64_t values[3] = {}; + }; + ReadGroup group; + const ssize_t bytes = read(leader_fd_, &group, sizeof(group)); + if (bytes != static_cast(sizeof(group)) || group.count != 3) { + close_all(); + return false; + } + + values->cycles = group.values[0]; + values->instructions = group.values[1]; + values->cache_misses = group.values[2]; + close_all(); + return true; +#else + (void)values; + return false; +#endif + } + + private: + static bool enabled() { + const char* value = std::getenv("PIXIE_BENCH_PERF_COUNTERS"); + return value != nullptr && value[0] != '\0' && + !(value[0] == '0' && value[1] == '\0'); + } + +#ifdef __linux__ + static int open_hardware_event(std::uint64_t config, + int group_fd, + bool disabled) { + perf_event_attr attr = {}; + attr.type = PERF_TYPE_HARDWARE; + attr.size = sizeof(attr); + attr.config = config; + attr.disabled = disabled ? 1 : 0; + attr.exclude_kernel = 1; + attr.exclude_hv = 1; + attr.read_format = PERF_FORMAT_GROUP; + return static_cast( + syscall(__NR_perf_event_open, &attr, 0, -1, group_fd, 0)); + } + + void close_fd(int* fd) { + if (*fd >= 0) { + close(*fd); + *fd = -1; + } + } + + void close_all() { + if (active_ && leader_fd_ >= 0) { + ioctl(leader_fd_, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP); + active_ = false; + } + close_fd(&cache_misses_fd_); + close_fd(&instructions_fd_); + close_fd(&leader_fd_); + } + + int leader_fd_ = -1; + int instructions_fd_ = -1; + int cache_misses_fd_ = -1; + bool active_ = false; +#else + void close_all() {} +#endif +}; + +void set_perf_counters(benchmark::State& state, + const PerfCounterGroup::Values& values) { + const double iterations = static_cast(state.iterations()); + if (iterations == 0.0) { + return; + } + state.counters["cycles_per_query"] = + static_cast(values.cycles) / iterations; + state.counters["instructions_per_query"] = + static_cast(values.instructions) / iterations; + state.counters["ipc"] = values.cycles == 0 + ? 0.0 + : static_cast(values.instructions) / + static_cast(values.cycles); + state.counters["cache_misses_per_query"] = + static_cast(values.cache_misses) / iterations; +} + Dataset make_dataset(std::size_t size, std::size_t max_width) { Dataset dataset; dataset.size = size; dataset.max_width = max_width; dataset.values.resize(size); - dataset.ranges.resize(kQueryCount); std::mt19937_64 rng(kSeed ^ (size * 0x9E3779B185EBCA87ull) ^ (max_width * 0xBF58476D1CE4E5B9ull)); std::uniform_int_distribution value_dist(-1'000'000, 1'000'000); std::generate(dataset.values.begin(), dataset.values.end(), [&] { return value_dist(rng); }); + return dataset; +} - std::uniform_int_distribution left_dist(0, size - 1); - for (auto& [left, right] : dataset.ranges) { - left = left_dist(rng); - const std::size_t available = size - left; - const std::size_t width_limit = std::min(max_width, available); - std::uniform_int_distribution width_dist(1, width_limit); - right = left + width_dist(rng) - 1; +std::size_t value_dataset_variant_count(std::size_t size) { + if (const char* value = std::getenv(kValueDatasetVariantsEnv); + value != nullptr && value[0] != '\0') { + char* end = nullptr; + const unsigned long long parsed = std::strtoull(value, &end, 10); + if (end != value && *end == '\0' && parsed != 0) { + return std::min(static_cast(parsed), + kMaxValueDatasetVariants); + } + } + return size > kValueDatasetVariantLargeSize ? kLargeValueDatasetVariants + : kDefaultValueDatasetVariants; +} + +std::size_t value_dataset_global_min_position(std::size_t size, + std::size_t variant, + std::size_t variant_count) { + return ((2 * variant + 1) * size) / (2 * variant_count); +} + +std::vector make_value_dataset_variants(std::size_t size, + std::size_t max_width, + std::size_t variant_count) { + variant_count = std::max( + 1, std::min(variant_count, std::max(size, 1))); + + std::vector base_values(size); + std::mt19937_64 rng(kSeed ^ (size * 0x9E3779B185EBCA87ull)); + std::uniform_int_distribution value_dist(-1'000'000, 1'000'000); + std::generate(base_values.begin(), base_values.end(), + [&] { return value_dist(rng); }); + + std::vector variants; + variants.reserve(variant_count); + for (std::size_t variant = 0; variant < variant_count; ++variant) { + Dataset dataset; + dataset.size = size; + dataset.max_width = max_width; + dataset.values = base_values; + if (!dataset.values.empty()) { + dataset.values[value_dataset_global_min_position( + size, variant, variant_count)] = kValueDatasetGlobalMinimum; + } + variants.push_back(std::move(dataset)); + } + return variants; +} + +DeltaBitDataset make_delta_bit_dataset(std::size_t size) { + DeltaBitDataset dataset; + dataset.size = size; + if (size == 0) { + return dataset; + } + dataset.bits.assign((size - 1 + 63) / 64, 0); + + std::mt19937_64 rng(kSeed ^ (size * 0xD6E8FEB86659FD93ull)); + for (std::size_t i = 0; i + 1 < size; ++i) { + if ((rng() & 1u) != 0) { + dataset.bits[i >> 6] |= std::uint64_t{1} << (i & 63); + } } return dataset; } +struct SparseTableFootprintStats { + std::size_t query_count = 0; + std::size_t max_observed_width = 0; + std::size_t over_max_width = 0; + std::uint64_t total_width = 0; + std::uint64_t total_level = 0; + std::uint64_t same_table_line = 0; + std::uint64_t same_value_line = 0; + std::array level_counts = {}; + std::vector table_lines; + std::vector value_lines; + + void finalize() { + sort_unique(table_lines); + sort_unique(value_lines); + } + + double avg_width() const { + return query_count == 0 ? 0.0 + : static_cast(total_width) / query_count; + } + + double avg_level() const { + return query_count == 0 ? 0.0 + : static_cast(total_level) / query_count; + } + + double ratio(std::uint64_t count) const { + return query_count == 0 ? 0.0 : static_cast(count) / query_count; + } + + std::size_t percentile_level(std::uint64_t numerator, + std::uint64_t denominator) const { + if (query_count == 0) { + return 0; + } + const std::uint64_t target = + (static_cast(query_count) * numerator + denominator - + 1) / + denominator; + std::uint64_t cumulative = 0; + for (std::size_t level = 0; level < level_counts.size(); ++level) { + cumulative += level_counts[level]; + if (cumulative >= target) { + return level; + } + } + return level_counts.size() - 1; + } + + private: + static void sort_unique(std::vector& values) { + std::sort(values.begin(), values.end()); + values.erase(std::unique(values.begin(), values.end()), values.end()); + } +}; + +SparseTableFootprintStats compute_sparse_table_footprint( + const pixie::rmq::SparseTable, Index>& + rmq, + std::size_t size, + std::size_t max_width) { + constexpr std::size_t kIndexEntriesPerLine = kCacheLineBytes / sizeof(Index); + constexpr std::size_t kValueEntriesPerLine = + kCacheLineBytes / sizeof(std::int64_t); + + SparseTableFootprintStats stats; + stats.query_count = kSparseFootprintCycles * kQueryCount; + stats.table_lines.reserve(stats.query_count * 2); + stats.value_lines.reserve(stats.query_count * 2); + + RandomQueryGenerator queries(size, max_width, + kSeed ^ (size * 0xC2B2AE3D27D4EB4Full) ^ + (max_width * 0x165667B19E3779F9ull)); + + for (std::size_t i = 0; i < stats.query_count; ++i) { + const auto [left, right] = queries.get_random_query(); + const std::size_t width = right - left; + const std::size_t level = std::bit_width(width) - 1; + const std::size_t span = std::size_t{1} << level; + const std::size_t first_table_position = left; + const std::size_t second_table_position = right - span; + const std::uint64_t first_table_line = + (static_cast(level) << 56) | + (first_table_position / kIndexEntriesPerLine); + const std::uint64_t second_table_line = + (static_cast(level) << 56) | + (second_table_position / kIndexEntriesPerLine); + + const std::size_t first_value_position = + rmq.arg_min(first_table_position, first_table_position + span); + const std::size_t second_value_position = + rmq.arg_min(second_table_position, second_table_position + span); + const std::uint64_t first_value_line = + first_value_position / kValueEntriesPerLine; + const std::uint64_t second_value_line = + second_value_position / kValueEntriesPerLine; + + stats.total_width += width; + stats.max_observed_width = std::max(stats.max_observed_width, width); + stats.over_max_width += width > max_width ? 1 : 0; + stats.total_level += level; + ++stats.level_counts[level]; + stats.same_table_line += first_table_line == second_table_line ? 1 : 0; + stats.same_value_line += first_value_line == second_value_line ? 1 : 0; + stats.table_lines.push_back(first_table_line); + stats.table_lines.push_back(second_table_line); + stats.value_lines.push_back(first_value_line); + stats.value_lines.push_back(second_value_line); + } + + stats.finalize(); + return stats; +} + +void set_sparse_table_footprint_counters( + benchmark::State& state, + const SparseTableFootprintStats& stats) { + const double query_count = static_cast(stats.query_count); + state.counters["sample_queries"] = query_count; + state.counters["sample_cycles"] = static_cast(kSparseFootprintCycles); + state.counters["avg_width"] = stats.avg_width(); + state.counters["max_observed_width"] = + static_cast(stats.max_observed_width); + state.counters["over_max_width_ratio"] = stats.ratio(stats.over_max_width); + state.counters["avg_level"] = stats.avg_level(); + state.counters["level_p50"] = + static_cast(stats.percentile_level(50, 100)); + state.counters["level_p90"] = + static_cast(stats.percentile_level(90, 100)); + state.counters["level_p99"] = + static_cast(stats.percentile_level(99, 100)); + state.counters["table_distinct_lines_per_query"] = + 2.0 - stats.ratio(stats.same_table_line); + state.counters["value_distinct_lines_per_query"] = + 2.0 - stats.ratio(stats.same_value_line); + state.counters["table_working_set_lines_per_query"] = + static_cast(stats.table_lines.size()) / query_count; + state.counters["value_working_set_lines_per_query"] = + static_cast(stats.value_lines.size()) / query_count; + state.counters["total_working_set_lines_per_query"] = + static_cast(stats.table_lines.size() + stats.value_lines.size()) / + query_count; + + for (std::size_t level = 0; level < stats.level_counts.size(); ++level) { + if (stats.level_counts[level] == 0) { + continue; + } + state.counters["level_" + std::to_string(level) + "_ratio"] = + stats.ratio(stats.level_counts[level]); + } +} + DepthDataset make_depth_dataset(std::size_t size, std::size_t max_width) { DepthDataset dataset; dataset.size = size; @@ -151,13 +585,11 @@ DepthDataset make_depth_dataset(std::size_t size, std::size_t max_width) { } } - std::uniform_int_distribution left_dist(0, size - 1); - for (auto& [left, right] : dataset.ranges) { - left = left_dist(rng); - const std::size_t available = size - left; - const std::size_t width_limit = std::min(max_width, available); - std::uniform_int_distribution width_dist(1, width_limit); - right = left + width_dist(rng) - 1; + RandomQueryGenerator queries(size, max_width, + kSeed ^ (size * 0xA24BAED4963EE407ull) ^ + (max_width * 0x9FB21C651E98DF25ull)); + for (auto& range : dataset.ranges) { + range = queries.get_random_query(); } return dataset; } @@ -182,30 +614,36 @@ const std::uint64_t* bp_block_bits_direct(const DepthDataset& dataset, return dataset.bits.data() + block * (kBpBlockSize / 64); } -BpQueryShape compute_bp_query_shape(const DepthDataset& dataset) { +BpQueryShape compute_bp_query_shape(const DepthDataset& dataset, + std::size_t block_size) { BpQueryShape shape; for (const auto [left, right] : dataset.ranges) { - const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = right / kBpBlockSize; + const std::size_t last = right - 1; + const std::size_t left_block = left / block_size; + const std::size_t right_block = last / block_size; if (left_block == right_block) { ++shape.same_block; ++shape.excess_calls; - shape.same_block_width += right - left + 1; + shape.same_block_width += right - left; continue; } ++shape.cross_block; - const std::size_t left_offset = left % kBpBlockSize; - const std::size_t right_offset = right % kBpBlockSize; - const std::size_t left_width = kBpBlockSize - left_offset; + const std::size_t left_offset = left % block_size; + const std::size_t right_offset = last % block_size; + const std::size_t left_width = block_size - left_offset; const std::size_t right_width = right_offset + 1; if (left_offset > right_offset) { ++shape.disjoint_boundary; - if ((kBpBlockSize - 1 - left_offset) + right_offset >= 32) { + if (left_offset < block_size - 1 && right_offset > 0) { ++shape.fused_boundary; + ++shape.excess_calls; + } else { + shape.excess_calls += 2; } + } else { + shape.excess_calls += 2; } - shape.excess_calls += 2; shape.left_boundary_width += left_width; shape.right_boundary_width += right_width; if (left_block + 1 < right_block) { @@ -217,7 +655,8 @@ BpQueryShape compute_bp_query_shape(const DepthDataset& dataset) { void set_depth_counters(benchmark::State& state, const DepthDataset& dataset, - bool include_shape) { + bool include_shape, + std::size_t block_size = kBpBlockSize) { state.counters["N"] = static_cast(dataset.size); state.counters["max_width"] = static_cast(dataset.max_width); state.counters["index_bytes"] = static_cast(sizeof(Index)); @@ -226,7 +665,7 @@ void set_depth_counters(benchmark::State& state, return; } - const BpQueryShape shape = compute_bp_query_shape(dataset); + const BpQueryShape shape = compute_bp_query_shape(dataset, block_size); const double query_count = static_cast(dataset.ranges.size()); const double cross_count = static_cast(shape.cross_block); const double same_count = static_cast(shape.same_block); @@ -238,7 +677,7 @@ void set_depth_counters(benchmark::State& state, static_cast(shape.middle_block) / query_count; state.counters["disjoint_boundary_ratio"] = static_cast(shape.disjoint_boundary) / query_count; - state.counters["fused_boundary_eligible_ratio"] = + state.counters["fused_boundary_ratio"] = static_cast(shape.fused_boundary) / query_count; state.counters["excess_calls_per_query"] = static_cast(shape.excess_calls) / query_count; @@ -260,10 +699,11 @@ std::vector make_same_block_ranges(const DepthDataset& dataset) { std::vector out; out.reserve(dataset.ranges.size()); for (const auto [left, right] : dataset.ranges) { + const std::size_t last = right - 1; const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = right / kBpBlockSize; + const std::size_t right_block = last / kBpBlockSize; if (left_block == right_block) { - out.push_back({left_block, left % kBpBlockSize, right % kBpBlockSize}); + out.push_back({left_block, left % kBpBlockSize, last % kBpBlockSize}); } } return out; @@ -273,8 +713,9 @@ std::vector make_left_boundary_ranges(const DepthDataset& dataset) { std::vector out; out.reserve(dataset.ranges.size()); for (const auto [left, right] : dataset.ranges) { + const std::size_t last = right - 1; const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = right / kBpBlockSize; + const std::size_t right_block = last / kBpBlockSize; if (left_block != right_block) { out.push_back({left_block, left % kBpBlockSize, bp_block_size(dataset, left_block) - 1}); @@ -288,10 +729,11 @@ std::vector make_right_boundary_ranges( std::vector out; out.reserve(dataset.ranges.size()); for (const auto [left, right] : dataset.ranges) { + const std::size_t last = right - 1; const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = right / kBpBlockSize; + const std::size_t right_block = last / kBpBlockSize; if (left_block != right_block) { - out.push_back({right_block, 0, right % kBpBlockSize}); + out.push_back({right_block, 0, last % kBpBlockSize}); } } return out; @@ -302,12 +744,13 @@ std::vector> make_boundary_pairs( std::vector> out; out.reserve(dataset.ranges.size()); for (const auto [left, right] : dataset.ranges) { + const std::size_t last = right - 1; const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = right / kBpBlockSize; + const std::size_t right_block = last / kBpBlockSize; if (left_block != right_block) { out.push_back({{left_block, left % kBpBlockSize, bp_block_size(dataset, left_block) - 1}, - {right_block, 0, right % kBpBlockSize}}); + {right_block, 0, last % kBpBlockSize}}); } } return out; @@ -318,10 +761,11 @@ std::vector> make_disjoint_boundary_pairs( std::vector> out; out.reserve(dataset.ranges.size()); for (const auto [left, right] : dataset.ranges) { + const std::size_t last = right - 1; const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = right / kBpBlockSize; + const std::size_t right_block = last / kBpBlockSize; const std::size_t left_offset = left % kBpBlockSize; - const std::size_t right_offset = right % kBpBlockSize; + const std::size_t right_offset = last % kBpBlockSize; if (left_block != right_block && left_offset > right_offset) { out.push_back( {{left_block, left_offset, bp_block_size(dataset, left_block) - 1}, @@ -335,10 +779,11 @@ std::vector make_macro_ranges(const DepthDataset& dataset) { std::vector out; out.reserve(dataset.ranges.size()); for (const auto [left, right] : dataset.ranges) { + const std::size_t last = right - 1; const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = right / kBpBlockSize; + const std::size_t right_block = last / kBpBlockSize; if (left_block + 1 < right_block) { - out.push_back({left_block + 1, right_block - 1}); + out.push_back({left_block + 1, right_block}); } } return out; @@ -367,20 +812,111 @@ template void run_queries(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); const std::size_t max_width = static_cast(state.range(1)); - const Dataset dataset = make_dataset(size, max_width); - const Rmq rmq(std::span(dataset.values)); + const std::vector datasets = make_value_dataset_variants( + size, max_width, value_dataset_variant_count(size)); + std::vector rmqs; + rmqs.reserve(datasets.size()); + for (const Dataset& dataset : datasets) { + rmqs.emplace_back(std::span(dataset.values)); + } - std::size_t query_index = 0; + const std::uint64_t query_seed = kSeed ^ (size * 0xC2B2AE3D27D4EB4Full) ^ + (max_width * 0x165667B19E3779F9ull); + std::vector queries; + queries.reserve(datasets.size()); + for (std::size_t i = 0; i < datasets.size(); ++i) { + queries.emplace_back(size, max_width, query_seed); + } + + std::size_t variant = 0; + PerfCounterGroup perf_counters; for (auto _ : state) { - const auto [left, right] = - dataset.ranges[query_index++ % dataset.ranges.size()]; - std::size_t result = rmq.arg_min(left, right); + const auto [left, right] = queries[variant].get_random_query(); + std::size_t result = rmqs[variant].arg_min(left, right); benchmark::DoNotOptimize(result); + ++variant; + if (variant == rmqs.size()) { + variant = 0; + } + } + PerfCounterGroup::Values perf_values; + if (perf_counters.stop(&perf_values)) { + set_perf_counters(state, perf_values); } state.counters["N"] = static_cast(size); state.counters["max_width"] = static_cast(max_width); state.counters["index_bytes"] = static_cast(sizeof(Index)); + state.counters["value_arrays"] = static_cast(datasets.size()); +} + +void set_build_counters(benchmark::State& state, + std::size_t size, + std::size_t input_bytes) { + state.counters["N"] = static_cast(size); + state.counters["input_bytes"] = static_cast(input_bytes); + state.counters["index_bytes"] = static_cast(sizeof(Index)); + state.SetItemsProcessed(static_cast(state.iterations()) * + static_cast(size)); +} + +template +void run_value_rmq_build(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const std::vector datasets = make_value_dataset_variants( + size, size, value_dataset_variant_count(size)); + + std::size_t variant = 0; + for (auto _ : state) { + const Dataset& dataset = datasets[variant]; + Rmq rmq(std::span(dataset.values)); + std::size_t built_size = rmq.size(); + benchmark::DoNotOptimize(built_size); + benchmark::ClobberMemory(); + ++variant; + if (variant == datasets.size()) { + variant = 0; + } + } + + set_build_counters(state, size, + datasets.front().values.size() * sizeof(std::int64_t)); + state.counters["value_arrays"] = static_cast(datasets.size()); +} + +template +void run_bp_rmq_build(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const DeltaBitDataset dataset = make_delta_bit_dataset(size); + + for (auto _ : state) { + Rmq rmq(std::span(dataset.bits), dataset.size); + std::size_t built_size = rmq.size(); + benchmark::DoNotOptimize(built_size); + benchmark::ClobberMemory(); + } + + set_build_counters(state, size, dataset.bits.size() * sizeof(std::uint64_t)); +} + +void run_sparse_table_footprint(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const std::size_t max_width = static_cast(state.range(1)); + const Dataset dataset = make_dataset(size, max_width); + const pixie::rmq::SparseTable, Index> + rmq(std::span(dataset.values)); + const SparseTableFootprintStats stats = + compute_sparse_table_footprint(rmq, size, max_width); + + for (auto _ : state) { + std::size_t query_count = stats.query_count; + benchmark::DoNotOptimize(query_count); + } + + state.counters["N"] = static_cast(size); + state.counters["max_width"] = static_cast(max_width); + state.counters["index_bytes"] = static_cast(sizeof(Index)); + set_sparse_table_footprint_counters(state, stats); } void run_bp_boundary_stack( @@ -543,30 +1079,86 @@ void run_bp_macro_only(benchmark::State& state) { state.counters["diagnostic_ranges"] = static_cast(ranges.size()); } -template +template void run_depth_queries(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); const std::size_t max_width = static_cast(state.range(1)); const DepthDataset dataset = make_depth_dataset(size, max_width); const Rmq rmq(std::span(dataset.bits), dataset.depths.size()); + RandomQueryGenerator queries(size, max_width, + kSeed ^ (size * 0xD1B54A32D192ED03ull) ^ + (max_width * 0x94D049BB133111EBull)); - std::size_t query_index = 0; + PerfCounterGroup perf_counters; for (auto _ : state) { - const auto [left, right] = - dataset.ranges[query_index++ % dataset.ranges.size()]; + const auto [left, right] = queries.get_random_query(); std::size_t result = rmq.arg_min(left, right); benchmark::DoNotOptimize(result); } + PerfCounterGroup::Values perf_values; + if (perf_counters.stop(&perf_values)) { + set_perf_counters(state, perf_values); + } - set_depth_counters(state, dataset, true); + set_depth_counters(state, dataset, true, BlockSize); } void register_benchmarks() { const std::vector sizes = {1ull << 10, 1ull << 14, 1ull << 18, - 1ull << 22, 1ull << 26}; + 1ull << 22, 1ull << 24, 1ull << 26}; + const std::vector build_sizes = { + 1ull << 10, 1ull << 14, 1ull << 18, 1ull << 22, 1ull << 24}; const std::vector widths = {64, 4096, 1ull << 18, 1ull << 22, 1ull << 26}; + // Pure sparse-table rows materialize O(n log n) indexes; above 2^22 they + // dominate memory/runtime and make large benchmark passes noisy. + constexpr std::size_t kSparseTableBenchmarkMaxSize = 1ull << 22; + + for (const std::size_t size : build_sizes) { + if (size <= kSparseTableBenchmarkMaxSize) { + benchmark::RegisterBenchmark( + "rmq_build_sparse_table", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); + } + benchmark::RegisterBenchmark( + "rmq_build_segment_tree", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); + benchmark::RegisterBenchmark( + "rmq_build_cartesian_tree", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); + benchmark::RegisterBenchmark( + "rmq_build_node_euler_btree", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); + benchmark::RegisterBenchmark( + "rmq_build_node_euler_btree_exp", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); + benchmark::RegisterBenchmark( + "rmq_build_bp_plus_minus_one", + &run_bp_rmq_build>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); + benchmark::RegisterBenchmark( + "rmq_build_bp_one_interval_btree", + &run_bp_rmq_build>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); + } for (const std::size_t size : sizes) { std::vector effective_widths; @@ -583,13 +1175,30 @@ void register_benchmarks() { effective_widths.end()); for (const std::size_t width : effective_widths) { - benchmark::RegisterBenchmark( - "rmq_sparse_table", - &run_queries, Index>>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); + if (size <= kSparseTableBenchmarkMaxSize) { + if (std::getenv("PIXIE_BENCH_SPARSE_FOOTPRINT") != nullptr) { + benchmark::RegisterBenchmark("rmq_sparse_table_footprint", + run_sparse_table_footprint) + ->Args({static_cast(size), + static_cast(width)}) + ->Iterations(1) + ->Unit(benchmark::kNanosecond); + } + benchmark::RegisterBenchmark( + "rmq_sparse_table", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_sparse_table_unaligned", + &run_queries, Index, 0>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + } benchmark::RegisterBenchmark( "rmq_segment_tree", &run_queriesArgs({static_cast(size), static_cast(width)}) ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_node_euler_btree", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_node_euler_btree_exp", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); benchmark::RegisterBenchmark( "rmq_bp_plus_minus_one", - &run_depth_queries>) + &run_depth_queries, 128>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_bp_one_interval_btree", + &run_depth_queries< + pixie::rmq::OneIntervalBTreeRmq, + pixie::rmq::OneIntervalBTreeRmq::kBlockSize>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_bp_one_interval_btree_h1", + &run_depth_queries< + pixie::rmq::OneIntervalBTreeRmq, + pixie::rmq::OneIntervalBTreeRmq::kBlockSize>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_bp_one_interval_btree_h2", + &run_depth_queries< + pixie::rmq::OneIntervalBTreeRmq, + pixie::rmq::OneIntervalBTreeRmq::kBlockSize>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_bp_one_interval_btree_h4", + &run_depth_queries< + pixie::rmq::OneIntervalBTreeRmq, + pixie::rmq::OneIntervalBTreeRmq::kBlockSize>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_bp_one_interval_btree_records", + &run_depth_queries< + pixie::rmq::OneIntervalBTreeRmqBoundaryRecords, + pixie::rmq::OneIntervalBTreeRmqBoundaryRecords< + Index>::kBlockSize>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_bp_one_interval_btree_records_h1", + &run_depth_queries< + pixie::rmq::OneIntervalBTreeRmqBoundaryRecords, + pixie::rmq::OneIntervalBTreeRmqBoundaryRecords::kBlockSize>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_bp_one_interval_btree_records_h2", + &run_depth_queries< + pixie::rmq::OneIntervalBTreeRmqBoundaryRecords, + pixie::rmq::OneIntervalBTreeRmqBoundaryRecords::kBlockSize>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_bp_one_interval_btree_records_h4", + &run_depth_queries< + pixie::rmq::OneIntervalBTreeRmqBoundaryRecords, + pixie::rmq::OneIntervalBTreeRmqBoundaryRecords::kBlockSize>) ->Args({static_cast(size), static_cast(width)}) ->Unit(benchmark::kNanosecond); diff --git a/src/benchmarks/benchmarks.cpp b/src/benchmarks/benchmarks.cpp index 11acbf6..8e5a9ea 100644 --- a/src/benchmarks/benchmarks.cpp +++ b/src/benchmarks/benchmarks.cpp @@ -92,7 +92,7 @@ static void PrepareRandomBits87p5Fill(std::span bits) { static void BM_RankNonInterleaved(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); auto bits_as_words = bits.As64BitInts(); PrepareRandomBits50pFill(bits_as_words); pixie::BitVector bv(bits_as_words, n); @@ -106,7 +106,7 @@ static void BM_RankNonInterleaved(benchmark::State& state) { static void BM_RankZeroNonInterleaved(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits50pFill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -120,7 +120,7 @@ static void BM_RankZeroNonInterleaved(benchmark::State& state) { static void BM_RankInterleaved(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits50pFill(bits.As64BitInts()); pixie::BitVectorInterleaved bv(bits.As64BitInts(), n); @@ -134,7 +134,7 @@ static void BM_RankInterleaved(benchmark::State& state) { static void BM_SelectNonInterleaved(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits50pFill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -150,7 +150,7 @@ static void BM_SelectNonInterleaved(benchmark::State& state) { static void BM_SelectZeroNonInterleaved(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits50pFill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -166,7 +166,7 @@ static void BM_SelectZeroNonInterleaved(benchmark::State& state) { static void BM_RankNonInterleaved12p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits12p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -187,7 +187,7 @@ static void BM_RankNonInterleaved12p5PercentFill(benchmark::State& state) { static void BM_RankZeroNonInterleaved12p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits12p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -201,7 +201,7 @@ static void BM_RankZeroNonInterleaved12p5PercentFill(benchmark::State& state) { static void BM_SelectNonInterleaved12p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits12p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -218,7 +218,7 @@ static void BM_SelectZeroNonInterleaved12p5PercentFill( benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits12p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -234,7 +234,7 @@ static void BM_SelectZeroNonInterleaved12p5PercentFill( static void BM_RankNonInterleaved87p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits87p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -248,7 +248,7 @@ static void BM_RankNonInterleaved87p5PercentFill(benchmark::State& state) { static void BM_RankZeroNonInterleaved87p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits87p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -262,7 +262,7 @@ static void BM_RankZeroNonInterleaved87p5PercentFill(benchmark::State& state) { static void BM_SelectNonInterleaved87p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits87p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -279,7 +279,7 @@ static void BM_SelectZeroNonInterleaved87p5PercentFill( benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits87p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); diff --git a/src/benchmarks/excess_positions_benchmarks.cpp b/src/benchmarks/excess_positions_benchmarks.cpp index 148ebe6..501536a 100644 --- a/src/benchmarks/excess_positions_benchmarks.cpp +++ b/src/benchmarks/excess_positions_benchmarks.cpp @@ -25,6 +25,10 @@ using pixie::experimental::excess_min_128_split64_sse; using pixie::experimental::excess_min_128_nibble_lut; using pixie::experimental::excess_min_128_scalar_bits; +// excess_record_lows_128, excess_record_lows_128_byte_lut, +// excess_record_lows_128_lut, excess_record_lows_128_avx2 are in the global +// namespace from pixie/bits.h + static std::vector> make_blocks( size_t num_blocks = 4096) { std::mt19937_64 rng(42); @@ -47,6 +51,16 @@ static std::vector> make_128_blocks( return blocks; } +static std::vector> make_64_blocks( + size_t num_blocks = 4096) { + std::mt19937_64 rng(42); + std::vector> blocks(num_blocks); + for (auto& b : blocks) { + b = {rng()}; + } + return blocks; +} + static std::vector> make_128_ranges( size_t num_ranges = 4096) { std::mt19937_64 rng(43); @@ -63,6 +77,22 @@ static std::vector> make_128_ranges( return ranges; } +static std::vector> make_64_ranges( + size_t num_ranges = 4096) { + std::mt19937_64 rng(43); + std::uniform_int_distribution offset_dist(0, 64); + std::vector> ranges(num_ranges); + for (auto& range : ranges) { + size_t left = offset_dist(rng); + size_t right = offset_dist(rng); + if (left > right) { + std::swap(left, right); + } + range = {left, right}; + } + return ranges; +} + static std::vector> make_disjoint_boundary_ranges( size_t num_ranges = 4096) { std::mt19937_64 rng(45); @@ -126,6 +156,44 @@ BENCHMARK(BM_ExcessMin128) ->Args({63, 64}) ->Args({17, 17}); +static void BM_ExcessMin64(benchmark::State& state) { + const size_t left = static_cast(state.range(0)); + const size_t right = static_cast(state.range(1)); + const auto blocks = make_64_blocks(); + const size_t num_blocks = blocks.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + ExcessResult result = excess_min_64(s.data(), left, right); + benchmark::DoNotOptimize(result.min_excess); + benchmark::DoNotOptimize(result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin64) + ->ArgNames({"left", "right"}) + ->Args({0, 64}) + ->Args({0, 63}) + ->Args({0, 16}) + ->Args({0, 32}) + ->Args({0, 48}) + ->Args({0, 31}) + ->Args({1, 17}) + ->Args({3, 35}) + ->Args({5, 37}) + ->Args({32, 64}) + ->Args({33, 64}) + ->Args({56, 64}) + ->Args({56, 63}) + ->Args({60, 64}) + ->Args({60, 63}) + ->Args({63, 64}) + ->Args({17, 17}); + static void BM_ExcessMin128BoundaryPairIndependent(benchmark::State& state) { const auto blocks = make_128_blocks(); const auto ranges = make_disjoint_boundary_ranges(); @@ -177,6 +245,90 @@ static void BM_ExcessMin128BoundaryPairFused(benchmark::State& state) { BENCHMARK(BM_ExcessMin128BoundaryPairFused); +static void BM_ExcessMin128BoundaryPairFixedIndependent( + benchmark::State& state) { + const size_t suffix_left = static_cast(state.range(0)); + const size_t prefix_right = static_cast(state.range(1)); + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& suffix = blocks[idx % num_blocks]; + const auto& prefix = blocks[(idx + 1) % num_blocks]; + ExcessResult suffix_result = + excess_min_128(suffix.data(), suffix_left, 127); + ExcessResult prefix_result = excess_min_128(prefix.data(), 0, prefix_right); + benchmark::DoNotOptimize(suffix_result.min_excess); + benchmark::DoNotOptimize(suffix_result.offset); + benchmark::DoNotOptimize(prefix_result.min_excess); + benchmark::DoNotOptimize(prefix_result.offset); + ++idx; + } + + state.counters["gate_score"] = + static_cast((127 - suffix_left) + prefix_right); + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin128BoundaryPairFixedIndependent) + ->ArgNames({"suffix_left", "prefix_right"}) + ->Args({127, 0}) + ->Args({126, 0}) + ->Args({127, 1}) + ->Args({126, 1}) + ->Args({125, 1}) + ->Args({125, 2}) + ->Args({125, 3}) + ->Args({124, 3}) + ->Args({120, 7}) + ->Args({112, 15}) + ->Args({111, 16}) + ->Args({96, 31}) + ->Args({80, 47}) + ->Args({64, 63}); + +static void BM_ExcessMin128BoundaryPairFixedFused(benchmark::State& state) { + const size_t suffix_left = static_cast(state.range(0)); + const size_t prefix_right = static_cast(state.range(1)); + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& suffix = blocks[idx % num_blocks]; + const auto& prefix = blocks[(idx + 1) % num_blocks]; + ExcessBoundaryPairResult result = excess_min_128_disjoint_suffix_prefix( + suffix.data(), suffix_left, prefix.data(), prefix_right); + benchmark::DoNotOptimize(result.suffix.min_excess); + benchmark::DoNotOptimize(result.suffix.offset); + benchmark::DoNotOptimize(result.prefix.min_excess); + benchmark::DoNotOptimize(result.prefix.offset); + ++idx; + } + + state.counters["gate_score"] = + static_cast((127 - suffix_left) + prefix_right); + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin128BoundaryPairFixedFused) + ->ArgNames({"suffix_left", "prefix_right"}) + ->Args({127, 0}) + ->Args({126, 0}) + ->Args({127, 1}) + ->Args({126, 1}) + ->Args({125, 1}) + ->Args({125, 2}) + ->Args({125, 3}) + ->Args({124, 3}) + ->Args({120, 7}) + ->Args({112, 15}) + ->Args({111, 16}) + ->Args({96, 31}) + ->Args({80, 47}) + ->Args({64, 63}); + template static void BM_ExcessMin128Variant(benchmark::State& state) { const size_t left = static_cast(state.range(0)); @@ -261,6 +413,27 @@ static void BM_ExcessMin128RandomRange(benchmark::State& state) { state.SetItemsProcessed(state.iterations()); } +static void BM_ExcessMin64RandomRange(benchmark::State& state) { + const auto blocks = make_64_blocks(); + const auto ranges = make_64_ranges(); + const size_t num_blocks = blocks.size(); + const size_t num_ranges = ranges.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + const auto [left, right] = ranges[idx % num_ranges]; + ExcessResult result = excess_min_64(s.data(), left, right); + benchmark::DoNotOptimize(result.min_excess); + benchmark::DoNotOptimize(result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin64RandomRange); + #define PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT(name, fn) \ BENCHMARK_TEMPLATE(BM_ExcessMin128RandomRange, fn)->Name(name) @@ -559,4 +732,129 @@ PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( "BM_ExcessPositions512_ByteLUT_RandomTarget", excess_positions_512_byte_lut); +static void naive_excess_record_lows_128(const uint64_t* s, uint64_t* out) { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + for (size_t i = 0; i < 128; ++i) { + const uint64_t w = s[i >> 6]; + const int bit = static_cast((w >> (i & 63)) & 1ull); + cur += bit ? +1 : -1; + if (cur < best) { + best = cur; + out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } +} + +static void BM_ExcessRecordLows128(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + excess_record_lows_128(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128); + +static void BM_ExcessRecordLows128_Naive(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + naive_excess_record_lows_128(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128_Naive); + +static void BM_ExcessRecordLows128_ByteLUT(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + excess_record_lows_128_byte_lut(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128_ByteLUT); + +static void BM_ExcessRecordLows128_LUT(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + excess_record_lows_128_lut(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128_LUT); + +#ifdef PIXIE_AVX2_SUPPORT +static void BM_ExcessRecordLows128_AVX2(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + excess_record_lows_128_avx2(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128_AVX2); + +static void BM_ExcessRecordLows128_NibbleLUT(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + excess_record_lows_128_nibble_lut(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128_NibbleLUT); +#endif + #undef PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM diff --git a/src/tests/excess_positions_tests.cpp b/src/tests/excess_positions_tests.cpp index c5c43f7..507c248 100644 --- a/src/tests/excess_positions_tests.cpp +++ b/src/tests/excess_positions_tests.cpp @@ -79,6 +79,16 @@ static int naive_prefix_excess_128(const uint64_t* s, size_t end_offset) { return cur; } +static int naive_prefix_excess_64(const uint64_t* s, size_t end_offset) { + end_offset = std::min(end_offset, 64); + int cur = 0; + for (size_t i = 0; i < end_offset; ++i) { + const int bit = int((s[0] >> i) & 1ull); + cur += bit ? +1 : -1; + } + return cur; +} + static ExcessResult naive_excess_min_128(const uint64_t* s, size_t left, size_t right) { @@ -114,6 +124,41 @@ static ExcessResult naive_excess_min_128(const uint64_t* s, return {best, best_offset}; } +static ExcessResult naive_excess_min_64(const uint64_t* s, + size_t left, + size_t right) { + if (left > right) { + return {}; + } + left = std::min(left, 64); + right = std::min(right, 64); + + int cur = 0; + int best = 0; + size_t best_offset = 0; + bool found = false; + if (left == 0) { + found = true; + } + for (size_t bit = 0; bit < right; ++bit) { + cur += ((s[0] >> bit) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (offset < left) { + continue; + } + if (!found || cur < best) { + best = cur; + best_offset = offset; + found = true; + } + } + if (!found) { + best = naive_prefix_excess_64(s, left); + best_offset = left; + } + return {best, best_offset}; +} + static size_t naive_forward_search_128(const uint64_t* s, int target_x, size_t start_offset) { @@ -255,6 +300,21 @@ TEST(ExcessPositions128, PrefixExcessMatchesNaive) { } } +TEST(ExcessPositions64, PrefixExcessMatchesNaive) { + std::mt19937_64 rng(44); + const std::array offsets = {0, 1, 2, 31, 32, 33, + 62, 63, 64, 65, 96}; + + for (int t = 0; t < 1000; ++t) { + const std::array s = {rng()}; + for (size_t offset : offsets) { + EXPECT_EQ(prefix_excess_64(s.data(), offset), + naive_prefix_excess_64(s.data(), offset)) + << "case=" << t << " offset=" << offset; + } + } +} + TEST(ExcessPositions128, MinMatchesNaiveFixedCases) { const std::array, 5> cases = {{ {0, 0}, @@ -290,6 +350,41 @@ TEST(ExcessPositions128, MinMatchesNaiveFixedCases) { } } +TEST(ExcessPositions64, MinMatchesNaiveFixedCases) { + const std::array, 5> cases = {{ + {0}, + {UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull}, + {0x0123456789ABCDEFull}, + {0x0000FFFF0000FFFFull}, + }}; + const std::array, 12> ranges = {{ + {0, 64}, + {0, 0}, + {1, 1}, + {31, 33}, + {32, 32}, + {32, 64}, + {3, 6}, + {5, 5}, + {63, 64}, + {64, 64}, + {56, 63}, + {65, 96}, + }}; + + for (const auto& s : cases) { + for (const auto [left, right] : ranges) { + const ExcessResult result = excess_min_64(s.data(), left, right); + const ExcessResult expected = naive_excess_min_64(s.data(), left, right); + EXPECT_EQ(result.min_excess, expected.min_excess) + << "left=" << left << " right=" << right; + EXPECT_EQ(result.offset, expected.offset) + << "left=" << left << " right=" << right; + } + } +} + TEST(ExcessPositions128, MinReturnsFirstTie) { const std::array s = {0x5555555555555555ull, 0x5555555555555555ull}; @@ -318,6 +413,18 @@ TEST(ExcessPositions128, MinHandlesRightBoundary) { EXPECT_EQ(with_last.offset, 128u); } +TEST(ExcessPositions64, MinHandlesRightBoundary) { + const std::array s = {0}; + + const ExcessResult without_last = excess_min_64(s.data(), 0, 63); + EXPECT_EQ(without_last.min_excess, -63); + EXPECT_EQ(without_last.offset, 63u); + + const ExcessResult with_last = excess_min_64(s.data(), 0, 64); + EXPECT_EQ(with_last.min_excess, -64); + EXPECT_EQ(with_last.offset, 64u); +} + TEST(ExcessPositions128, MinPartialNibbleBoundsExcludeOuterMin) { const std::array s = {0, 0}; @@ -345,6 +452,13 @@ TEST(ExcessPositions128, MinInvalidRangeUsesSentinel) { EXPECT_EQ(result.offset, 128u); } +TEST(ExcessPositions64, MinInvalidRangeUsesSentinel) { + const std::array s = {0}; + const ExcessResult result = excess_min_64(s.data(), 17, 16); + EXPECT_EQ(result.min_excess, 0); + EXPECT_EQ(result.offset, 128u); +} + TEST(ExcessPositions128, MinMatchesNaiveRandom) { std::mt19937_64 rng(43); std::uniform_int_distribution offset_dist(0, 128); @@ -367,6 +481,74 @@ TEST(ExcessPositions128, MinMatchesNaiveRandom) { } } +TEST(ExcessPositions64, MinMatchesNaiveRandom) { + std::mt19937_64 rng(45); + std::uniform_int_distribution offset_dist(0, 64); + + for (int t = 0; t < 1000; ++t) { + const std::array s = {rng()}; + for (int q = 0; q < 32; ++q) { + size_t left = offset_dist(rng); + size_t right = offset_dist(rng); + if (left > right) { + std::swap(left, right); + } + const ExcessResult result = excess_min_64(s.data(), left, right); + const ExcessResult expected = naive_excess_min_64(s.data(), left, right); + ASSERT_EQ(result.min_excess, expected.min_excess) + << "case=" << t << " left=" << left << " right=" << right; + ASSERT_EQ(result.offset, expected.offset) + << "case=" << t << " left=" << left << " right=" << right; + } + } +} + +TEST(ExcessPositions64, DisjointBoundaryPairMatchesIndependentFixedCases) { + const std::array, 5> cases = {{ + {0}, + {UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull}, + {0x0123456789ABCDEFull}, + {0x0000FFFF0000FFFFull}, + }}; + const std::array, 8> ranges = {{ + {1, 0}, + {4, 3}, + {17, 5}, + {31, 8}, + {47, 11}, + {61, 13}, + {62, 62}, + {64, 64}, + }}; + + for (const auto& suffix : cases) { + for (const auto& prefix : cases) { + for (const auto [suffix_left, prefix_right] : ranges) { + const ExcessBoundaryPairResult result = + excess_min_64_disjoint_suffix_prefix(suffix.data(), suffix_left, + prefix.data(), prefix_right); + const ExcessResult expected_suffix = + excess_min_64(suffix.data(), suffix_left, 63); + const ExcessResult expected_prefix = + excess_min_64(prefix.data(), 0, prefix_right); + EXPECT_EQ(result.suffix.min_excess, expected_suffix.min_excess) + << "suffix_left=" << suffix_left + << " prefix_right=" << prefix_right; + EXPECT_EQ(result.suffix.offset, expected_suffix.offset) + << "suffix_left=" << suffix_left + << " prefix_right=" << prefix_right; + EXPECT_EQ(result.prefix.min_excess, expected_prefix.min_excess) + << "suffix_left=" << suffix_left + << " prefix_right=" << prefix_right; + EXPECT_EQ(result.prefix.offset, expected_prefix.offset) + << "suffix_left=" << suffix_left + << " prefix_right=" << prefix_right; + } + } + } +} + TEST(ExcessPositions128, DisjointBoundaryPairMatchesIndependentFixedCases) { const std::array, 5> cases = {{ {0, 0}, diff --git a/src/tests/excess_record_lows_tests.cpp b/src/tests/excess_record_lows_tests.cpp new file mode 100644 index 0000000..6a79668 --- /dev/null +++ b/src/tests/excess_record_lows_tests.cpp @@ -0,0 +1,293 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Naive reference implementations +// --------------------------------------------------------------------------- + +static void naive_excess_record_lows_128(const uint64_t* s, uint64_t* out) { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + for (size_t i = 0; i < 128; ++i) { + const uint64_t w = s[i >> 6]; + const int bit = static_cast((w >> (i & 63)) & 1ull); + cur += bit ? +1 : -1; + if (cur < best) { + best = cur; + out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } +} + +static void naive_excess_record_lows_512(const uint64_t* s, uint64_t* out) { + for (int w = 0; w < 8; ++w) { + out[w] = 0; + } + int cur = 0; + int best = 0; + for (size_t i = 0; i < 512; ++i) { + const uint64_t w = s[i >> 6]; + const int bit = static_cast((w >> (i & 63)) & 1ull); + cur += bit ? +1 : -1; + if (cur < best) { + best = cur; + out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +static void check_128_matches_naive(const uint64_t* s, int case_id = 0) { + alignas(16) uint64_t out[2]; + alignas(16) uint64_t ref[2]; + excess_record_lows_128(s, out); + naive_excess_record_lows_128(s, ref); + EXPECT_EQ(out[0], ref[0]) << "case=" << case_id; + EXPECT_EQ(out[1], ref[1]) << "case=" << case_id; +} + +static void check_512_matches_naive(const uint64_t* s, int case_id = 0) { + alignas(64) uint64_t out[8]; + alignas(64) uint64_t ref[8]; + excess_record_lows_512(s, out); + naive_excess_record_lows_512(s, ref); + for (int w = 0; w < 8; ++w) { + EXPECT_EQ(out[w], ref[w]) << "case=" << case_id << " word=" << w; + } +} + +static size_t count_bits(const uint64_t* words, int num_words) { + size_t cnt = 0; + for (int i = 0; i < num_words; ++i) { + cnt += static_cast(std::popcount(words[i])); + } + return cnt; +} + +// --------------------------------------------------------------------------- +// 128-bit tests +// --------------------------------------------------------------------------- + +TEST(ExcessRecordLows128, AllZeros) { + const std::array s = {0, 0}; + uint64_t out[2]; + excess_record_lows_128(s.data(), out); + // Excess goes 0, -1, -2, ..., -128. Every position is a new record low. + EXPECT_EQ(out[0], UINT64_MAX); + EXPECT_EQ(out[1], UINT64_MAX); +} + +TEST(ExcessRecordLows128, AllOnes) { + const std::array s = {UINT64_MAX, UINT64_MAX}; + uint64_t out[2]; + excess_record_lows_128(s.data(), out); + // Excess goes 0, 1, 2, ..., 128. No new record lows after start. + EXPECT_EQ(out[0], 0u); + EXPECT_EQ(out[1], 0u); +} + +TEST(ExcessRecordLows128, Alternating) { + const std::array s = {0xAAAAAAAAAAAAAAAAull, + 0x5555555555555555ull}; + uint64_t out[2]; + excess_record_lows_128(s.data(), out); + // Pattern 1010... -> excess: 0, -1, 0, -1, 0, ... + // Only the first position (i=0) is a record low (excess=-1). + // Actually i=0 -> first bit=1 -> excess=+1, i=1 -> bit=0 -> excess=0, wait. + // 0xAA = 10101010, 0x55 = 01010101. + // Let's just trust the naive for correctness. + alignas(16) uint64_t ref[2]; + naive_excess_record_lows_128(s.data(), ref); + EXPECT_EQ(out[0], ref[0]); + EXPECT_EQ(out[1], ref[1]); +} + +TEST(ExcessRecordLows128, FixedCases) { + const std::array, 6> cases = {{ + {0, 0}, + {UINT64_MAX, UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull}, + {0x0000FFFF0000FFFFull, 0xFFFF0000FFFF0000ull}, + {0x1111222233334444ull, 0x8888777766665555ull}, + }}; + + int case_id = 0; + for (const auto& s : cases) { + check_128_matches_naive(s.data(), case_id++); + } +} + +TEST(ExcessRecordLows128, ExhaustiveSmall16) { + alignas(16) uint64_t s[2]; + alignas(16) uint64_t out[2]; + alignas(16) uint64_t ref[2]; + + for (uint64_t pattern = 0; pattern < (1ull << 16); ++pattern) { + s[0] = pattern; + s[1] = 0; + excess_record_lows_128(s, out); + naive_excess_record_lows_128(s, ref); + EXPECT_EQ(out[0], ref[0]) << "pattern=" << pattern; + EXPECT_EQ(out[1], ref[1]) << "pattern=" << pattern; + } +} + +TEST(ExcessRecordLows128, Random) { + const int cases = [] { + const char* env = std::getenv("RECORD_LOWS_CASES"); + return env ? std::atoi(env) : 10000; + }(); + const uint64_t seed = [] { + const char* env = std::getenv("RECORD_LOWS_SEED"); + return env ? std::stoull(env) : 42ull; + }(); + + std::mt19937_64 rng(static_cast(seed)); + + for (int t = 0; t < cases; ++t) { + const std::array s = {rng(), rng()}; + check_128_matches_naive(s.data(), t); + } +} + +#ifdef PIXIE_AVX2_SUPPORT +static void check_nibble_lut_matches_naive(const uint64_t* s, + int case_id = 0) { + alignas(16) uint64_t out[2]; + alignas(16) uint64_t ref[2]; + excess_record_lows_128_nibble_lut(s, out); + naive_excess_record_lows_128(s, ref); + EXPECT_EQ(out[0], ref[0]) << "nibble case=" << case_id; + EXPECT_EQ(out[1], ref[1]) << "nibble case=" << case_id; +} + +TEST(ExcessRecordLows128, NibbleLUTExhaustiveSmall16) { + alignas(16) uint64_t s[2]; + for (uint64_t pattern = 0; pattern < (1ull << 16); ++pattern) { + s[0] = pattern; + s[1] = 0; + check_nibble_lut_matches_naive(s, static_cast(pattern)); + } +} + +TEST(ExcessRecordLows128, NibbleLUTRandom) { + std::mt19937_64 rng(12345); + for (int t = 0; t < 10000; ++t) { + const std::array s = {rng(), rng()}; + check_nibble_lut_matches_naive(s.data(), t); + } +} +#endif + +// --------------------------------------------------------------------------- +// 512-bit tests +// --------------------------------------------------------------------------- + +TEST(ExcessRecordLows512, AllZeros) { + alignas(64) uint64_t s[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + alignas(64) uint64_t out[8]; + excess_record_lows_512(s, out); + for (int w = 0; w < 8; ++w) { + EXPECT_EQ(out[w], UINT64_MAX) << "word=" << w; + } +} + +TEST(ExcessRecordLows512, AllOnes) { + alignas(64) uint64_t s[8] = {UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX, + UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX}; + alignas(64) uint64_t out[8]; + excess_record_lows_512(s, out); + for (int w = 0; w < 8; ++w) { + EXPECT_EQ(out[w], 0u) << "word=" << w; + } +} + +TEST(ExcessRecordLows512, FixedCases) { + const std::array, 4> cases = {{ + {0, 0, 0, 0, 0, 0, 0, 0}, + {UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX, + UINT64_MAX, UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull, 0xAAAAAAAAAAAAAAAAull, + 0x5555555555555555ull, 0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull, + 0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull, 0x0000FFFF0000FFFFull, + 0xFFFF0000FFFF0000ull, 0x1111222233334444ull, 0x8888777766665555ull, + 0x5555555555555555ull, 0xAAAAAAAAAAAAAAAAull}, + }}; + + int case_id = 0; + for (const auto& s : cases) { + check_512_matches_naive(s.data(), case_id++); + } +} + +TEST(ExcessRecordLows512, ExhaustiveSmall16) { + alignas(64) uint64_t s[8]; + alignas(64) uint64_t out[8]; + alignas(64) uint64_t ref[8]; + + for (uint64_t pattern = 0; pattern < (1ull << 16); ++pattern) { + s[0] = pattern; + for (int w = 1; w < 8; ++w) { + s[w] = 0; + } + excess_record_lows_512(s, out); + naive_excess_record_lows_512(s, ref); + for (int w = 0; w < 8; ++w) { + EXPECT_EQ(out[w], ref[w]) << "pattern=" << pattern << " word=" << w; + } + } +} + +TEST(ExcessRecordLows512, Random) { + const int cases = [] { + const char* env = std::getenv("RECORD_LOWS_CASES"); + return env ? std::atoi(env) : 10000; + }(); + const uint64_t seed = [] { + const char* env = std::getenv("RECORD_LOWS_SEED"); + return env ? std::stoull(env) : 42ull; + }(); + + std::mt19937_64 rng(static_cast(seed)); + alignas(64) uint64_t s[8]; + + for (int t = 0; t < cases; ++t) { + for (int w = 0; w < 8; ++w) { + s[w] = rng(); + } + check_512_matches_naive(s, t); + } +} + +TEST(ExcessRecordLows512, CountMonotonic) { + // Record-low count in a 512-bit all-zeros should be 512. + alignas(64) uint64_t s[8] = {0}; + alignas(64) uint64_t out[8]; + excess_record_lows_512(s, out); + EXPECT_EQ(count_bits(out, 8), 512u); + + // Record-low count in a 512-bit all-ones should be 0. + for (int w = 0; w < 8; ++w) { + s[w] = UINT64_MAX; + } + excess_record_lows_512(s, out); + EXPECT_EQ(count_bits(out, 8), 0u); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index 5f13d8b..64834b1 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -1,12 +1,16 @@ #include #include +#include +#include #include +#include #include #include #include #include #include +#include #include namespace { @@ -16,11 +20,11 @@ std::size_t naive_arg_min(std::span values, std::size_t left, std::size_t right, Compare compare) { - if (left > right || right >= values.size()) { + if (left >= right || right > values.size()) { return pixie::rmq::SparseTable::npos; } std::size_t best = left; - for (std::size_t i = left + 1; i <= right; ++i) { + for (std::size_t i = left + 1; i < right; ++i) { if (compare(values[i], values[best])) { best = i; } @@ -34,12 +38,12 @@ void check_all_ranges(const Rmq& rmq, Compare compare) { ASSERT_EQ(rmq.size(), values.size()); for (std::size_t left = 0; left < values.size(); ++left) { - for (std::size_t right = left; right < values.size(); ++right) { + for (std::size_t right = left + 1; right <= values.size(); ++right) { const std::size_t expected = naive_arg_min(values, left, right, compare); EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << "]"; + << "range=[" << left << "," << right << ")"; EXPECT_EQ(rmq.range_min(left, right), values[expected]) - << "range=[" << left << "," << right << "]"; + << "range=[" << left << "," << right << ")"; } } } @@ -50,16 +54,19 @@ void check_all_arg_min_ranges(const Rmq& rmq, Compare compare) { ASSERT_EQ(rmq.size(), values.size()); for (std::size_t left = 0; left < values.size(); ++left) { - for (std::size_t right = left; right < values.size(); ++right) { + for (std::size_t right = left + 1; right <= values.size(); ++right) { const std::size_t expected = naive_arg_min(values, left, right, compare); EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << "]"; + << "range=[" << left << "," << right << ")"; } } } std::vector pack_depth_deltas( std::span depths) { + if (depths.size() <= 1) { + return {}; + } std::vector bits((depths.size() - 1 + 63) / 64, 0); for (std::size_t i = 1; i < depths.size(); ++i) { if (depths[i] - depths[i - 1] == 1) { @@ -69,96 +76,833 @@ std::vector pack_depth_deltas( return bits; } +bool packed_bit(std::span words, std::size_t position) { + return ((words[position >> 6] >> (position & 63)) & 1u) != 0; +} + +template +void expect_valid_bp_encoding(const Rmq& rmq, std::size_t value_count) { + const std::size_t bit_count = rmq.bp_bit_count(); + const std::span words = rmq.bp_words(); + EXPECT_EQ(bit_count, 2 * value_count); + EXPECT_EQ(words.size(), (bit_count + 63) / 64); + + std::size_t ones = 0; + std::size_t zeros = 0; + std::int64_t excess = 0; + for (std::size_t position = 0; position < bit_count; ++position) { + if (packed_bit(words, position)) { + ++ones; + ++excess; + } else { + ++zeros; + --excess; + } + EXPECT_GE(excess, 0) << "position=" << position; + } + + EXPECT_EQ(ones, value_count); + EXPECT_EQ(zeros, value_count); + EXPECT_EQ(excess, 0); +} + +struct SparseTableCase { + using Rmq = pixie::rmq::SparseTable; + using MaxRmq = pixie::rmq::SparseTable>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + +struct SegmentTreeCase { + using Rmq = pixie::rmq::SegmentTree; + using MaxRmq = pixie::rmq::SegmentTree>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + +struct CartesianTreeCase { + using Rmq = pixie::rmq::CartesianTreeRmq; + using MaxRmq = pixie::rmq::CartesianTreeRmq>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + +struct NodeEulerBTreeCase { + using Rmq = pixie::rmq::NodeEulerBTreeRmq; + using MaxRmq = pixie::rmq::NodeEulerBTreeRmq>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + +struct ExperimentalNodeEulerBTreeCase { + using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; + using MaxRmq = + pixie::rmq::experimental::NodeEulerBTreeRmq>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + +struct BpPlusMinusOne128Case { + using Rmq = pixie::rmq::BpPlusMinusOneRmq<>; + static constexpr std::size_t kBlockSize = 128; + + static Rmq make(std::span bits, + std::size_t depth_count) { + return Rmq(bits, depth_count); + } +}; + +struct BpPlusMinusOne64Case { + using Rmq = pixie::rmq::BpPlusMinusOneRmq; + static constexpr std::size_t kBlockSize = 64; + + static Rmq make(std::span bits, + std::size_t depth_count) { + return Rmq(bits, depth_count); + } +}; + +struct OneIntervalBTreeCase { + using Rmq = pixie::rmq::OneIntervalBTreeRmq<>; + static constexpr std::size_t kBlockSize = Rmq::kBlockSize; + + static Rmq make(std::span bits, + std::size_t depth_count) { + return Rmq(bits, depth_count); + } +}; + +struct OneIntervalBTreeBoundaryRecordsCase { + using Rmq = pixie::rmq::OneIntervalBTreeRmqBoundaryRecords<>; + static constexpr std::size_t kBlockSize = Rmq::kBlockSize; + + static Rmq make(std::span bits, + std::size_t depth_count) { + return Rmq(bits, depth_count); + } +}; + +template +typename DepthCase::Rmq make_depth_rmq(const std::vector& bits, + std::size_t depth_count) { + return DepthCase::make(std::span(bits), depth_count); +} + +template +void check_depths_all_arg_min(std::span depths) { + const std::vector bits = pack_depth_deltas(depths); + const typename DepthCase::Rmq rmq = + make_depth_rmq(bits, depths.size()); + check_all_arg_min_ranges(rmq, depths, std::less()); +} + +template +void check_depth_ranges( + const Rmq& rmq, + std::span depths, + std::span> ranges) { + for (const auto [left, right] : ranges) { + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(depths, left, right, std::less())) + << "range=[" << left << "," << right << ")"; + } +} + +template +void check_depth_case_ranges( + std::span depths, + std::span> ranges) { + const std::vector bits = pack_depth_deltas(depths); + const typename DepthCase::Rmq rmq = + make_depth_rmq(bits, depths.size()); + check_depth_ranges(rmq, depths, ranges); +} + +void check_one_interval_btree_ranges( + std::span depths, + std::span> ranges) { + const std::vector bits = pack_depth_deltas(depths); + const pixie::rmq::OneIntervalBTreeRmq<> rmq(bits, depths.size()); + const pixie::rmq::OneIntervalBTreeRmqBoundaryRecords<> records(bits, + depths.size()); + const pixie::rmq::BpPlusMinusOneRmq<> baseline(bits, depths.size()); + for (const auto [left, right] : ranges) { + const std::size_t expected = + naive_arg_min(std::span(depths), left, right, + std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(rmq.arg_min(left, right), baseline.arg_min(left, right)) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(records.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(records.arg_min(left, right), baseline.arg_min(left, right)) + << "range=[" << left << "," << right << ")"; + } +} + } // namespace -TEST(RmqSparseTable, ExhaustiveSmallArray) { +template +class ValueRmqContractTest : public ::testing::Test {}; + +using ValueRmqCases = ::testing::Types; +TYPED_TEST_SUITE(ValueRmqContractTest, ValueRmqCases); + +TYPED_TEST(ValueRmqContractTest, ExhaustiveSmallArray) { const std::vector values = {4, 1, 3, 1, 5, 0, 0, 2}; - const pixie::rmq::SparseTable rmq(values); + const typename TypeParam::Rmq rmq = + TypeParam::make(std::span(values)); check_all_ranges(rmq, std::span(values), std::less()); } -TEST(RmqSegmentTree, ExhaustiveSmallArray) { - const std::vector values = {4, 1, 3, 1, 5, 0, 0, 2}; - const pixie::rmq::SegmentTree rmq(values); - check_all_ranges(rmq, std::span(values), std::less()); +TYPED_TEST(ValueRmqContractTest, FirstMinimumTieBreaking) { + const std::vector values = {7, 2, 2, 3, 2}; + const typename TypeParam::Rmq rmq = + TypeParam::make(std::span(values)); + + EXPECT_EQ(rmq.arg_min(0, 5), 1u); + EXPECT_EQ(rmq.arg_min(2, 5), 2u); +} + +TYPED_TEST(ValueRmqContractTest, InvalidAndEmptyRanges) { + using Rmq = typename TypeParam::Rmq; + + const std::vector values = {3, 1, 2}; + const Rmq rmq = TypeParam::make(std::span(values)); + + EXPECT_EQ(rmq.arg_min(2, 2), Rmq::npos); + EXPECT_EQ(rmq.arg_min(2, 1), Rmq::npos); + EXPECT_EQ(rmq.arg_min(0, values.size() + 1), Rmq::npos); + EXPECT_EQ(rmq.range_min(2, 2), 0); + EXPECT_EQ(rmq.arg_min(0, values.size()), 1u); + + const std::vector empty_values; + const Rmq default_rmq; + const Rmq empty_rmq = TypeParam::make(std::span(empty_values)); + EXPECT_TRUE(default_rmq.empty()); + EXPECT_TRUE(empty_rmq.empty()); + EXPECT_EQ(default_rmq.arg_min(0, 0), Rmq::npos); + EXPECT_EQ(empty_rmq.arg_min(0, 0), Rmq::npos); +} + +TYPED_TEST(ValueRmqContractTest, ComparatorCanSelectMaximum) { + const std::vector values = {1, 8, 3, 8, 4}; + const typename TypeParam::MaxRmq rmq = + TypeParam::make_max(std::span(values)); + + check_all_ranges(rmq, std::span(values), std::greater()); + EXPECT_EQ(rmq.arg_min(0, 5), 1u); +} + +TYPED_TEST(ValueRmqContractTest, MonotoneArrays) { + const std::vector increasing = {1, 2, 3, 4, 5, 6}; + const std::vector decreasing = {6, 5, 4, 3, 2, 1}; + const typename TypeParam::Rmq increasing_rmq = + TypeParam::make(std::span(increasing)); + const typename TypeParam::Rmq decreasing_rmq = + TypeParam::make(std::span(decreasing)); + + check_all_ranges(increasing_rmq, std::span(increasing), + std::less()); + check_all_ranges(decreasing_rmq, std::span(decreasing), + std::less()); +} + +TYPED_TEST(ValueRmqContractTest, DifferentialRandom) { + std::mt19937_64 rng(42); + std::uniform_int_distribution value_dist(-50, 50); + for (std::size_t size = 1; size <= 257; size += 17) { + std::vector values(size); + std::generate(values.begin(), values.end(), + [&] { return value_dist(rng); }); + + const typename TypeParam::Rmq rmq = + TypeParam::make(std::span(values)); + check_all_ranges(rmq, std::span(values), std::less()); + } +} + +TEST(RmqNodeEulerBTree, BoundarySizesAroundLeavesAndInternalNodes) { + using Rmq = pixie::rmq::NodeEulerBTreeRmq; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + constexpr std::size_t kFanout = Rmq::kFanout; + const std::vector sizes = { + kLeaf - 1, + kLeaf, + kLeaf + 1, + kLeaf * kFanout - 1, + kLeaf * kFanout, + kLeaf * kFanout + 1, + kLeaf * kFanout + kLeaf + 1, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 17 + i / 5) % 31); + if (i % 9 == 0 || i % 32 == 3) { + values[i] = 4; + } + } + + const Rmq rmq{std::span(values)}; + const auto check_range = [&](std::size_t left, std::size_t right) { + ASSERT_LT(left, right); + ASSERT_LE(right, values.size()); + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(rmq.range_min(left, right), values[expected]) + << "range=[" << left << "," << right << ")"; + }; + + check_range(0, 1); + check_range(0, size); + check_range(size - 1, size); + + const std::vector boundaries = { + kLeaf, + kLeaf * kFanout, + }; + for (const std::size_t boundary : boundaries) { + if (boundary >= size) { + continue; + } + const std::size_t near_left = boundary > 19 ? boundary - 19 : 0; + const std::size_t wide_left = + boundary > kLeaf + 3 ? boundary - kLeaf - 3 : 0; + check_range(boundary - 1, boundary); + check_range(boundary, boundary + 1); + check_range(boundary - 1, boundary + 1); + check_range(near_left, std::min(size, boundary + 23)); + check_range(wide_left, std::min(size, boundary + kLeaf + 5)); + check_range(1, std::min(size, boundary + 1)); + check_range(boundary, size); + } + } +} + +TEST(RmqNodeEulerBTree, CommonAncestorQueryShapes) { + using Rmq = pixie::rmq::NodeEulerBTreeRmq; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + constexpr std::size_t kFanout = Rmq::kFanout; + const std::size_t first_level_span = kLeaf * kFanout; + std::vector values(first_level_span + 3 * kLeaf + 17); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 29 + i / 13) % 101); + if (i % 257 == 0) { + values[i] = -7; + } + if (i % 4099 == 3) { + values[i] = -9; + } + } + + const Rmq rmq{std::span(values)}; + const std::vector> ranges = { + {7, 99}, + {kLeaf - 3, 2 * kLeaf + 5}, + {17, 18 * kLeaf + 111}, + {first_level_span - 4, first_level_span + kLeaf + 9}, + {kLeaf * (kFanout - 7) + 13, first_level_span + 2 * kLeaf + 19}, + {0, first_level_span}, + {kLeaf, first_level_span + kLeaf}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqNodeEulerBTree, SelectorFirstCorrectsInvalidBorderMinima) { + using Rmq = pixie::rmq::NodeEulerBTreeRmq; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + std::vector values(3 * kLeaf, 1000); + values[0] = -1000; + values[kLeaf / 2] = 50; + values[kLeaf + 44] = -10; + values[2 * kLeaf + 10] = 60; + values[2 * kLeaf + 200] = -900; + + const Rmq rmq{std::span(values)}; + const std::vector> ranges = { + {kLeaf / 2, 2 * kLeaf + 100}, + {kLeaf / 2, kLeaf + 100}, + {kLeaf - 50, 3 * kLeaf - 100}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqNodeEulerBTree, DuplicateHeavyRandomDifferentialTo8193) { + using Rmq = pixie::rmq::NodeEulerBTreeRmq; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + constexpr std::size_t kFanout = Rmq::kFanout; + std::mt19937_64 rng(811); + std::uniform_int_distribution value_dist(-3, 3); + const std::vector sizes = { + 1, 2, 17, 255, 256, 257, 1024, 8191, 8192, 8193, kLeaf * kFanout + 1, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = value_dist(rng); + if ((i % 11) < 5) { + values[i] = 0; + } + } + + const Rmq rmq{std::span(values)}; + std::uniform_int_distribution width_dist(1, size); + for (std::size_t query = 0; query < 2000; ++query) { + const std::size_t width = width_dist(rng); + std::uniform_int_distribution left_dist(0, size - width); + const std::size_t left = left_dist(rng); + const std::size_t right = left + width; + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } + } +} + +TEST(RmqNodeEulerBTree, TopDepthSparseSelectorLevelShapes) { + using MinRmq = + pixie::rmq::NodeEulerBTreeRmq, std::size_t, 4, 4>; + using MaxRmq = + pixie::rmq::NodeEulerBTreeRmq, std::size_t, 4, 4>; + const std::vector sizes = { + 4, // no internal level + 5, // one internal level + 17, // two internal levels + 65, // three internal levels + 257, // more than three internal levels + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 19 + i / 3) % 23); + if (i % 7 == 0 || i % 11 == 3) { + values[i] = -2; + } + if (i % 29 == 5) { + values[i] = 31; + } + } + + const MinRmq min_rmq{std::span(values)}; + const MaxRmq max_rmq{std::span(values)}; + check_all_arg_min_ranges(min_rmq, std::span(values), + std::less()); + check_all_arg_min_ranges(max_rmq, std::span(values), + std::greater()); + } +} + +TEST(RmqNodeEulerBTree, TopDepthSparseSelectorDefaultFanoutRandom) { + using Rmq = pixie::rmq::NodeEulerBTreeRmq; + using MaxRmq = pixie::rmq::NodeEulerBTreeRmq>; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + constexpr std::size_t kFanout = Rmq::kFanout; + const std::size_t size = kLeaf * kFanout + kLeaf + 37; + + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 31 + i / 9) % 127); + if (i % 257 == 3 || i % 8191 == 17) { + values[i] = -9; + } + if (i % 4099 == 19) { + values[i] = 211; + } + } + + const Rmq rmq{std::span(values)}; + const MaxRmq max_rmq{std::span(values)}; + std::mt19937_64 rng(424242); + std::uniform_int_distribution width_dist(1, values.size()); + for (std::size_t query = 0; query < 3000; ++query) { + const std::size_t width = width_dist(rng); + std::uniform_int_distribution left_dist(0, + values.size() - width); + const std::size_t left = left_dist(rng); + const std::size_t right = left + width; + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(max_rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::greater())) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqNodeEulerBTree, CopyAndMovePreserveSelectors) { + using Rmq = pixie::rmq::NodeEulerBTreeRmq; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + constexpr std::size_t kFanout = Rmq::kFanout; + std::vector values(kLeaf * kFanout + 512); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 43 + i / 7) % 97); + if (i % 13 == 0) { + values[i] = -5; + } + } + + const Rmq original{std::span(values)}; + Rmq copied(original); + Rmq assigned; + assigned = original; + Rmq moved(std::move(copied)); + Rmq move_assigned; + move_assigned = std::move(assigned); + + const std::vector> ranges = { + {0, 1}, {0, values.size()}, + {255, 257}, {kLeaf * kFanout - 5, kLeaf * kFanout + 6}, + {1234, 5678}, {values.size() - 17, values.size()}, + }; + + const std::array rmqs = {&original, &moved, &move_assigned}; + for (const Rmq* rmq : rmqs) { + ASSERT_EQ(rmq->size(), values.size()); + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq->arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } + } +} + +TEST(RmqExperimentalNodeEulerBTree, SplitLevelBoundarySizes) { + using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + constexpr std::size_t kFanout = Rmq::kFanout; + constexpr std::size_t kMediumSpan = kLeaf * kFanout; + const std::vector sizes = { + kLeaf - 1, + kLeaf, + kLeaf + 1, + kMediumSpan - 1, + kMediumSpan, + kMediumSpan + 1, + kMediumSpan + 3 * kLeaf + 17, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 37 + i / 11) % 113); + if (i % 17 == 0 || i % 257 == 3) { + values[i] = -4; + } + if (i % 4099 == 19) { + values[i] = -11; + } + } + + const Rmq rmq{std::span(values)}; + const auto check_range = [&](std::size_t left, std::size_t right) { + ASSERT_LT(left, right); + ASSERT_LE(right, values.size()); + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(rmq.range_min(left, right), values[expected]) + << "range=[" << left << "," << right << ")"; + }; + + check_range(0, 1); + check_range(0, size); + check_range(size - 1, size); + + for (const std::size_t boundary : {kLeaf, kMediumSpan}) { + if (boundary == 0 || boundary >= size) { + continue; + } + check_range(boundary - 1, boundary); + check_range(boundary, boundary + 1); + check_range(boundary - 1, boundary + 1); + check_range(boundary > 67 ? boundary - 67 : 0, + std::min(size, boundary + 71)); + check_range(boundary > kLeaf + 5 ? boundary - kLeaf - 5 : 0, + std::min(size, boundary + kLeaf + 9)); + } + } +} + +TEST(RmqExperimentalNodeEulerBTree, LeafEmbeddedOffsetBoundaryRanges) { + using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + + std::vector values(2 * kLeaf + 5); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 29 + i / 7) % 97) + 10; + } + values[kLeaf - 1] = -7; + values[kLeaf] = -7; + values[2 * kLeaf] = -9; + + const Rmq rmq{std::span(values)}; + const std::vector> ranges = { + {0, kLeaf}, {0, kLeaf + 1}, + {kLeaf, kLeaf + 1}, {kLeaf - 3, kLeaf + 4}, + {kLeaf, kLeaf + 4}, {2 * kLeaf - 2, 2 * kLeaf + 3}, + {0, values.size()}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(rmq.range_min(left, right), values[expected]) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqExperimentalNodeEulerBTree, DuplicateHeavyRandomDifferentialTo8193) { + using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; + std::mt19937_64 rng(9127); + std::uniform_int_distribution value_dist(-3, 3); + const std::vector sizes = { + 1, 2, 17, 255, 256, 257, 1024, 4096, 8191, 8192, 8193, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = value_dist(rng); + if ((i % 13) < 7) { + values[i] = 0; + } + } + + const Rmq rmq{std::span(values)}; + std::uniform_int_distribution width_dist(1, size); + for (std::size_t query = 0; query < 2000; ++query) { + const std::size_t width = width_dist(rng); + std::uniform_int_distribution left_dist(0, size - width); + const std::size_t left = left_dist(rng); + const std::size_t right = left + width; + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } + } +} + +TEST(RmqExperimentalNodeEulerBTree, HighSparseSelectorWithMiddleLevels) { + using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + constexpr std::size_t kFanout = Rmq::kFanout; + constexpr std::size_t kMiddleBoundary = kLeaf * kFanout * kFanout; + const std::size_t size = kMiddleBoundary + 2 * kLeaf + 17; + + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 53 + i / 17) % 251); + if (i % 257 == 3 || i % 65537 == 11) { + values[i] = -5; + } + } + values[13] = -100; + values[kMiddleBoundary - 29] = -90; + values[kMiddleBoundary + kLeaf + 7] = -110; + + const Rmq rmq{std::span(values)}; + const std::vector> ranges = { + {0, values.size()}, + {kMiddleBoundary - 2 * kLeaf, kMiddleBoundary + 2 * kLeaf}, + {kMiddleBoundary - 31, kMiddleBoundary + kLeaf + 31}, + {kMiddleBoundary + 1, values.size()}, + {kLeaf * (kFanout - 1) + 9, kMiddleBoundary + kLeaf + 13}, + {values.size() - 3 * kLeaf, values.size()}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqExperimentalNodeEulerBTree, CopyAndMovePreserveSplitStorage) { + using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + constexpr std::size_t kFanout = Rmq::kFanout; + std::vector values(kLeaf * kFanout + 2 * kLeaf + 9); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 41 + i / 3) % 127); + if (i % 23 == 0) { + values[i] = -8; + } + } + + const Rmq original{std::span(values)}; + Rmq copied(original); + Rmq assigned; + assigned = original; + Rmq moved(std::move(copied)); + Rmq move_assigned; + move_assigned = std::move(assigned); + + const std::vector> ranges = { + {0, 1}, + {0, values.size()}, + {255, 257}, + {kLeaf * kFanout - 5, kLeaf * kFanout + 6}, + {values.size() - 513, values.size() - 3}, + {values.size() - 17, values.size()}, + }; + + const std::array rmqs = {&original, &moved, &move_assigned}; + for (const Rmq* rmq : rmqs) { + ASSERT_EQ(rmq->size(), values.size()); + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq->arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } + } } -TEST(RmqBpPlusMinusOne, ExhaustiveSmallDepthArray) { +template +class DepthRmqContractTest : public ::testing::Test {}; + +using DepthRmqCases = ::testing::Types; +TYPED_TEST_SUITE(DepthRmqContractTest, DepthRmqCases); + +TYPED_TEST(DepthRmqContractTest, EmptyAndSingleDepthInputs) { + using Rmq = typename TypeParam::Rmq; + + const std::vector bits; + const Rmq empty = TypeParam::make(std::span(bits), 0); + EXPECT_TRUE(empty.empty()); + EXPECT_EQ(empty.size(), 0u); + EXPECT_EQ(empty.arg_min(0, 0), Rmq::npos); + + const Rmq single = TypeParam::make(std::span(bits), 1); + EXPECT_FALSE(single.empty()); + EXPECT_EQ(single.size(), 1u); + EXPECT_EQ(single.arg_min(0, 1), 0u); + EXPECT_EQ(single.arg_min(1, 1), Rmq::npos); + EXPECT_EQ(single.arg_min(0, 2), Rmq::npos); +} + +TYPED_TEST(DepthRmqContractTest, ExhaustiveSmallDepthArray) { const std::vector depths = {0, 1, 0, 1, 2, 1, 2, 1, 0}; - const std::vector bits = pack_depth_deltas(depths); - const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); - check_all_arg_min_ranges(rmq, std::span(depths), - std::less()); + check_depths_all_arg_min(std::span(depths)); } -TEST(RmqBpPlusMinusOne, CrossBlockRanges) { - std::vector depths(385); +TYPED_TEST(DepthRmqContractTest, CrossBlockRanges) { + constexpr std::size_t kBlock = TypeParam::kBlockSize; + std::vector depths(3 * kBlock + 1); for (std::size_t i = 1; i < depths.size(); ++i) { const bool up = (i % 7 == 0) || (i % 7 == 1) || (i % 7 == 4); depths[i] = depths[i - 1] + (up ? 1 : -1); } - const std::vector bits = pack_depth_deltas(depths); - const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); - check_all_arg_min_ranges(rmq, std::span(depths), - std::less()); + check_depths_all_arg_min(std::span(depths)); } -TEST(RmqBpPlusMinusOne, BoundaryRangesAround128PositionBlocks) { - std::vector depths(260); +TYPED_TEST(DepthRmqContractTest, BoundaryRangesAroundBlocks) { + constexpr std::size_t kBlock = TypeParam::kBlockSize; + std::vector depths(3 * kBlock + 6); for (std::size_t i = 1; i < depths.size(); ++i) { - depths[i] = depths[i - 1] + ((i % 5 == 0 || i % 11 == 0) ? 1 : -1); + depths[i] = + depths[i - 1] + ((i % 5 == 0 || i % 11 == 0 || i % 17 == 0) ? 1 : -1); } - const std::vector bits = pack_depth_deltas(depths); - const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); const std::vector> ranges = { - {0, 0}, {126, 127}, {127, 128}, {128, 128}, - {128, 255}, {129, 255}, {255, 256}, {0, 259}, + {0, 1}, + {kBlock - 2, kBlock}, + {kBlock - 1, kBlock + 1}, + {kBlock, kBlock + 1}, + {kBlock, 2 * kBlock}, + {kBlock + 1, 2 * kBlock}, + {2 * kBlock - 1, 2 * kBlock + 1}, + {0, depths.size()}, + {kBlock - 31, 2 * kBlock + 3}, + {kBlock - 12, 2 * kBlock + 6}, + {2 * kBlock, depths.size()}, }; - - for (const auto [left, right] : ranges) { - EXPECT_EQ(rmq.arg_min(left, right), - naive_arg_min(std::span(depths), left, right, - std::less())) - << "range=[" << left << "," << right << "]"; - } + check_depth_case_ranges( + std::span(depths), + std::span>(ranges)); } -TEST(RmqBpPlusMinusOne, LongSequenceRangesNearEnd) { +TYPED_TEST(DepthRmqContractTest, LongSequenceRangesNearEnd) { std::vector depths(8193); for (std::size_t i = 1; i < depths.size(); ++i) { const bool up = (i % 13 == 0) || (i % 17 == 0) || (i % 19 == 0); depths[i] = depths[i - 1] + (up ? 1 : -1); } - const std::vector bits = pack_depth_deltas(depths); - const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); const std::vector> ranges = { - {depths.size() - 1, depths.size() - 1}, - {depths.size() - 128, depths.size() - 1}, - {depths.size() - 513, depths.size() - 3}, - {depths.size() - 4097, depths.size() - 7}, + {depths.size() - 1, depths.size()}, + {depths.size() - 128, depths.size()}, + {depths.size() - 513, depths.size() - 2}, + {depths.size() - 4097, depths.size() - 6}, }; - - for (const auto [left, right] : ranges) { - EXPECT_EQ(rmq.arg_min(left, right), - naive_arg_min(std::span(depths), left, right, - std::less())) - << "range=[" << left << "," << right << "]"; - } + check_depth_case_ranges( + std::span(depths), + std::span>(ranges)); } -TEST(RmqBpPlusMinusOne, CrossBlockTieKeepsFirstPosition) { - std::vector depths(384); +TYPED_TEST(DepthRmqContractTest, CrossBlockTieKeepsFirstPosition) { + constexpr std::size_t kBlock = TypeParam::kBlockSize; + std::vector depths(3 * kBlock); for (std::size_t i = 1; i < depths.size(); ++i) { depths[i] = (i % 2 == 0) ? 0 : 1; } const std::vector bits = pack_depth_deltas(depths); - const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); - const std::size_t left = 120; - const std::size_t right = 260; + const typename TypeParam::Rmq rmq = + make_depth_rmq(bits, depths.size()); + const std::size_t left = kBlock - 8; + const std::size_t right = 2 * kBlock + 5; EXPECT_EQ(rmq.arg_min(left, right), left); EXPECT_EQ(rmq.arg_min(left, right), @@ -166,110 +910,167 @@ TEST(RmqBpPlusMinusOne, CrossBlockTieKeepsFirstPosition) { std::less())); } -TEST(RmqBpPlusMinusOne, DisjointBoundaryRangeMatchesNaive) { - std::vector depths(384); +TYPED_TEST(DepthRmqContractTest, DisjointBoundaryRangeMatchesNaive) { + constexpr std::size_t kBlock = TypeParam::kBlockSize; + std::vector depths(3 * kBlock + 7); for (std::size_t i = 1; i < depths.size(); ++i) { const bool up = (i % 9 == 0) || (i % 9 == 3) || (i % 11 == 0); depths[i] = depths[i - 1] + (up ? 1 : -1); } + const std::vector> ranges = { + {kBlock - 32, kBlock + 33}, {kBlock - 8, kBlock + 13}, + {kBlock - 1, kBlock + 2}, {2 * kBlock - 10, 2 * kBlock + 5}, + {2 * kBlock - 1, 2 * kBlock + 4}, + }; + check_depth_case_ranges( + std::span(depths), + std::span>(ranges)); +} + +TYPED_TEST(DepthRmqContractTest, RejectsTooSmallBitSpan) { + const std::vector bits; + EXPECT_THROW( + { + const typename TypeParam::Rmq rmq = + TypeParam::make(std::span(bits), 2); + (void)rmq; + }, + std::invalid_argument); +} + +TYPED_TEST(DepthRmqContractTest, DifferentialRandomWalks) { + std::mt19937_64 rng(77); + for (std::size_t size = 1; size <= 257; size += 17) { + std::vector depths(size); + for (std::size_t i = 1; i < depths.size(); ++i) { + depths[i] = depths[i - 1] + ((rng() & 1u) != 0 ? 1 : -1); + } + + check_depths_all_arg_min(std::span(depths)); + } +} + +TEST(RmqOneIntervalBTree, FusedBoundaryTiesKeepFirstPosition) { + constexpr std::size_t kBlock = pixie::rmq::OneIntervalBTreeRmq<>::kBlockSize; + std::vector depths(4 * kBlock + 9); + for (std::size_t i = 0; i < depths.size(); ++i) { + depths[i] = (i & 1u) == 0 ? 0 : 1; + } + + const std::vector> ranges = { + {kBlock - 8, 2 * kBlock + 8}, + {kBlock - 1, 3 * kBlock + 4}, + {kBlock + 3, 4 * kBlock + 2}, + {2 * kBlock - 5, 4 * kBlock + 6}, + }; + check_one_interval_btree_ranges( + std::span(depths), + std::span>(ranges)); +} + +TEST(RmqOneIntervalBTreeBoundaryRecords, CrossBlockAndTailRanges) { + using Rmq = pixie::rmq::OneIntervalBTreeRmqBoundaryRecords<>; + constexpr std::size_t kBlock = Rmq::kBlockSize; + std::vector depths(2 * kBlock + 17); + for (std::size_t i = 1; i < depths.size(); ++i) { + const bool up = (i % 6 == 0) || (i % 10 == 1) || (i % 17 == 3); + depths[i] = depths[i - 1] + (up ? 1 : -1); + } + const std::vector bits = pack_depth_deltas(depths); - const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); + const Rmq records(bits, depths.size()); + const pixie::rmq::OneIntervalBTreeRmq<> baseline(bits, depths.size()); const std::vector> ranges = { - {96, 160}, {120, 140}, {127, 129}, {190, 258}, {250, 260}, + {kBlock - 6, kBlock + 1}, {kBlock - 3, 2 * kBlock + 2}, + {kBlock - 7, 2 * kBlock}, {kBlock, 2 * kBlock + 5}, + {kBlock - 5, 2 * kBlock + 6}, {2 * kBlock - 3, depths.size()}, + {0, depths.size()}, }; for (const auto [left, right] : ranges) { - EXPECT_EQ(rmq.arg_min(left, right), - naive_arg_min(std::span(depths), left, right, - std::less())) - << "range=[" << left << "," << right << "]"; + const std::size_t expected = + naive_arg_min(std::span(depths), left, right, + std::less()); + EXPECT_EQ(records.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(records.arg_min(left, right), baseline.arg_min(left, right)) + << "range=[" << left << "," << right << ")"; } } -TEST(RmqBpPlusMinusOne, RejectsTooSmallBitSpan) { - const std::vector bits; - EXPECT_THROW((pixie::rmq::BpPlusMinusOneRmq<>(bits, 2)), - std::invalid_argument); +TEST(RmqOneIntervalBTree, LowNodeBoundaryRanges) { + constexpr std::size_t kBlock = pixie::rmq::OneIntervalBTreeRmq<>::kBlockSize; + constexpr std::size_t kLowBoundary = + kBlock * pixie::rmq::OneIntervalBTreeRmq<>::kLowFanout; + std::vector depths(kLowBoundary + 4 * kBlock + 1); + for (std::size_t i = 1; i < depths.size(); ++i) { + depths[i] = + depths[i - 1] + ((i % 7 == 0 || i % 19 == 0 || i % 23 == 0) ? 1 : -1); + } + + const std::vector> ranges = { + {kLowBoundary - 2, kLowBoundary + 2}, + {kLowBoundary - kBlock - 1, kLowBoundary + kBlock + 1}, + {1, kLowBoundary + 1}, + {kBlock, kLowBoundary + kBlock}, + {kLowBoundary, depths.size()}, + {0, depths.size()}, + }; + check_one_interval_btree_ranges( + std::span(depths), + std::span>(ranges)); } -TEST(RmqCartesianTree, ExhaustiveSmallArray) { - const std::vector values = {4, 1, 3, 1, 5, 0, 0, 2}; - const pixie::rmq::CartesianTreeRmq rmq(values); - check_all_ranges(rmq, std::span(values), std::less()); +TEST(RmqOneIntervalBTree, HighNodeBoundaryRanges) { + constexpr std::size_t kBlock = pixie::rmq::OneIntervalBTreeRmq<>::kBlockSize; + constexpr std::size_t kHighBoundary = + kBlock * pixie::rmq::OneIntervalBTreeRmq<>::kLowFanout * + pixie::rmq::OneIntervalBTreeRmq<>::kHighFanout; + std::vector depths(kHighBoundary + 32 * kBlock + 1); + for (std::size_t i = 1; i < depths.size(); ++i) { + depths[i] = + depths[i - 1] + ((i % 13 == 0 || i % 29 == 0 || i % 31 == 0) ? 1 : -1); + } + + const std::vector> ranges = { + {kHighBoundary - 2, kHighBoundary + 2}, + {kHighBoundary - 16 * kBlock - 1, kHighBoundary + 16 * kBlock + 1}, + {kBlock, kHighBoundary + kBlock}, + {kHighBoundary, depths.size()}, + {0, depths.size()}, + }; + check_one_interval_btree_ranges( + std::span(depths), + std::span>(ranges)); } -TEST(Rmq, FirstMinimumTieBreaking) { - const std::vector values = {7, 2, 2, 3, 2}; - const pixie::rmq::SparseTable sparse(values); - const pixie::rmq::SegmentTree segment(values); - const pixie::rmq::CartesianTreeRmq cartesian(values); +TEST(RmqCartesianTree, DuplicateHeavyArrays) { + const std::vector> cases = { + {5, 5, 5, 5, 5, 5, 5}, + {4, 1, 4, 1, 4, 1, 4, 1}, + {3, 2, 2, 2, 1, 1, 2, 1, 3}, + {9, 8, 9, 8, 9, 7, 7, 7, 8, 9}, + }; - EXPECT_EQ(sparse.arg_min(0, 4), 1u); - EXPECT_EQ(segment.arg_min(0, 4), 1u); - EXPECT_EQ(cartesian.arg_min(0, 4), 1u); - EXPECT_EQ(sparse.arg_min(2, 4), 2u); - EXPECT_EQ(segment.arg_min(2, 4), 2u); - EXPECT_EQ(cartesian.arg_min(2, 4), 2u); + for (std::size_t case_index = 0; case_index < cases.size(); ++case_index) { + SCOPED_TRACE(case_index); + const std::vector& values = cases[case_index]; + const pixie::rmq::CartesianTreeRmq cartesian(values); + expect_valid_bp_encoding(cartesian, values.size()); + check_all_ranges(cartesian, std::span(values), std::less()); + } } -TEST(Rmq, InvalidAndEmptyRanges) { - const std::vector values = {3, 1, 2}; - const pixie::rmq::SparseTable sparse(values); - const pixie::rmq::SegmentTree segment(values); - const pixie::rmq::CartesianTreeRmq cartesian(values); - - EXPECT_EQ(sparse.arg_min(2, 1), pixie::rmq::SparseTable::npos); - EXPECT_EQ(segment.arg_min(2, 1), pixie::rmq::SegmentTree::npos); - EXPECT_EQ(cartesian.arg_min(2, 1), pixie::rmq::CartesianTreeRmq::npos); - EXPECT_EQ(sparse.arg_min(0, values.size()), - pixie::rmq::SparseTable::npos); - EXPECT_EQ(segment.arg_min(0, values.size()), - pixie::rmq::SegmentTree::npos); - EXPECT_EQ(cartesian.arg_min(0, values.size()), - pixie::rmq::CartesianTreeRmq::npos); - EXPECT_EQ(sparse.range_min(2, 1), 0); - EXPECT_EQ(segment.range_min(2, 1), 0); - EXPECT_EQ(cartesian.range_min(2, 1), 0); - - const std::vector empty; - const pixie::rmq::SparseTable empty_sparse(empty); - const pixie::rmq::SegmentTree empty_segment(empty); - const pixie::rmq::CartesianTreeRmq empty_cartesian(empty); - EXPECT_TRUE(empty_sparse.empty()); - EXPECT_TRUE(empty_segment.empty()); - EXPECT_TRUE(empty_cartesian.empty()); - EXPECT_EQ(empty_sparse.arg_min(0, 0), pixie::rmq::SparseTable::npos); - EXPECT_EQ(empty_segment.arg_min(0, 0), pixie::rmq::SegmentTree::npos); - EXPECT_EQ(empty_cartesian.arg_min(0, 0), - pixie::rmq::CartesianTreeRmq::npos); -} - -TEST(Rmq, ComparatorCanSelectMaximum) { - const std::vector values = {1, 8, 3, 8, 4}; - const pixie::rmq::SparseTable> sparse(values); - const pixie::rmq::SegmentTree> segment(values); +TEST(RmqCartesianTree, ComparatorTiesKeepFirstMaximum) { + const std::vector values = {1, 8, 8, 7, 8, 3, 8}; const pixie::rmq::CartesianTreeRmq> cartesian(values); - check_all_ranges(sparse, std::span(values), std::greater()); - check_all_ranges(segment, std::span(values), std::greater()); + expect_valid_bp_encoding(cartesian, values.size()); check_all_ranges(cartesian, std::span(values), std::greater()); - EXPECT_EQ(sparse.arg_min(0, 4), 1u); - EXPECT_EQ(segment.arg_min(0, 4), 1u); - EXPECT_EQ(cartesian.arg_min(0, 4), 1u); -} - -TEST(RmqCartesianTree, MonotoneArrays) { - const std::vector increasing = {1, 2, 3, 4, 5, 6}; - const std::vector decreasing = {6, 5, 4, 3, 2, 1}; - const pixie::rmq::CartesianTreeRmq increasing_rmq(increasing); - const pixie::rmq::CartesianTreeRmq decreasing_rmq(decreasing); - - check_all_ranges(increasing_rmq, std::span(increasing), - std::less()); - check_all_ranges(decreasing_rmq, std::span(decreasing), - std::less()); + EXPECT_EQ(cartesian.arg_min(0, values.size()), 1u); + EXPECT_EQ(cartesian.arg_min(2, values.size()), 2u); } TEST(RmqCartesianTree, CopyAndMoveRebuildInternalSpans) { @@ -285,45 +1086,45 @@ TEST(RmqCartesianTree, CopyAndMoveRebuildInternalSpans) { check_all_ranges(moved, std::span(values), std::less()); } -TEST(RmqCartesianTree, EulerDepthsArePlusMinusOne) { +TEST(RmqCartesianTree, BpEncodingIsBalanced) { const std::vector values = {4, 1, 3, 2, 5}; const pixie::rmq::CartesianTreeRmq rmq(values); - const auto depths = rmq.euler_depths(); - - ASSERT_FALSE(depths.empty()); - for (std::size_t i = 1; i < depths.size(); ++i) { - EXPECT_EQ(std::abs(depths[i] - depths[i - 1]), 1); - } + expect_valid_bp_encoding(rmq, values.size()); } -TEST(Rmq, DifferentialRandom) { - std::mt19937_64 rng(42); - std::uniform_int_distribution value_dist(-50, 50); - for (std::size_t size = 1; size <= 257; size += 17) { +TEST(RmqCartesianTree, BoundarySizesExerciseBpBlocks) { + for (std::size_t size : {1u, 63u, 64u, 65u, 127u, 128u, 129u, 255u, 256u, + 257u, 511u, 512u, 513u}) { + SCOPED_TRACE(size); std::vector values(size); - std::generate(values.begin(), values.end(), - [&] { return value_dist(rng); }); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 37 + i / 3) % 23); + if (i % 11 == 0) { + values[i] = 7; + } + } - const pixie::rmq::SparseTable sparse(values); - const pixie::rmq::SegmentTree segment(values); - const pixie::rmq::CartesianTreeRmq cartesian(values); - check_all_ranges(sparse, std::span(values), std::less()); - check_all_ranges(segment, std::span(values), std::less()); - check_all_ranges(cartesian, std::span(values), std::less()); - } -} + const pixie::rmq::CartesianTreeRmq rmq(values); + expect_valid_bp_encoding(rmq, values.size()); -TEST(RmqBpPlusMinusOne, DifferentialRandomWalks) { - std::mt19937_64 rng(77); - for (std::size_t size = 1; size <= 257; size += 17) { - std::vector depths(size); - for (std::size_t i = 1; i < depths.size(); ++i) { - depths[i] = depths[i - 1] + ((rng() & 1u) != 0 ? 1 : -1); - } + const auto check_range = [&](std::size_t left, std::size_t right) { + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())) + << "range=[" << left << "," << right << ")"; + }; - const std::vector bits = pack_depth_deltas(depths); - const pixie::rmq::BpPlusMinusOneRmq<> rmq(bits, depths.size()); - check_all_arg_min_ranges(rmq, std::span(depths), - std::less()); + check_range(0, 1); + check_range(0, size); + check_range(size - 1, size); + check_range(size / 3, std::min(size, size / 3 + 9)); + check_range(size / 2, std::min(size, size / 2 + 17)); + + for (std::size_t boundary : {64u, 128u, 256u, 512u}) { + if (boundary < size) { + check_range(boundary - 1, std::min(size, boundary + 2)); + check_range(boundary / 2, std::min(size, boundary + 1)); + } + } } } From 061540dcf24061caa16261148552e088e2b96808 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Mon, 8 Jun 2026 14:31:37 +0300 Subject: [PATCH 09/27] Cleanup --- .../rmq/experimental/node_euler_btree_rmq.h | 503 +++-- include/pixie/rmq/segment_btree_xl.h | 1638 +++++++++++++++++ src/benchmarks/bench_rmq.cpp | 28 + src/tests/rmq_tests.cpp | 197 +- 4 files changed, 2177 insertions(+), 189 deletions(-) create mode 100644 include/pixie/rmq/segment_btree_xl.h diff --git a/include/pixie/rmq/experimental/node_euler_btree_rmq.h b/include/pixie/rmq/experimental/node_euler_btree_rmq.h index 156e04a..bdd77a6 100644 --- a/include/pixie/rmq/experimental/node_euler_btree_rmq.h +++ b/include/pixie/rmq/experimental/node_euler_btree_rmq.h @@ -23,16 +23,24 @@ namespace pixie::rmq::experimental { */ struct BpLeafSelectorTag {}; +/** + * @brief Low-level selector tag for prefix/suffix mask leaves. + */ +struct PrefixSuffixMaskLeafSelectorTag {}; + /** * @brief Experimental value RMQ with compact per-level BP selectors. * - * @details Values are split into 252-value leaves. Low nodes store 504 BP bits - * plus an 8-bit local-minimum offset in one 512-bit selector. Middle internal - * nodes use 192-way fanout and store 384 BP bits, a 64-bit absolute + * @details The default low level uses 252-value leaves with local BP selectors. + * The prefix/suffix mask low level supports 248-value leaves with an 8-bit + * local-minimum offset packed into 32 bytes, or 496-value leaves with a 16-bit + * local-minimum offset packed into 64 bytes. Middle internal nodes use 192-way + * fanout and store 384 BP bits, a 64-bit absolute * subtree-minimum position, and 64 bits of zero-rank prefix metadata in the - * selector. High nodes use 256-way fanout and are only the root and its child - * level, so there are at most 257 of them. High nodes add a sparse depth RMQ - * side table and cache each child slot's value range and subtree minimum. + * selector, with a side cache for each middle node's minimum value. High nodes + * use 256-way fanout and are only the root and its child level, so there are at + * most 257 of them. High nodes add sparse tables over child-minimum slots and + * cache each child slot's value range and subtree minimum. * * This backend is intentionally not included by `pixie/rmq.h`; include this * header directly while evaluating the experiment. @@ -41,11 +49,11 @@ struct BpLeafSelectorTag {}; * @tparam Compare Strict weak ordering used to choose minima. * @tparam Index Unsigned integer type used for stored positions. * @tparam LeafSize Number of original values per leaf. This experimental - * backend currently supports only 252. + * backend currently supports 252 with BP leaves and 248 or 496 with mask + * leaves. * @tparam Fanout Number of children per high internal node. This experimental * backend currently supports only 256. Middle internal nodes use 192. - * @tparam LowLevelSelector Compile-time low-level selector tag. The default - * BP leaf selector is the only implemented tag in this cleanup pass. + * @tparam LowLevelSelector Compile-time low-level selector tag. */ template , @@ -63,14 +71,19 @@ class NodeEulerBTreeRmq : public RmqBase, "NodeEulerBTreeRmq index type must be unsigned"); - static_assert(LeafSize == 252, - "experimental NodeEulerBTreeRmq currently requires 252-value " - "leaves"); + static constexpr bool kBpLeafSelector = + std::is_same_v; + static constexpr bool kMaskLeafSelector = + std::is_same_v; + static_assert((kBpLeafSelector && LeafSize == 252) || + (kMaskLeafSelector && (LeafSize == 248 || LeafSize == 496)), + "experimental NodeEulerBTreeRmq requires 252-value BP leaves " + "or 248/496-value prefix/suffix mask leaves"); static_assert(Fanout == 256, "experimental NodeEulerBTreeRmq currently requires 256-way " "internal nodes"); - static_assert(std::is_same_v, - "only BP low-level leaves are implemented"); + static_assert(kBpLeafSelector || kMaskLeafSelector, + "unsupported experimental low-level selector tag"); using Self = NodeEulerBTreeRmq; @@ -140,7 +153,7 @@ class NodeEulerBTreeRmq : public RmqBase(prefix_excess(position)); - } - void set_embedded_min_offset(std::size_t offset) { bp_bits_[kEmbeddedOffsetBit >> 6] = (bp_bits_[kEmbeddedOffsetBit >> 6] & ~kEmbeddedOffsetMask) | @@ -482,112 +496,159 @@ class NodeEulerBTreeRmq : public RmqBase + void build(std::size_t entry_count, EntryLess entry_less) { + if (entry_count > kMaskEntries) { + throw std::length_error( + "NodeEulerBTreeRmq prefix/suffix leaf selector too large"); } - log_count_ = std::bit_width(depth_count); - sparse_positions_.assign(log_count_ * depth_count, 0); - for (std::size_t position = 0; position < depth_count; ++position) { - sparse_positions_[position] = static_cast(position); + words_.fill(0); + if (entry_count == 0) { + set_embedded_min_offset(0); + return; } - for (std::size_t level = 1; level < log_count_; ++level) { - const std::size_t half_span = std::size_t{1} << (level - 1); - const std::size_t span = half_span << 1; - if (span > depth_count) { - break; + std::size_t prefix_best = 0; + set_mask_bit(0); + for (std::size_t slot = 1; slot < entry_count; ++slot) { + if (entry_less(slot, prefix_best)) { + prefix_best = slot; + set_mask_bit(slot); } - const std::size_t previous_offset = (level - 1) * depth_count; - const std::size_t current_offset = level * depth_count; - for (std::size_t position = 0; position + span <= depth_count; - ++position) { - sparse_positions_[current_offset + position] = better_depth_position( - sparse_positions_[previous_offset + position], - sparse_positions_[previous_offset + position + half_span]); + } + + std::size_t suffix_best = entry_count - 1; + set_mask_bit(suffix_best); + for (std::size_t slot = entry_count - 1; slot > 0;) { + --slot; + if (!entry_less(suffix_best, slot)) { + suffix_best = slot; + set_mask_bit(slot); } } + + set_embedded_min_offset(prefix_best); } - std::size_t arg_min(const Bp512Selector& selector, - std::size_t slot_left, + std::size_t arg_min(std::size_t slot_left, std::size_t slot_right, std::size_t entry_count) const { if (slot_left >= slot_right || slot_right > entry_count || - entry_count > kSelectorEntries) { + entry_count > kMaskEntries) { return npos; } if (slot_left + 1 == slot_right) { return slot_left; } - const std::size_t bit_count = 2 * entry_count; - const std::size_t first_close = selector.close_position(slot_left); - const std::size_t last_close = selector.close_position(slot_right - 1); - if (first_close > last_close || last_close >= bit_count) { - return npos; + const std::size_t min_offset = embedded_min_offset(); + if (slot_left <= min_offset && min_offset < slot_right) { + return min_offset; } - - const std::size_t shifted_min = - depth_arg_min(first_close + 1, last_close + 2); - if (shifted_min == npos || shifted_min == 0) { - return npos; + if (slot_left == 0 && slot_right <= min_offset) { + return previous_set_bit_before(slot_right); } - - const std::size_t zero_rank = selector.rank0_at(shifted_min, bit_count); - if (zero_rank == 0) { - return npos; + if (slot_right == entry_count && min_offset < slot_left) { + return next_set_bit_at_or_after(slot_left); } - const std::size_t entry = zero_rank - 1; - return entry < entry_count ? entry : npos; + return npos; + } + + void set_embedded_min_offset(std::size_t offset) { + words_[kOffsetWord] = + (words_[kOffsetWord] & kMaskWordMask) | + ((static_cast(offset) & kOffsetMask) << kOffsetShift); + } + + std::size_t embedded_min_offset() const { + return static_cast((words_[kOffsetWord] >> kOffsetShift) & + kOffsetMask); } private: - std::uint16_t better_depth_position(std::uint16_t left, - std::uint16_t right) const { - const std::int16_t left_depth = depths_[left]; - const std::int16_t right_depth = depths_[right]; - if (right_depth < left_depth) { - return right; + static constexpr std::size_t kMaskEntries = LeafSize; + static constexpr std::size_t kOffsetBits = LeafSize == 496 ? 16 : 8; + static constexpr std::size_t kPackedBits = kMaskEntries + kOffsetBits; + static constexpr std::size_t kWordCount = (kPackedBits + 63) / 64; + static constexpr std::size_t kOffsetWord = kMaskEntries / 64; + static constexpr std::size_t kOffsetShift = kMaskEntries & 63; + + static constexpr std::uint64_t low_bits_mask(std::size_t bits) { + if (bits == 0) { + return 0; } - if (left_depth < right_depth) { - return left; + if (bits >= 64) { + return std::numeric_limits::max(); } - return std::min(left, right); + return (std::uint64_t{1} << bits) - 1; + } + + static constexpr std::uint64_t kOffsetMask = low_bits_mask(kOffsetBits); + static constexpr std::uint64_t kMaskWordMask = low_bits_mask(kOffsetShift); + + static_assert(!kMaskLeafSelector || kPackedBits % 64 == 0); + static_assert(!kMaskLeafSelector || kOffsetShift + kOffsetBits == 64); + + void set_mask_bit(std::size_t slot) { + words_[slot >> 6] |= std::uint64_t{1} << (slot & 63); } - std::size_t depth_arg_min(std::size_t left, std::size_t right) const { - if (left >= right || right > depth_count_) { + std::uint64_t mask_word(std::size_t word) const { + return word == kOffsetWord ? words_[word] & kMaskWordMask : words_[word]; + } + + std::size_t previous_set_bit_before(std::size_t limit) const { + if (limit == 0) { return npos; } - const std::size_t length = right - left; - const std::size_t level = std::bit_width(length) - 1; - const std::size_t span = std::size_t{1} << level; - const std::size_t offset = level * depth_count_; - return better_depth_position(sparse_positions_[offset + left], - sparse_positions_[offset + right - span]); - } - - std::vector depths_; - std::vector sparse_positions_; - std::size_t depth_count_ = 0; - std::size_t log_count_ = 0; + + std::size_t word = (limit - 1) >> 6; + std::uint64_t bits = + mask_word(word) & first_bits_mask(((limit - 1) & 63) + 1); + while (true) { + if (bits != 0) { + return word * 64 + 63 - std::countl_zero(bits); + } + if (word == 0) { + break; + } + --word; + bits = mask_word(word); + } + return npos; + } + + std::size_t next_set_bit_at_or_after(std::size_t slot) const { + std::size_t word = slot >> 6; + std::uint64_t bits = mask_word(word) & ~first_bits_mask(slot & 63); + while (word < kWordCount) { + if (bits != 0) { + const std::size_t result = word * 64 + std::countr_zero(bits); + return result < kMaskEntries ? result : npos; + } + ++word; + bits = word < kWordCount ? mask_word(word) : 0; + } + return npos; + } + + std::array words_{}; }; + static_assert(!kMaskLeafSelector || sizeof(PrefixSuffixMaskLeafSelector) == + (LeafSize == 496 ? 64 : 32)); + static_assert(!kMaskLeafSelector || alignof(PrefixSuffixMaskLeafSelector) == + (LeafSize == 496 ? 64 : 32)); + + using LeafSelector = std::conditional_t; + bool missing_position(std::size_t position) const { if constexpr (kInvalidIndexEqualsNpos) { return position == npos; @@ -597,38 +658,94 @@ class NodeEulerBTreeRmq : public RmqBase(subtree_min_position(level - 1, child)); } - build_high_sparse_min_slots(high_flat, count); + build_high_sparse_min_slots(level, node, count); } selector.build(count, [&](std::size_t left, std::size_t right) { if (high_level) { - return strictly_better_position( - high_child_metadata_at(high_flat, left).min_position, - high_child_metadata_at(high_flat, right).min_position); + return strictly_better_high_child_slot(level, node, left, right); } - return strictly_better_position( - subtree_min_position(level - 1, first_child + left), - subtree_min_position(level - 1, first_child + right)); + return strictly_better_subtree_child_slot(level, node, left, right); }); const std::size_t slot = selector.arg_min(0, count, count); @@ -758,9 +874,10 @@ class NodeEulerBTreeRmq : public RmqBase(min_position); - mutable_high_depth_selector_at(level, node).build(selector, count); + high_min_values_.push_back(values_[min_position]); } else { selector.set_embedded_min_position(min_position); + medium_min_values_.push_back(values_[min_position]); selector.build_zero_prefix_metadata(2 * count); } } @@ -788,18 +905,28 @@ class NodeEulerBTreeRmq : public RmqBase= slot_right) { - return npos; + return {}; } const std::size_t count = entry_count(level, node); @@ -918,7 +1045,7 @@ class NodeEulerBTreeRmq : public RmqBase= slot_right) { - return npos; + return {}; } const std::size_t slot = @@ -991,30 +1118,29 @@ class NodeEulerBTreeRmq : public RmqBase= right) { - return npos; + return {}; } const std::size_t begin = node_value_begin(level, node); const std::size_t end = node_value_end(level, node); if (left <= begin && end <= right) { - return subtree_min_position(level, node); + return subtree_min_candidate(level, node); } if (level == 0) { - return leaf_range_min(node, left, right); + return value_candidate(leaf_range_min(node, left, right)); } const std::size_t child_level = level - 1; @@ -1064,10 +1190,17 @@ class NodeEulerBTreeRmq : public RmqBase(slot); @@ -1162,12 +1296,13 @@ class NodeEulerBTreeRmq : public RmqBase(better_high_child_slot( - high_flat, previous[slot], previous[slot + half_span])); + level, node, previous[slot], previous[slot + half_span])); } } } - std::size_t high_sparse_arg_min(std::size_t high_flat, + std::size_t high_sparse_arg_min(std::size_t level, + std::size_t node, std::size_t slot_left, std::size_t slot_right, std::size_t count) const { @@ -1179,31 +1314,23 @@ class NodeEulerBTreeRmq : public RmqBase values_; Compare compare_; - std::vector leaf_selectors_; + std::vector leaf_selectors_; std::vector medium_selectors_; + std::vector medium_min_values_; std::vector high_selectors_; - std::vector high_depth_selectors_; std::vector high_min_positions_; + std::vector high_min_values_; std::vector high_child_metadata_; std::vector high_sparse_min_slots_; std::vector medium_level_offsets_; diff --git a/include/pixie/rmq/segment_btree_xl.h b/include/pixie/rmq/segment_btree_xl.h new file mode 100644 index 0000000..1558312 --- /dev/null +++ b/include/pixie/rmq/segment_btree_xl.h @@ -0,0 +1,1638 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief Low-level selector tag for BP-based 252-value leaves. + */ +struct BpLeafSelectorTag {}; + +/** + * @brief Low-level selector tag for prefix/suffix mask leaves. + */ +struct PrefixSuffixMaskLeafSelectorTag {}; + +/** + * @brief Segment B-tree RMQ with compact per-level selectors and XL leaves. + * + * @details This is a static, non-owning value-RMQ index over an external value + * array. Values are split into leaves, leaves are grouped into a B-tree, and + * each node stores a selector over the minima of its immediate children. Query + * ranges use the library-wide half-open convention `[left, right)`, invalid + * ranges return `npos`, and equal values resolve to the smaller original + * position. + * + * The default low level uses 496-value prefix/suffix mask leaves. A leaf stores + * one bit for each prefix/suffix record position plus a 16-bit local-minimum + * offset in one 64-byte object. Prefix or suffix ranges can be answered from + * this mask; interior partial ranges fall back to a linear scan of the original + * values. The same implementation also supports 248-value mask leaves and + * 252-value BP leaves for controlled experiments. + * + * Middle internal nodes use 192-way fanout. Their selector is a 512-bit local + * Cartesian-tree balanced-parentheses encoding over child minima: 384 BP bits, + * a 64-bit absolute subtree-minimum position, and 64 bits of zero-rank prefix + * metadata. Middle-node minimum values are cached in a side vector so comparing + * candidates does not require repeatedly descending to leaves. + * + * High nodes use 256-way fanout and are limited to the root and its child + * level, so there are at most 257 high nodes. They keep the same local BP + * selector, cache child value ranges and child subtree minima, and add sparse + * tables over child-minimum slots. This spends more space at the top of the + * tree to reduce work on wide queries while keeping the high-level metadata + * small enough to stay cache-resident. + * + * Querying starts at the lowest node that covers both endpoint leaves. At each + * internal node, the selector is asked for the best child over the intersecting + * child-slot range. If that child is fully covered, or its stored subtree + * minimum lies inside the query, that candidate is final for the node. + * Otherwise only the affected border child is corrected recursively and + * compared with the middle full-child range. All comparisons use `(value, + * position)` semantics so traversal order cannot change tie-breaking. + * + * @tparam T Value type in the indexed array. + * @tparam Compare Strict weak ordering used to choose minima. + * @tparam Index Unsigned integer type used for stored positions. + * @tparam LeafSize Number of original values per leaf. This backend currently + * supports 252 with BP leaves and 248 or 496 with mask leaves. + * @tparam Fanout Number of children per high internal node. This backend + * currently supports only 256. Middle internal nodes use 192. + * @tparam LowLevelSelector Compile-time low-level selector tag. + */ +template , + class Index = std::size_t, + std::size_t LeafSize = 496, + std::size_t Fanout = 256, + class LowLevelSelector = PrefixSuffixMaskLeafSelectorTag> +class SegmentBTreeXl + : public RmqBase< + SegmentBTreeXl, + T> { + public: + static_assert(std::is_unsigned_v, + "SegmentBTreeXl index type must be unsigned"); + static constexpr bool kBpLeafSelector = + std::is_same_v; + static constexpr bool kMaskLeafSelector = + std::is_same_v; + static_assert((kBpLeafSelector && LeafSize == 252) || + (kMaskLeafSelector && (LeafSize == 248 || LeafSize == 496)), + "SegmentBTreeXl requires 252-value BP leaves " + "or 248/496-value prefix/suffix mask leaves"); + static_assert(Fanout == 256, + "SegmentBTreeXl currently requires 256-way internal nodes"); + static_assert(kBpLeafSelector || kMaskLeafSelector, + "unsupported SegmentBTreeXl low-level selector tag"); + + using Self = + SegmentBTreeXl; + + static constexpr std::size_t npos = RmqBase::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + static constexpr std::size_t kLeafSize = LeafSize; + static constexpr std::size_t kFanout = Fanout; + static constexpr std::size_t kMiddleFanout = 192; + + /** + * @brief Construct an empty RMQ index. + */ + SegmentBTreeXl() = default; + + /** + * @brief Copy an RMQ index while preserving its non-owning value span. + */ + SegmentBTreeXl(const SegmentBTreeXl&) = default; + + /** + * @brief Move an RMQ index while preserving selector and cache storage. + */ + SegmentBTreeXl(SegmentBTreeXl&&) noexcept = default; + + /** + * @brief Copy-assign an RMQ index and its cached metadata. + */ + SegmentBTreeXl& operator=(const SegmentBTreeXl&) = default; + + /** + * @brief Move-assign an RMQ index and its cached metadata. + */ + SegmentBTreeXl& operator=(SegmentBTreeXl&&) noexcept = default; + + /** + * @brief Build a segment B-tree RMQ index over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values keep the smaller original position as the answer. + * + * @param values Values to index. + * @param compare Ordering used to choose minima. + * @throws std::length_error if @p Index cannot represent all positions. + */ + explicit SegmentBTreeXl(std::span values, + Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + /** + * @brief Return the number of indexed values. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + * + * @details Empty or invalid ranges return `npos`. Ties are reduced by + * comparing `(value, position)`, so traversal order cannot change first-min + * semantics. + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size() || level_sizes_.empty()) { + return npos; + } + + const std::size_t root_level = level_count() - 1; + if (left == 0 && right == values_.size()) { + return subtree_min_position(root_level, 0); + } + + const std::size_t left_leaf = leaf_for_value(left); + const std::size_t right_leaf = leaf_for_value(right - 1); + if (left_leaf == right_leaf) { + return leaf_range_min(left_leaf, left, right); + } + + const auto [level, node_index] = covering_node(left_leaf, right_leaf); + return query_node(level, node_index, left, right).position; + } + + private: + static constexpr std::size_t kSelectorEntries = 256; + static constexpr std::size_t kSelectorBits = 2 * kSelectorEntries; + static constexpr std::size_t kSelectorWords = kSelectorBits / 64; + static constexpr std::size_t kEmbeddedOffsetEntries = 252; + static constexpr std::size_t kEmbeddedPositionEntries = 192; + static constexpr std::size_t kEmbeddedOffsetBit = 2 * kEmbeddedOffsetEntries; + static constexpr std::size_t kEmbeddedPositionBit = + 2 * kEmbeddedPositionEntries; + static constexpr std::size_t kMiddleZeroPrefixWord = 7; + static constexpr std::size_t kHighSparseLevels = + static_cast(std::bit_width(Fanout)); + static constexpr std::size_t kHighSparseSlotsPerNode = + kHighSparseLevels * Fanout; + static constexpr std::size_t kLeafLinearScanThreshold = 64; + static constexpr std::size_t kLeafAvx2ScanThreshold = 16; + static constexpr std::uint64_t kEmbeddedOffsetMask = + std::uint64_t{0xFF} << (kEmbeddedOffsetBit & 63); + static constexpr bool kInvalidIndexEqualsNpos = + static_cast(invalid_index) == npos; + + static_assert(kEmbeddedOffsetBit + 8 == kSelectorBits); + static_assert(kEmbeddedPositionBit + 128 == kSelectorBits); + static_assert(!kBpLeafSelector || LeafSize <= kEmbeddedOffsetEntries); + static_assert(kMiddleFanout <= kEmbeddedPositionEntries); + + struct HighChildMetadata { + std::size_t value_begin = 0; + std::size_t value_end = 0; + Index min_position = invalid_index; + }; + + struct MinCandidate { + std::size_t position = npos; + const T* value = nullptr; + }; + + class alignas(64) Bp512Selector { + public: + /** + * @brief Construct an empty packed BP selector. + */ + Bp512Selector() = default; + + /** + * @brief Build a stable local Cartesian-tree BP selector. + * + * @details The selector indexes @p entry_count logical entries. The + * supplied comparator returns whether the left entry is strictly better + * than the right entry. Equal entries keep the smaller slot by using the + * same stable rule as the value-level RMQ. + */ + template + void build(std::size_t entry_count, EntryLess entry_less) { + if (entry_count > kSelectorEntries) { + throw std::length_error("SegmentBTreeXl local selector too large"); + } + + bp_bits_.fill(0); + if (entry_count == 0) { + return; + } + + std::array stack{}; + std::size_t stack_size = 0; + std::size_t write_position = 2 * entry_count; + + for (std::size_t i = entry_count; i-- > 0;) { + while (stack_size != 0 && !entry_less(stack[stack_size - 1], i)) { + --stack_size; + prepend_bp_bit(write_position, true); + } + stack[stack_size++] = static_cast(i); + prepend_bp_bit(write_position, false); + } + + while (write_position != 0) { + prepend_bp_bit(write_position, true); + } + } + + /** + * @brief Return the first minimum slot in a local entry range. + * + * @details This path is used by BP leaves and high nodes whose BP words do + * not share storage with zero-rank prefix metadata. + */ + std::size_t arg_min(std::size_t slot_left, + std::size_t slot_right, + std::size_t entry_count) const { + if (slot_left >= slot_right || slot_right > entry_count || + entry_count > kSelectorEntries) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t bit_count = 2 * entry_count; + const std::size_t first_close = close_position(slot_left); + const std::size_t last_close = close_position(slot_right - 1); + if (first_close > last_close || last_close >= bit_count) { + return npos; + } + + const std::size_t shifted_min = + depth_arg_min(first_close + 1, last_close + 2, bit_count); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + + const std::size_t zero_rank = rank0_at(shifted_min, bit_count); + if (zero_rank == 0) { + return npos; + } + const std::size_t entry = zero_rank - 1; + return entry < entry_count ? entry : npos; + } + + /** + * @brief Return the first minimum slot using packed zero-rank metadata. + * + * @details Middle nodes overwrite the last selector word with zero-rank + * prefixes, so rank/select operations must consult the packed metadata + * instead of treating all eight words as raw BP bits. + */ + std::size_t arg_min_with_zero_prefix(std::size_t slot_left, + std::size_t slot_right, + std::size_t entry_count) const { + if (slot_left >= slot_right || slot_right > entry_count || + entry_count > kEmbeddedPositionEntries) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t bit_count = 2 * entry_count; + const std::size_t first_close = + close_position_with_zero_prefix(slot_left, bit_count); + const std::size_t last_close = + close_position_with_zero_prefix(slot_right - 1, bit_count); + if (first_close > last_close || last_close >= bit_count) { + return npos; + } + + const std::size_t shifted_min = depth_arg_min_with_zero_prefix( + first_close + 1, last_close + 2, bit_count); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + + const std::size_t zero_rank = + rank0_at_with_zero_prefix(shifted_min, bit_count); + if (zero_rank == 0) { + return npos; + } + const std::size_t entry = zero_rank - 1; + return entry < entry_count ? entry : npos; + } + + /** + * @brief Return the BP close position for a local entry slot. + */ + std::size_t close_position(std::size_t slot) const { + return select0_512(bp_bits_.data(), slot); + } + + /** + * @brief Count zero bits before @p position in the raw BP sequence. + */ + std::size_t rank0_at(std::size_t position, std::size_t bit_count) const { + position = std::min(position, bit_count); + return position - rank_512(bp_bits_.data(), position); + } + + /** + * @brief Pack a leaf-local minimum offset into the spare BP bits. + */ + void set_embedded_min_offset(std::size_t offset) { + bp_bits_[kEmbeddedOffsetBit >> 6] = + (bp_bits_[kEmbeddedOffsetBit >> 6] & ~kEmbeddedOffsetMask) | + ((static_cast(offset) & 0xFFu) + << (kEmbeddedOffsetBit & 63)); + } + + /** + * @brief Return the leaf-local minimum offset stored in the selector. + */ + std::uint8_t embedded_min_offset() const { + return static_cast( + (bp_bits_[kEmbeddedOffsetBit >> 6] & kEmbeddedOffsetMask) >> + (kEmbeddedOffsetBit & 63)); + } + + /** + * @brief Store a node's absolute subtree minimum position. + */ + void set_embedded_min_position(std::size_t position) { + bp_bits_[kEmbeddedPositionBit >> 6] = + static_cast(position); + } + + /** + * @brief Return the absolute subtree minimum position stored in a node. + */ + std::size_t embedded_min_position() const { + return static_cast(bp_bits_[kEmbeddedPositionBit >> 6]); + } + + /** + * @brief Pack per-word zero-count prefixes for middle-node selectors. + */ + void build_zero_prefix_metadata(std::size_t bit_count) { + const std::size_t word_count = (bit_count + 63) / 64; + std::uint64_t packed = 0; + std::size_t zeros = 0; + for (std::size_t word = 0; word <= word_count; ++word) { + packed |= (static_cast(zeros) & 0xFFu) << (8 * word); + if (word == word_count) { + break; + } + const std::size_t word_begin = word * 64; + const std::size_t word_bits = + std::min(64, bit_count - word_begin); + const std::uint64_t bits = bp_bits_[word] & first_bits_mask(word_bits); + zeros += word_bits - std::popcount(bits); + } + bp_bits_[kMiddleZeroPrefixWord] = packed; + } + + private: + /** + * @brief Prepend one BP bit while building the sequence right-to-left. + */ + std::size_t prepend_bp_bit(std::size_t& write_position, bool bit) { + --write_position; + if (bit) { + bp_bits_[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); + } + return write_position; + } + + /** + * @brief Return open-minus-close excess before a raw BP position. + */ + int prefix_excess(std::size_t position) const { + position = std::min(position, kSelectorBits); + const std::size_t ones = rank_512(bp_bits_.data(), position); + return static_cast(2 * ones) - static_cast(position); + } + + /** + * @brief Return the packed zero-count prefix at a 64-bit word boundary. + */ + std::size_t zero_prefix_at_word(std::size_t word) const { + return static_cast(bp_bits_[kMiddleZeroPrefixWord] >> + (8 * word)); + } + + /** + * @brief Count close parentheses before @p position using packed prefixes. + */ + std::size_t rank0_at_with_zero_prefix(std::size_t position, + std::size_t bit_count) const { + position = std::min(position, bit_count); + const std::size_t full_words = position >> 6; + std::size_t zeros = zero_prefix_at_word(full_words); + const std::size_t tail_bits = position & 63; + if (tail_bits != 0) { + const std::uint64_t tail = + bp_bits_[full_words] & first_bits_mask(tail_bits); + zeros += tail_bits - std::popcount(tail); + } + return zeros; + } + + /** + * @brief Return BP excess when the last word stores zero-rank metadata. + */ + int prefix_excess_with_zero_prefix(std::size_t position, + std::size_t bit_count) const { + position = std::min(position, bit_count); + return static_cast(position) - + 2 * static_cast( + rank0_at_with_zero_prefix(position, bit_count)); + } + + /** + * @brief Select a close parenthesis using packed per-word zero counts. + */ + std::size_t close_position_with_zero_prefix(std::size_t slot, + std::size_t bit_count) const { + const std::size_t word_count = (bit_count + 63) / 64; + for (std::size_t word = 0; word < word_count; ++word) { + const std::size_t next_zero_prefix = zero_prefix_at_word(word + 1); + if (next_zero_prefix <= slot) { + continue; + } + const std::size_t local_rank = slot - zero_prefix_at_word(word); + const std::size_t word_begin = word * 64; + const std::size_t word_bits = + std::min(64, bit_count - word_begin); + const std::uint64_t zeros = + (~bp_bits_[word]) & first_bits_mask(word_bits); + return word_begin + select_64(zeros, local_rank); + } + return npos; + } + + /** + * @brief Return the position of the minimum BP depth in a depth range. + */ + std::size_t depth_arg_min(std::size_t left, + std::size_t right, + std::size_t bit_count) const { + const std::size_t depth_count = bit_count + 1; + if (left >= right || right > depth_count) { + return npos; + } + + std::size_t position = left; + int best_depth = prefix_excess(position); + std::size_t best_position = position; + + while (position < right) { + const std::size_t chunk_begin = (position / 128) * 128; + const std::size_t local_left = position - chunk_begin; + const std::size_t local_right = + std::min(right - 1, chunk_begin + 128) - chunk_begin; + + int candidate_depth; + std::size_t candidate_position; + if (chunk_begin >= bit_count) { + candidate_depth = prefix_excess(chunk_begin); + candidate_position = chunk_begin; + } else { + const std::size_t word = chunk_begin >> 6; + const ExcessResult result = + excess_min_128(bp_bits_.data() + word, local_left, local_right); + candidate_depth = prefix_excess(chunk_begin) + result.min_excess; + candidate_position = chunk_begin + result.offset; + } + + if (candidate_depth < best_depth) { + best_depth = candidate_depth; + best_position = candidate_position; + } + + position = chunk_begin + local_right + 1; + } + + return best_position; + } + + /** + * @brief Return the minimum-depth position for middle-node packed BP data. + */ + std::size_t depth_arg_min_with_zero_prefix(std::size_t left, + std::size_t right, + std::size_t bit_count) const { + const std::size_t depth_count = bit_count + 1; + if (left >= right || right > depth_count) { + return npos; + } + + std::size_t position = left; + int best_depth = prefix_excess_with_zero_prefix(position, bit_count); + std::size_t best_position = position; + + while (position < right) { + const std::size_t chunk_begin = (position / 128) * 128; + const std::size_t local_left = position - chunk_begin; + const std::size_t local_right = + std::min(right - 1, chunk_begin + 128) - chunk_begin; + + int candidate_depth; + std::size_t candidate_position; + if (chunk_begin >= bit_count) { + candidate_depth = + prefix_excess_with_zero_prefix(chunk_begin, bit_count); + candidate_position = chunk_begin; + } else { + const std::size_t word = chunk_begin >> 6; + const ExcessResult result = + excess_min_128(bp_bits_.data() + word, local_left, local_right); + candidate_depth = + prefix_excess_with_zero_prefix(chunk_begin, bit_count) + + result.min_excess; + candidate_position = chunk_begin + result.offset; + } + + if (candidate_depth < best_depth) { + best_depth = candidate_depth; + best_position = candidate_position; + } + + position = chunk_begin + local_right + 1; + } + + return best_position; + } + + std::array bp_bits_{}; + }; + + static_assert(sizeof(Bp512Selector) == 64); + + class alignas(LeafSize == 496 ? 64 : 32) PrefixSuffixMaskLeafSelector { + public: + /** + * @brief Construct an empty prefix/suffix mask selector. + */ + PrefixSuffixMaskLeafSelector() = default; + + /** + * @brief Build prefix/suffix record masks and store the local minimum. + * + * @details A bit is set for every prefix-minimum record and every + * suffix-minimum record. Equal suffix values set the smaller slot so + * first-minimum tie-breaking is preserved. + */ + template + void build(std::size_t entry_count, EntryLess entry_less) { + if (entry_count > kMaskEntries) { + throw std::length_error( + "SegmentBTreeXl prefix/suffix leaf selector too large"); + } + + words_.fill(0); + if (entry_count == 0) { + set_embedded_min_offset(0); + return; + } + + std::size_t prefix_best = 0; + set_mask_bit(0); + for (std::size_t slot = 1; slot < entry_count; ++slot) { + if (entry_less(slot, prefix_best)) { + prefix_best = slot; + set_mask_bit(slot); + } + } + + std::size_t suffix_best = entry_count - 1; + set_mask_bit(suffix_best); + for (std::size_t slot = entry_count - 1; slot > 0;) { + --slot; + if (!entry_less(suffix_best, slot)) { + suffix_best = slot; + set_mask_bit(slot); + } + } + + set_embedded_min_offset(prefix_best); + } + + /** + * @brief Return a local minimum slot when the mask can answer directly. + * + * @details The mask answers full-leaf ranges, prefixes before the global + * leaf minimum, and suffixes after the global leaf minimum. Interior ranges + * that cannot be represented by prefix/suffix records return `npos` so the + * caller can scan original values. + */ + std::size_t arg_min(std::size_t slot_left, + std::size_t slot_right, + std::size_t entry_count) const { + if (slot_left >= slot_right || slot_right > entry_count || + entry_count > kMaskEntries) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t min_offset = embedded_min_offset(); + if (slot_left <= min_offset && min_offset < slot_right) { + return min_offset; + } + if (slot_left == 0 && slot_right <= min_offset) { + return previous_set_bit_before(slot_right); + } + if (slot_right == entry_count && min_offset < slot_left) { + return next_set_bit_at_or_after(slot_left); + } + return npos; + } + + /** + * @brief Pack the local minimum offset into the high bits of the mask. + */ + void set_embedded_min_offset(std::size_t offset) { + words_[kOffsetWord] = + (words_[kOffsetWord] & kMaskWordMask) | + ((static_cast(offset) & kOffsetMask) << kOffsetShift); + } + + /** + * @brief Return the local minimum offset packed into the mask words. + */ + std::size_t embedded_min_offset() const { + return static_cast((words_[kOffsetWord] >> kOffsetShift) & + kOffsetMask); + } + + private: + static constexpr std::size_t kMaskEntries = LeafSize; + static constexpr std::size_t kOffsetBits = LeafSize == 496 ? 16 : 8; + static constexpr std::size_t kPackedBits = kMaskEntries + kOffsetBits; + static constexpr std::size_t kWordCount = (kPackedBits + 63) / 64; + static constexpr std::size_t kOffsetWord = kMaskEntries / 64; + static constexpr std::size_t kOffsetShift = kMaskEntries & 63; + + /** + * @brief Return a mask with the lowest @p bits set. + */ + static constexpr std::uint64_t low_bits_mask(std::size_t bits) { + if (bits == 0) { + return 0; + } + if (bits >= 64) { + return std::numeric_limits::max(); + } + return (std::uint64_t{1} << bits) - 1; + } + + static constexpr std::uint64_t kOffsetMask = low_bits_mask(kOffsetBits); + static constexpr std::uint64_t kMaskWordMask = low_bits_mask(kOffsetShift); + + static_assert(!kMaskLeafSelector || kPackedBits % 64 == 0); + static_assert(!kMaskLeafSelector || kOffsetShift + kOffsetBits == 64); + + /** + * @brief Set the prefix/suffix-record bit for a local leaf slot. + */ + void set_mask_bit(std::size_t slot) { + words_[slot >> 6] |= std::uint64_t{1} << (slot & 63); + } + + /** + * @brief Return a mask word with embedded offset bits hidden. + */ + std::uint64_t mask_word(std::size_t word) const { + return word == kOffsetWord ? words_[word] & kMaskWordMask : words_[word]; + } + + /** + * @brief Return the last set record bit strictly before @p limit. + */ + std::size_t previous_set_bit_before(std::size_t limit) const { + if (limit == 0) { + return npos; + } + + std::size_t word = (limit - 1) >> 6; + std::uint64_t bits = + mask_word(word) & first_bits_mask(((limit - 1) & 63) + 1); + while (true) { + if (bits != 0) { + return word * 64 + 63 - std::countl_zero(bits); + } + if (word == 0) { + break; + } + --word; + bits = mask_word(word); + } + return npos; + } + + /** + * @brief Return the first set record bit at or after @p slot. + */ + std::size_t next_set_bit_at_or_after(std::size_t slot) const { + std::size_t word = slot >> 6; + std::uint64_t bits = mask_word(word) & ~first_bits_mask(slot & 63); + while (word < kWordCount) { + if (bits != 0) { + const std::size_t result = word * 64 + std::countr_zero(bits); + return result < kMaskEntries ? result : npos; + } + ++word; + bits = word < kWordCount ? mask_word(word) : 0; + } + return npos; + } + + std::array words_{}; + }; + + static_assert(!kMaskLeafSelector || sizeof(PrefixSuffixMaskLeafSelector) == + (LeafSize == 496 ? 64 : 32)); + static_assert(!kMaskLeafSelector || alignof(PrefixSuffixMaskLeafSelector) == + (LeafSize == 496 ? 64 : 32)); + + using LeafSelector = std::conditional_t; + + /** + * @brief Return whether a stored position is one of the missing sentinels. + */ + bool missing_position(std::size_t position) const { + if constexpr (kInvalidIndexEqualsNpos) { + return position == npos; + } else { + return position == npos || + position == static_cast(invalid_index); + } + } + + /** + * @brief Wrap an original value position as a comparable candidate. + */ + MinCandidate value_candidate(std::size_t position) const { + if (missing_position(position)) { + return {}; + } + return {position, values_.data() + position}; + } + + /** + * @brief Return a node's cached subtree-minimum candidate. + */ + MinCandidate subtree_min_candidate(std::size_t level, + std::size_t node) const { + const std::size_t position = subtree_min_position(level, node); + if (missing_position(position)) { + return {}; + } + return {position, &subtree_min_value(level, node)}; + } + + /** + * @brief Return a child slot's subtree-minimum candidate. + */ + MinCandidate subtree_child_min_candidate(std::size_t level, + std::size_t node, + std::size_t slot) const { + const std::size_t child_level = level - 1; + return subtree_min_candidate(child_level, + node * fanout_at_level(level) + slot); + } + + /** + * @brief Return a high-node child candidate using high-child metadata. + */ + MinCandidate high_child_min_candidate(std::size_t level, + std::size_t node, + std::size_t slot) const { + const std::size_t child_level = level - 1; + const std::size_t child = node * fanout_at_level(level) + slot; + const std::size_t position = + high_child_metadata_at(high_flat_index(level, node), slot).min_position; + if (missing_position(position)) { + return {}; + } + return {position, &subtree_min_value(child_level, child)}; + } + + /** + * @brief Return whether @p left is strictly better than @p right. + */ + bool strictly_better_candidate(MinCandidate left, MinCandidate right) const { + if (missing_position(left.position)) { + return false; + } + if (missing_position(right.position)) { + return true; + } + return compare_(*left.value, *right.value); + } + + /** + * @brief Choose the better candidate using value, then smaller position. + */ + MinCandidate better_candidate(MinCandidate left, MinCandidate right) const { + if (missing_position(left.position)) { + return right; + } + if (missing_position(right.position)) { + return left; + } + if (compare_(*right.value, *left.value)) { + return right; + } + if (compare_(*left.value, *right.value)) { + return left; + } + return right.position < left.position ? right : left; + } + + /** + * @brief Compare two regular child slots while building a local selector. + */ + bool strictly_better_subtree_child_slot(std::size_t level, + std::size_t node, + std::size_t left_slot, + std::size_t right_slot) const { + return strictly_better_candidate( + subtree_child_min_candidate(level, node, left_slot), + subtree_child_min_candidate(level, node, right_slot)); + } + + /** + * @brief Compare two high-node child slots while building high metadata. + */ + bool strictly_better_high_child_slot(std::size_t level, + std::size_t node, + std::size_t left_slot, + std::size_t right_slot) const { + return strictly_better_candidate( + high_child_min_candidate(level, node, left_slot), + high_child_min_candidate(level, node, right_slot)); + } + + /** + * @brief Build all levels, selectors, and cached minimum metadata. + */ + void build() { + leaf_selectors_.clear(); + medium_selectors_.clear(); + medium_min_values_.clear(); + high_selectors_.clear(); + high_min_positions_.clear(); + high_min_values_.clear(); + high_child_metadata_.clear(); + high_sparse_min_slots_.clear(); + medium_level_offsets_.clear(); + high_level_offsets_.clear(); + level_sizes_.clear(); + level_value_spans_.clear(); + level_fanouts_.clear(); + high_level_begin_ = std::numeric_limits::max(); + if (values_.empty()) { + return; + } + if (values_.size() > static_cast(invalid_index)) { + throw std::length_error("SegmentBTreeXl index type is too small"); + } + + initialize_layout((values_.size() + LeafSize - 1) / LeafSize); + for (std::size_t leaf = 0; leaf < level_sizes_[0]; ++leaf) { + build_leaf(leaf); + } + + for (std::size_t level = 1; level < level_count(); ++level) { + for (std::size_t node = 0; node < level_sizes_[level]; ++node) { + build_internal_node(level, node); + } + } + } + + /** + * @brief Compute level sizes, fanouts, flat offsets, and cache storage. + * + * @details The top two tree levels are marked as high levels. Levels below + * them use the middle-node layout. Level zero always stores leaf selectors. + */ + void initialize_layout(std::size_t leaf_count) { + level_sizes_.push_back(leaf_count); + level_value_spans_.push_back(LeafSize); + level_fanouts_.push_back(0); + + std::size_t current_count = leaf_count; + std::size_t current_span = LeafSize; + while (current_count > Fanout * Fanout) { + level_fanouts_.push_back(kMiddleFanout); + current_count = ceil_div(current_count, kMiddleFanout); + current_span = saturating_product(current_span, kMiddleFanout); + level_sizes_.push_back(current_count); + level_value_spans_.push_back(current_span); + } + while (current_count > 1) { + level_fanouts_.push_back(Fanout); + current_count = ceil_div(current_count, Fanout); + current_span = saturating_product(current_span, Fanout); + level_sizes_.push_back(current_count); + level_value_spans_.push_back(current_span); + } + + leaf_selectors_.resize(level_sizes_[0]); + + medium_level_offsets_.assign(level_count(), 0); + high_level_offsets_.assign(level_count(), 0); + if (level_count() <= 1) { + high_level_begin_ = std::numeric_limits::max(); + return; + } + + const std::size_t root_level = level_count() - 1; + high_level_begin_ = root_level > 1 ? root_level - 1 : 1; + + std::size_t medium_node_count = 0; + for (std::size_t level = 1; level < high_level_begin_; ++level) { + medium_level_offsets_[level] = medium_node_count; + medium_node_count += level_sizes_[level]; + } + medium_selectors_.resize(medium_node_count); + medium_min_values_.reserve(medium_node_count); + + std::size_t high_node_count = 0; + for (std::size_t level = high_level_begin_; level < level_count(); + ++level) { + high_level_offsets_[level] = high_node_count; + high_node_count += level_sizes_[level]; + } + high_selectors_.resize(high_node_count); + high_min_positions_.resize(high_node_count, invalid_index); + high_min_values_.reserve(high_node_count); + high_child_metadata_.resize(high_node_count * Fanout); + high_sparse_min_slots_.resize(high_node_count * kHighSparseSlotsPerNode); + } + + /** + * @brief Build one leaf selector over original values. + */ + void build_leaf(std::size_t leaf) { + LeafSelector& selector = leaf_selectors_[leaf]; + const std::size_t begin = node_value_begin(0, leaf); + const std::size_t count = entry_count(0, leaf); + selector.build(count, [&](std::size_t left, std::size_t right) { + return compare_(values_[begin + left], values_[begin + right]); + }); + + if constexpr (kBpLeafSelector) { + const std::size_t slot = selector.arg_min(0, count, count); + selector.set_embedded_min_offset(slot); + } + } + + /** + * @brief Build one internal node selector and its cached minima. + */ + void build_internal_node(std::size_t level, std::size_t node) { + Bp512Selector& selector = mutable_selector_at(level, node); + const std::size_t count = entry_count(level, node); + const std::size_t first_child = node * fanout_at_level(level); + const bool high_level = is_high_level(level); + const std::size_t high_flat = high_level ? high_flat_index(level, node) : 0; + if (high_level) { + for (std::size_t slot = 0; slot < count; ++slot) { + const std::size_t child = first_child + slot; + HighChildMetadata& metadata = + mutable_high_child_metadata_at(high_flat, slot); + metadata.value_begin = node_value_begin(level - 1, child); + metadata.value_end = node_value_end(level - 1, child); + metadata.min_position = + static_cast(subtree_min_position(level - 1, child)); + } + build_high_sparse_min_slots(level, node, count); + } + + selector.build(count, [&](std::size_t left, std::size_t right) { + if (high_level) { + return strictly_better_high_child_slot(level, node, left, right); + } + return strictly_better_subtree_child_slot(level, node, left, right); + }); + + const std::size_t slot = selector.arg_min(0, count, count); + const std::size_t min_position = + high_level ? high_child_metadata_at(high_flat, slot).min_position + : subtree_min_position(level - 1, first_child + slot); + if (high_level) { + high_min_positions_[high_flat] = static_cast(min_position); + high_min_values_.push_back(values_[min_position]); + } else { + selector.set_embedded_min_position(min_position); + medium_min_values_.push_back(values_[min_position]); + selector.build_zero_prefix_metadata(2 * count); + } + } + + /** + * @brief Multiply two extents, saturating at `std::size_t` maximum. + */ + static std::size_t saturating_product(std::size_t left, std::size_t right) { + if (left != 0 && right > std::numeric_limits::max() / left) { + return std::numeric_limits::max(); + } + return left * right; + } + + /** + * @brief Return `ceil(value / divisor)` for positive divisors. + */ + static std::size_t ceil_div(std::size_t value, std::size_t divisor) { + return (value + divisor - 1) / divisor; + } + + /** + * @brief Return the first minimum position for a possibly partial leaf. + * + * @details Full leaves use the embedded minimum. Mask leaves answer prefix + * and suffix ranges through their record bits and fall back to scanning only + * for unsupported interior ranges. + */ + std::size_t leaf_range_min(std::size_t leaf, + std::size_t left, + std::size_t right) const { + if (left >= right) { + return npos; + } + + const std::size_t begin = node_value_begin(0, leaf); + const std::size_t end = node_value_end(0, leaf); + if (left <= begin && end <= right) { + return subtree_min_position(0, leaf); + } + const std::size_t slot_left = left - begin; + const std::size_t slot_right = right - begin; + + if constexpr (kMaskLeafSelector) { + const std::size_t slot = leaf_selectors_[leaf].arg_min( + slot_left, slot_right, entry_count(0, leaf)); + if (slot != npos) { + return begin + slot; + } + return linear_range_min(left, right); + } else { + if (right - left <= kLeafLinearScanThreshold) { + return linear_range_min(left, right); + } + + const std::size_t slot = leaf_selectors_[leaf].arg_min( + slot_left, slot_right, entry_count(0, leaf)); + if (slot == npos) { + return npos; + } + return begin + slot; + } + } + + /** + * @brief Scan original values to answer a range inside a leaf. + */ + std::size_t linear_range_min(std::size_t left, std::size_t right) const { + if (left >= right) { + return npos; + } +#ifdef PIXIE_AVX2_SUPPORT + if constexpr (std::is_same_v && + std::is_same_v>) { + if (right - left >= kLeafAvx2ScanThreshold) { + return linear_range_min_i64_avx2(left, right); + } + } +#endif + std::size_t best = left; + for (std::size_t position = left + 1; position < right; ++position) { + if (compare_(values_[position], values_[best])) { + best = position; + } + } + return best; + } + +#ifdef PIXIE_AVX2_SUPPORT + /** + * @brief AVX2 scan for signed 64-bit minimum ranges. + * + * @details Vector lanes track the best value and first position seen so far. + * Equal values keep the earlier position by only replacing lanes on strict + * value improvement. + */ + std::size_t linear_range_min_i64_avx2(std::size_t left, + std::size_t right) const { + const std::int64_t* data = values_.data(); + std::size_t position = left; + + __m256i best_values = + _mm256_loadu_si256(reinterpret_cast(data + position)); + __m256i best_positions = _mm256_set_epi64x( + static_cast(position + 3), + static_cast(position + 2), + static_cast(position + 1), static_cast(position)); + position += 4; + + for (; position + 4 <= right; position += 4) { + const __m256i values = + _mm256_loadu_si256(reinterpret_cast(data + position)); + const __m256i positions = + _mm256_set_epi64x(static_cast(position + 3), + static_cast(position + 2), + static_cast(position + 1), + static_cast(position)); + const __m256i take_new = _mm256_cmpgt_epi64(best_values, values); + best_values = _mm256_blendv_epi8(best_values, values, take_new); + best_positions = _mm256_blendv_epi8(best_positions, positions, take_new); + } + + alignas(32) std::int64_t value_lanes[4]; + alignas(32) std::uint64_t position_lanes[4]; + _mm256_store_si256(reinterpret_cast<__m256i*>(value_lanes), best_values); + _mm256_store_si256(reinterpret_cast<__m256i*>(position_lanes), + best_positions); + + std::int64_t best_value = value_lanes[0]; + std::size_t best_position = static_cast(position_lanes[0]); + for (std::size_t lane = 1; lane < 4; ++lane) { + const std::size_t lane_position = + static_cast(position_lanes[lane]); + if (value_lanes[lane] < best_value || + (value_lanes[lane] == best_value && lane_position < best_position)) { + best_value = value_lanes[lane]; + best_position = lane_position; + } + } + + for (; position < right; ++position) { + if (data[position] < best_value) { + best_value = data[position]; + best_position = position; + } + } + return best_position; + } +#endif + + /** + * @brief Find the lowest tree node that contains both endpoint leaves. + */ + std::pair covering_node( + std::size_t left_leaf, + std::size_t right_leaf) const { + std::size_t level = 0; + std::size_t left_node = left_leaf; + std::size_t right_node = right_leaf; + while (left_node != right_node) { + ++level; + const std::size_t fanout = fanout_at_level(level); + left_node /= fanout; + right_node /= fanout; + } + return {level, left_node}; + } + + /** + * @brief Return the leaf index containing an original value position. + */ + std::size_t leaf_for_value(std::size_t position) const { + return position / LeafSize; + } + + /** + * @brief Return the child index at @p child_level containing a value. + */ + std::size_t child_for_value(std::size_t child_level, + std::size_t position) const { + return position / level_value_spans_[child_level]; + } + + /** + * @brief Query a child-slot range of an internal node. + * + * @details The node selector first picks the best child in the requested slot + * range. If that child cannot be accepted directly because only a border part + * is queried, the two border children are queried recursively and the middle + * full-child range is answered by the current node selector. + */ + MinCandidate query_child_slots(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t left, + std::size_t right) const { + if (slot_left >= slot_right) { + return {}; + } + + const std::size_t count = entry_count(level, node); + const std::size_t slot = + slot_left + 1 == slot_right + ? slot_left + : selector_arg_min(level, node, slot_left, slot_right, count); + if (slot == npos) { + return {}; + } + + const std::size_t child_level = level - 1; + const std::size_t first_child = node * fanout_at_level(level); + const std::size_t child = first_child + slot; + const HighChildMetadata* high_children = + is_high_level(level) ? high_child_metadata_begin(level, node) : nullptr; + const MinCandidate child_min = + high_children != nullptr ? high_child_min_candidate(level, node, slot) + : subtree_min_candidate(child_level, child); + const std::size_t child_begin = high_children != nullptr + ? high_children[slot].value_begin + : node_value_begin(child_level, child); + const std::size_t child_end = high_children != nullptr + ? high_children[slot].value_end + : node_value_end(child_level, child); + if ((left <= child_begin && child_end <= right) || + contains_position(left, right, child_min.position)) { + return child_min; + } + + const std::size_t last_slot = slot_right - 1; + const std::size_t left_child_begin = + high_children != nullptr + ? high_children[slot_left].value_begin + : node_value_begin(child_level, first_child + slot_left); + const std::size_t left_child_end = + high_children != nullptr + ? high_children[slot_left].value_end + : node_value_end(child_level, first_child + slot_left); + MinCandidate answer = query_node(child_level, first_child + slot_left, + std::max(left, left_child_begin), + std::min(right, left_child_end)); + + if (slot_left != last_slot) { + const std::size_t right_child_begin = + high_children != nullptr + ? high_children[last_slot].value_begin + : node_value_begin(child_level, first_child + last_slot); + const std::size_t right_child_end = + high_children != nullptr + ? high_children[last_slot].value_end + : node_value_end(child_level, first_child + last_slot); + answer = better_candidate(answer, + query_node(child_level, first_child + last_slot, + std::max(left, right_child_begin), + std::min(right, right_child_end))); + } + + if (slot_left + 1 < last_slot) { + answer = better_candidate( + answer, + full_child_slot_range_min(level, node, slot_left + 1, last_slot)); + } + + return answer; + } + + /** + * @brief Return the best candidate among fully covered child slots. + */ + MinCandidate full_child_slot_range_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right) const { + if (slot_left >= slot_right) { + return {}; + } + + const std::size_t slot = + slot_left + 1 == slot_right + ? slot_left + : selector_arg_min(level, node, slot_left, slot_right, + entry_count(level, node)); + if (slot == npos) { + return {}; + } + + if (is_high_level(level)) { + return high_child_min_candidate(level, node, slot); + } + return subtree_child_min_candidate(level, node, slot); + } + + /** + * @brief Query a tree node for the minimum candidate in a value range. + */ + MinCandidate query_node(std::size_t level, + std::size_t node, + std::size_t left, + std::size_t right) const { + if (left >= right) { + return {}; + } + const std::size_t begin = node_value_begin(level, node); + const std::size_t end = node_value_end(level, node); + if (left <= begin && end <= right) { + return subtree_min_candidate(level, node); + } + if (level == 0) { + return value_candidate(leaf_range_min(node, left, right)); + } + + const std::size_t child_level = level - 1; + const std::size_t left_child = child_for_value(child_level, left); + const std::size_t right_child = child_for_value(child_level, right - 1); + const std::size_t first_child = node * fanout_at_level(level); + const std::size_t left_slot = left_child - first_child; + const std::size_t right_slot = right_child - first_child + 1; + return query_child_slots(level, node, left_slot, right_slot, left, right); + } + + /** + * @brief Return whether a position lies inside a half-open range. + */ + bool contains_position(std::size_t left, + std::size_t right, + std::size_t position) const { + return !missing_position(position) && left <= position && position < right; + } + + /** + * @brief Return the number of B-tree levels, including leaves. + */ + std::size_t level_count() const { return level_sizes_.size(); } + + /** + * @brief Return the number of entries in a leaf or child slots in a node. + */ + std::size_t entry_count(std::size_t level, std::size_t node) const { + if (level == 0) { + const std::size_t begin = node_value_begin(0, node); + return std::min(LeafSize, values_.size() - begin); + } + const std::size_t first_child = node * fanout_at_level(level); + return std::min(fanout_at_level(level), + level_sizes_[level - 1] - first_child); + } + + /** + * @brief Return the first original value position covered by a node. + */ + std::size_t node_value_begin(std::size_t level, std::size_t node) const { + return node * level_value_spans_[level]; + } + + /** + * @brief Return one past the last original value position covered by a node. + */ + std::size_t node_value_end(std::size_t level, std::size_t node) const { + return std::min(values_.size(), + node_value_begin(level, node) + level_value_spans_[level]); + } + + /** + * @brief Return a node's absolute subtree-minimum position. + */ + std::size_t subtree_min_position(std::size_t level, std::size_t node) const { + if (level == 0) { + return node_value_begin(0, node) + + leaf_selectors_[node].embedded_min_offset(); + } + if (is_high_level(level)) { + return high_min_positions_[high_flat_index(level, node)]; + } + return selector_at(level, node).embedded_min_position(); + } + + /** + * @brief Return a cached reference to a node's subtree-minimum value. + */ + const T& subtree_min_value(std::size_t level, std::size_t node) const { + if (level == 0) { + return values_[subtree_min_position(0, node)]; + } + if (is_high_level(level)) { + return high_min_values_[high_flat_index(level, node)]; + } + return medium_min_values_[medium_flat_index(level, node)]; + } + + /** + * @brief Return an immutable internal-node BP selector. + */ + const Bp512Selector& selector_at(std::size_t level, std::size_t node) const { + if (is_high_level(level)) { + return high_selectors_[high_flat_index(level, node)]; + } + return medium_selectors_[medium_flat_index(level, node)]; + } + + /** + * @brief Return a mutable internal-node BP selector while building. + */ + Bp512Selector& mutable_selector_at(std::size_t level, std::size_t node) { + if (is_high_level(level)) { + return high_selectors_[high_flat_index(level, node)]; + } + return medium_selectors_[medium_flat_index(level, node)]; + } + + /** + * @brief Run the appropriate local selector for an internal node. + */ + std::size_t selector_arg_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t count) const { + if (is_high_level(level)) { + return high_sparse_arg_min(level, node, slot_left, slot_right, count); + } + const Bp512Selector& selector = selector_at(level, node); + return selector.arg_min_with_zero_prefix(slot_left, slot_right, count); + } + + /** + * @brief Return whether a level uses the high-node layout. + */ + bool is_high_level(std::size_t level) const { + return level > 0 && level >= high_level_begin_ && level < level_count(); + } + + /** + * @brief Map a medium-level node to its flat storage index. + */ + std::size_t medium_flat_index(std::size_t level, std::size_t node) const { + return medium_level_offsets_[level] + node; + } + + /** + * @brief Return the fanout used to group children at a level. + */ + std::size_t fanout_at_level(std::size_t level) const { + return level_fanouts_[level]; + } + + /** + * @brief Map a high-level node to its flat storage index. + */ + std::size_t high_flat_index(std::size_t level, std::size_t node) const { + return high_level_offsets_[level] + node; + } + + /** + * @brief Return the first child-metadata record for a high node. + */ + const HighChildMetadata* high_child_metadata_begin(std::size_t level, + std::size_t node) const { + return high_child_metadata_.data() + high_flat_index(level, node) * Fanout; + } + + /** + * @brief Return mutable sparse-slot storage for one high node. + */ + std::uint8_t* mutable_high_sparse_min_slots_begin(std::size_t high_flat) { + return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; + } + + /** + * @brief Return sparse-slot storage for one high node. + */ + const std::uint8_t* high_sparse_min_slots_begin(std::size_t high_flat) const { + return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; + } + + /** + * @brief Return high-child metadata by flat high-node index and slot. + */ + const HighChildMetadata& high_child_metadata_at(std::size_t high_flat, + std::size_t slot) const { + return high_child_metadata_[high_flat * Fanout + slot]; + } + + /** + * @brief Return mutable high-child metadata while building. + */ + HighChildMetadata& mutable_high_child_metadata_at(std::size_t high_flat, + std::size_t slot) { + return high_child_metadata_[high_flat * Fanout + slot]; + } + + /** + * @brief Choose the better high-node child slot. + */ + std::size_t better_high_child_slot(std::size_t level, + std::size_t node, + std::size_t left_slot, + std::size_t right_slot) const { + const MinCandidate left = high_child_min_candidate(level, node, left_slot); + const MinCandidate right = + high_child_min_candidate(level, node, right_slot); + return better_candidate(left, right).position == right.position ? right_slot + : left_slot; + } + + /** + * @brief Build sparse tables over high-node child minima. + */ + void build_high_sparse_min_slots(std::size_t level, + std::size_t node, + std::size_t count) { + const std::size_t high_flat = high_flat_index(level, node); + std::uint8_t* table = mutable_high_sparse_min_slots_begin(high_flat); + for (std::size_t slot = 0; slot < count; ++slot) { + table[slot] = static_cast(slot); + } + + for (std::size_t table_level = 1; table_level < kHighSparseLevels; + ++table_level) { + const std::size_t span = std::size_t{1} << table_level; + if (span > count) { + break; + } + const std::size_t half_span = span >> 1; + const std::uint8_t* previous = table + (table_level - 1) * Fanout; + std::uint8_t* current = table + table_level * Fanout; + for (std::size_t slot = 0; slot + span <= count; ++slot) { + current[slot] = static_cast(better_high_child_slot( + level, node, previous[slot], previous[slot + half_span])); + } + } + } + + /** + * @brief Return the best high-node child slot in a slot range. + */ + std::size_t high_sparse_arg_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t count) const { + if (slot_left >= slot_right || slot_right > count) { + return npos; + } + const std::size_t length = slot_right - slot_left; + if (length == 1) { + return slot_left; + } + + const std::size_t high_flat = high_flat_index(level, node); + const std::size_t table_level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << table_level; + const std::uint8_t* table = + high_sparse_min_slots_begin(high_flat) + table_level * Fanout; + return better_high_child_slot(level, node, table[slot_left], + table[slot_right - span]); + } + + std::span values_; + Compare compare_; + std::vector leaf_selectors_; + std::vector medium_selectors_; + std::vector medium_min_values_; + std::vector high_selectors_; + std::vector high_min_positions_; + std::vector high_min_values_; + std::vector high_child_metadata_; + std::vector high_sparse_min_slots_; + std::vector medium_level_offsets_; + std::vector high_level_offsets_; + std::vector level_sizes_; + std::vector level_value_spans_; + std::vector level_fanouts_; + std::size_t high_level_begin_ = std::numeric_limits::max(); +}; + +} // namespace pixie::rmq diff --git a/src/benchmarks/bench_rmq.cpp b/src/benchmarks/bench_rmq.cpp index d17e623..68bb5de 100644 --- a/src/benchmarks/bench_rmq.cpp +++ b/src/benchmarks/bench_rmq.cpp @@ -1148,6 +1148,19 @@ void register_benchmarks() { std::int64_t, std::less, Index>>) ->Arg(static_cast(size)) ->Unit(benchmark::kMillisecond); + benchmark::RegisterBenchmark( + "rmq_build_node_euler_btree_exp_mask_leaf", + &run_value_rmq_build, Index, 248, 256, + pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); + benchmark::RegisterBenchmark( + "rmq_build_segment_btree_xl", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); benchmark::RegisterBenchmark( "rmq_build_bp_plus_minus_one", &run_bp_rmq_build>) @@ -1227,6 +1240,21 @@ void register_benchmarks() { ->Args({static_cast(size), static_cast(width)}) ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_node_euler_btree_exp_mask_leaf", + &run_queries, Index, 248, 256, + pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_segment_btree_xl", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); benchmark::RegisterBenchmark( "rmq_bp_plus_minus_one", &run_depth_queries, 128>) diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index 64834b1..bdc9813 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -152,6 +152,36 @@ struct ExperimentalNodeEulerBTreeCase { static MaxRmq make_max(std::span values) { return MaxRmq(values); } }; +struct ExperimentalNodeEulerBTreeMaskLeafCase { + using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq< + int, + std::less, + std::size_t, + 248, + 256, + pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>; + using MaxRmq = pixie::rmq::experimental::NodeEulerBTreeRmq< + int, + std::greater, + std::size_t, + 248, + 256, + pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + +struct SegmentBTreeXlCase { + using Rmq = pixie::rmq::SegmentBTreeXl; + using MaxRmq = pixie::rmq::SegmentBTreeXl>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + struct BpPlusMinusOne128Case { using Rmq = pixie::rmq::BpPlusMinusOneRmq<>; static constexpr std::size_t kBlockSize = 128; @@ -260,7 +290,9 @@ using ValueRmqCases = ::testing::Types; + ExperimentalNodeEulerBTreeCase, + ExperimentalNodeEulerBTreeMaskLeafCase, + SegmentBTreeXlCase>; TYPED_TEST_SUITE(ValueRmqContractTest, ValueRmqCases); TYPED_TEST(ValueRmqContractTest, ExhaustiveSmallArray) { @@ -695,6 +727,101 @@ TEST(RmqExperimentalNodeEulerBTree, LeafEmbeddedOffsetBoundaryRanges) { } } +TEST(RmqExperimentalNodeEulerBTree, MaskLeafBoundaryAndFallbackRanges) { + using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq< + int, std::less, std::size_t, 248, 256, + pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + static_assert(kLeaf == 248); + + std::vector values(2 * kLeaf + 13, 1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast(i % 17); + } + + values[20] = 80; + values[40] = 70; + values[80] = 60; + values[90] = 60; + values[123] = -100; + values[160] = 10; + values[200] = 5; + values[220] = 0; + values[kLeaf - 1] = 0; + + values[kLeaf + 7] = -7; + values[kLeaf + 91] = -11; + values[2 * kLeaf + 3] = -13; + + const Rmq rmq{std::span(values)}; + const std::vector> ranges = { + {0, kLeaf}, + {0, 41}, + {0, 100}, + {0, 123}, + {90, 160}, + {130, kLeaf}, + {221, kLeaf}, + {130, 210}, + {kLeaf - 5, kLeaf + 8}, + {kLeaf, 2 * kLeaf}, + {2 * kLeaf - 3, values.size()}, + {0, values.size()}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(rmq.range_min(left, right), values[expected]) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqSegmentBTreeXl, BoundaryAndFallbackRanges) { + using Rmq = pixie::rmq::SegmentBTreeXl; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + static_assert(kLeaf == 496); + + std::vector values(2 * kLeaf + 17, 2000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 7 + i / 3) % 31); + } + + values[32] = 120; + values[96] = 70; + values[160] = 55; + values[240] = -100; + values[241] = -100; + values[320] = 30; + values[440] = 5; + values[kLeaf - 1] = 5; + + values[kLeaf + 11] = -9; + values[kLeaf + 173] = -17; + values[2 * kLeaf + 5] = -21; + + const Rmq rmq{std::span(values)}; + const std::vector> ranges = { + {0, kLeaf}, {0, 97}, + {0, 240}, {96, 241}, + {250, kLeaf}, {441, kLeaf}, + {300, 450}, {kLeaf - 7, kLeaf + 12}, + {kLeaf, 2 * kLeaf}, {2 * kLeaf - 5, values.size()}, + {0, values.size()}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(rmq.range_min(left, right), values[expected]) + << "range=[" << left << "," << right << ")"; + } +} + TEST(RmqExperimentalNodeEulerBTree, DuplicateHeavyRandomDifferentialTo8193) { using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; std::mt19937_64 rng(9127); @@ -728,6 +855,74 @@ TEST(RmqExperimentalNodeEulerBTree, DuplicateHeavyRandomDifferentialTo8193) { } } +TEST(RmqExperimentalNodeEulerBTree, MaskLeafDuplicateHeavyRandomTo8193) { + using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq< + int, std::less, std::size_t, 248, 256, + pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>; + std::mt19937_64 rng(901248); + std::uniform_int_distribution value_dist(-3, 3); + const std::vector sizes = { + 1, 2, 17, 247, 248, 249, 1024, 4096, 8191, 8192, 8193, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = value_dist(rng); + if ((i % 11) < 6) { + values[i] = 0; + } + } + + const Rmq rmq{std::span(values)}; + std::uniform_int_distribution width_dist(1, size); + for (std::size_t query = 0; query < 2000; ++query) { + const std::size_t width = width_dist(rng); + std::uniform_int_distribution left_dist(0, size - width); + const std::size_t left = left_dist(rng); + const std::size_t right = left + width; + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } + } +} + +TEST(RmqSegmentBTreeXl, DuplicateHeavyRandomTo8193) { + using Rmq = pixie::rmq::SegmentBTreeXl; + std::mt19937_64 rng(901496); + std::uniform_int_distribution value_dist(-3, 3); + const std::vector sizes = { + 1, 2, 17, 495, 496, 497, 1024, 4096, 8191, 8192, 8193, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = value_dist(rng); + if ((i % 11) < 6) { + values[i] = 0; + } + } + + const Rmq rmq{std::span(values)}; + std::uniform_int_distribution width_dist(1, size); + for (std::size_t query = 0; query < 2000; ++query) { + const std::size_t width = width_dist(rng); + std::uniform_int_distribution left_dist(0, size - width); + const std::size_t left = left_dist(rng); + const std::size_t right = left + width; + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } + } +} + TEST(RmqExperimentalNodeEulerBTree, HighSparseSelectorWithMiddleLevels) { using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; constexpr std::size_t kLeaf = Rmq::kLeafSize; From f828b613fcfc667718e32044b05c9640795742e3 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Mon, 8 Jun 2026 17:14:54 +0300 Subject: [PATCH 10/27] Updated info on RMQ --- README.md | 83 ++++++++++++++++++++++++ include/pixie/rmq.h | 1 + include/pixie/rmq/segment_btree_xl.h | 94 ++++++++++++++++++++-------- src/benchmarks/bench_rmq.cpp | 4 +- src/tests/rmq_tests.cpp | 36 +++++++++++ 5 files changed, 189 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 9537bfe..3d2c94c 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,89 @@ python3 scripts/excess_benchmark_table.py excess_positions.json -o src/docs/exce Generated benchmark documentation can be written to `src/docs/benchmark_results.md`; the documentation pipeline does not run benchmarks. +### Adding an RMQ Benchmark + +Value RMQ implementations are benchmarked through the common CRTP interface in +`pixie::rmq::RmqBase`. To add a comparable backend, implement a non-owning index +that can be constructed from `std::span` and provides: + +* `size_impl()` +* `arg_min_impl(left, right)` for half-open ranges `[left, right)` +* `value_at_impl(position)` + +The public `size()`, `empty()`, `arg_min()`, and `range_min()` methods are then +provided by `RmqBase`. Ties should return the smaller original position. + +Minimal example: + +```cpp +#include + +#include +#include +#include + +namespace pixie::rmq { + +template > +class LinearRmq : public RmqBase, T> { + public: + using Self = LinearRmq; + static constexpr std::size_t npos = RmqBase::npos; + + explicit LinearRmq(std::span values, Compare compare = Compare()) + : values_(values), compare_(compare) {} + + std::size_t size_impl() const { return values_.size(); } + + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size()) { + return npos; + } + std::size_t best = left; + for (std::size_t i = left + 1; i < right; ++i) { + if (compare_(values_[i], values_[best])) { + best = i; + } + } + return best; + } + + T value_at_impl(std::size_t position) const { return values_[position]; } + + private: + std::span values_; + Compare compare_; +}; + +} // namespace pixie::rmq +``` + +Then add rows to `src/benchmarks/bench_rmq.cpp` in `register_benchmarks()`. +Use `run_value_rmq_build` for construction cost and `run_queries` for query +cost: + +```cpp +benchmark::RegisterBenchmark( + "rmq_build_linear", + &run_value_rmq_build>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); + +benchmark::RegisterBenchmark( + "rmq_linear", + &run_queries>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); +``` + +The RMQ benchmark harness rotates through several value arrays so results are +less dependent on the global-minimum position. The `width` argument is the +maximum query width, not an exact width. + ### RmM Tree ```sh diff --git a/include/pixie/rmq.h b/include/pixie/rmq.h index 2d79c70..996a3d9 100644 --- a/include/pixie/rmq.h +++ b/include/pixie/rmq.h @@ -5,5 +5,6 @@ #include #include #include +#include #include #include diff --git a/include/pixie/rmq/segment_btree_xl.h b/include/pixie/rmq/segment_btree_xl.h index 1558312..a673ca6 100644 --- a/include/pixie/rmq/segment_btree_xl.h +++ b/include/pixie/rmq/segment_btree_xl.h @@ -19,14 +19,12 @@ namespace pixie::rmq { /** - * @brief Low-level selector tag for BP-based 252-value leaves. + * @brief Low-level selector implementation used by SegmentBTreeXL leaves. */ -struct BpLeafSelectorTag {}; - -/** - * @brief Low-level selector tag for prefix/suffix mask leaves. - */ -struct PrefixSuffixMaskLeafSelectorTag {}; +enum class SegmentBTreeXLLeafSelector { + PrefixSuffix, + BP, +}; /** * @brief Segment B-tree RMQ with compact per-level selectors and XL leaves. @@ -58,6 +56,34 @@ struct PrefixSuffixMaskLeafSelectorTag {}; * tree to reduce work on wide queries while keeping the high-level metadata * small enough to stay cache-resident. * + * @code + * value array, n entries + * | + * +-- L0: leaves + * | ceil(n / 496) nodes by default + * | each leaf covers <=496 values and stores one 64-byte selector + * | + * +-- L1..Lm: middle levels, only when the frontier is > 256 * 256 nodes + * | fanout 192 + * | each node stores one 512-bit BP selector over child minima + * | + * +-- H0: high child level + * | fanout 256, <=256 nodes + * | BP selector + child ranges + child minima + sparse child table + * | + * +-- H1: high root + * fanout 256, one node + * BP selector + child ranges + child minima + sparse child table + * + * Examples with default 496-value leaves: + * + * n = 2^24: + * 33,826 leaves -> 133 high nodes -> 1 high root + * + * n = 2^26: + * 135,301 leaves -> 705 middle nodes -> 3 high nodes -> 1 high root + * @endcode + * * Querying starts at the lowest node that covers both endpoint leaves. At each * internal node, the selector is asked for the best child over the intersecting * child-slot range. If that child is fully covered, or its stored subtree @@ -73,36 +99,37 @@ struct PrefixSuffixMaskLeafSelectorTag {}; * supports 252 with BP leaves and 248 or 496 with mask leaves. * @tparam Fanout Number of children per high internal node. This backend * currently supports only 256. Middle internal nodes use 192. - * @tparam LowLevelSelector Compile-time low-level selector tag. + * @tparam LeafSelectorKind Compile-time low-level selector kind. */ template , class Index = std::size_t, std::size_t LeafSize = 496, std::size_t Fanout = 256, - class LowLevelSelector = PrefixSuffixMaskLeafSelectorTag> -class SegmentBTreeXl + SegmentBTreeXLLeafSelector LeafSelectorKind = + SegmentBTreeXLLeafSelector::PrefixSuffix> +class SegmentBTreeXL : public RmqBase< - SegmentBTreeXl, + SegmentBTreeXL, T> { public: static_assert(std::is_unsigned_v, - "SegmentBTreeXl index type must be unsigned"); + "SegmentBTreeXL index type must be unsigned"); static constexpr bool kBpLeafSelector = - std::is_same_v; + LeafSelectorKind == SegmentBTreeXLLeafSelector::BP; static constexpr bool kMaskLeafSelector = - std::is_same_v; + LeafSelectorKind == SegmentBTreeXLLeafSelector::PrefixSuffix; static_assert((kBpLeafSelector && LeafSize == 252) || (kMaskLeafSelector && (LeafSize == 248 || LeafSize == 496)), - "SegmentBTreeXl requires 252-value BP leaves " + "SegmentBTreeXL requires 252-value BP leaves " "or 248/496-value prefix/suffix mask leaves"); static_assert(Fanout == 256, - "SegmentBTreeXl currently requires 256-way internal nodes"); + "SegmentBTreeXL currently requires 256-way internal nodes"); static_assert(kBpLeafSelector || kMaskLeafSelector, - "unsupported SegmentBTreeXl low-level selector tag"); + "unsupported SegmentBTreeXL low-level selector kind"); using Self = - SegmentBTreeXl; + SegmentBTreeXL; static constexpr std::size_t npos = RmqBase::npos; static constexpr Index invalid_index = std::numeric_limits::max(); @@ -113,27 +140,27 @@ class SegmentBTreeXl /** * @brief Construct an empty RMQ index. */ - SegmentBTreeXl() = default; + SegmentBTreeXL() = default; /** * @brief Copy an RMQ index while preserving its non-owning value span. */ - SegmentBTreeXl(const SegmentBTreeXl&) = default; + SegmentBTreeXL(const SegmentBTreeXL&) = default; /** * @brief Move an RMQ index while preserving selector and cache storage. */ - SegmentBTreeXl(SegmentBTreeXl&&) noexcept = default; + SegmentBTreeXL(SegmentBTreeXL&&) noexcept = default; /** * @brief Copy-assign an RMQ index and its cached metadata. */ - SegmentBTreeXl& operator=(const SegmentBTreeXl&) = default; + SegmentBTreeXL& operator=(const SegmentBTreeXL&) = default; /** * @brief Move-assign an RMQ index and its cached metadata. */ - SegmentBTreeXl& operator=(SegmentBTreeXl&&) noexcept = default; + SegmentBTreeXL& operator=(SegmentBTreeXL&&) noexcept = default; /** * @brief Build a segment B-tree RMQ index over @p values. @@ -145,7 +172,7 @@ class SegmentBTreeXl * @param compare Ordering used to choose minima. * @throws std::length_error if @p Index cannot represent all positions. */ - explicit SegmentBTreeXl(std::span values, + explicit SegmentBTreeXL(std::span values, Compare compare = Compare()) : values_(values), compare_(compare) { build(); @@ -243,7 +270,7 @@ class SegmentBTreeXl template void build(std::size_t entry_count, EntryLess entry_less) { if (entry_count > kSelectorEntries) { - throw std::length_error("SegmentBTreeXl local selector too large"); + throw std::length_error("SegmentBTreeXL local selector too large"); } bp_bits_.fill(0); @@ -615,7 +642,7 @@ class SegmentBTreeXl void build(std::size_t entry_count, EntryLess entry_less) { if (entry_count > kMaskEntries) { throw std::length_error( - "SegmentBTreeXl prefix/suffix leaf selector too large"); + "SegmentBTreeXL prefix/suffix leaf selector too large"); } words_.fill(0); @@ -928,7 +955,7 @@ class SegmentBTreeXl return; } if (values_.size() > static_cast(invalid_index)) { - throw std::length_error("SegmentBTreeXl index type is too small"); + throw std::length_error("SegmentBTreeXL index type is too small"); } initialize_layout((values_.size() + LeafSize - 1) / LeafSize); @@ -1635,4 +1662,17 @@ class SegmentBTreeXl std::size_t high_level_begin_ = std::numeric_limits::max(); }; +/** + * @brief Compatibility alias for the historical SegmentBTreeXl spelling. + */ +template , + class Index = std::size_t, + std::size_t LeafSize = 496, + std::size_t Fanout = 256, + SegmentBTreeXLLeafSelector LeafSelectorKind = + SegmentBTreeXLLeafSelector::PrefixSuffix> +using SegmentBTreeXl = + SegmentBTreeXL; + } // namespace pixie::rmq diff --git a/src/benchmarks/bench_rmq.cpp b/src/benchmarks/bench_rmq.cpp index 68bb5de..c5b5024 100644 --- a/src/benchmarks/bench_rmq.cpp +++ b/src/benchmarks/bench_rmq.cpp @@ -1157,7 +1157,7 @@ void register_benchmarks() { ->Unit(benchmark::kMillisecond); benchmark::RegisterBenchmark( "rmq_build_segment_btree_xl", - &run_value_rmq_build, Index>>) ->Arg(static_cast(size)) ->Unit(benchmark::kMillisecond); @@ -1250,7 +1250,7 @@ void register_benchmarks() { ->Unit(benchmark::kNanosecond); benchmark::RegisterBenchmark( "rmq_segment_btree_xl", - &run_queries, Index>>) ->Args({static_cast(size), static_cast(width)}) diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index bdc9813..e4689c5 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -822,6 +822,42 @@ TEST(RmqSegmentBTreeXl, BoundaryAndFallbackRanges) { } } +TEST(RmqSegmentBTreeXl, LeafSelectorEnumVariants) { + using MaskRmq = pixie::rmq::SegmentBTreeXL< + int, std::less, std::size_t, 248, 256, + pixie::rmq::SegmentBTreeXLLeafSelector::PrefixSuffix>; + using BpRmq = + pixie::rmq::SegmentBTreeXL, std::size_t, 252, 256, + pixie::rmq::SegmentBTreeXLLeafSelector::BP>; + + std::vector values(4099, 10000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 37 + i / 7) % 257); + } + values[13] = -50; + values[251] = -70; + values[252] = -70; + values[747] = -90; + values[2048] = -120; + values[4098] = -110; + + const MaskRmq mask_rmq{std::span(values)}; + const BpRmq bp_rmq{std::span(values)}; + const std::vector> ranges = { + {0, 1}, {0, 252}, {1, 251}, {248, 253}, {251, 753}, + {700, 2100}, {2048, 2049}, {0, values.size()}, {3000, 4099}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(mask_rmq.arg_min(left, right), expected) + << "mask range=[" << left << "," << right << ")"; + EXPECT_EQ(bp_rmq.arg_min(left, right), expected) + << "bp range=[" << left << "," << right << ")"; + } +} + TEST(RmqExperimentalNodeEulerBTree, DuplicateHeavyRandomDifferentialTo8193) { using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; std::mt19937_64 rng(9127); From 4d8cc156a951282caaff2466789ecb5f5c126721 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Mon, 8 Jun 2026 17:19:00 +0300 Subject: [PATCH 11/27] Updated bench description --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 3d2c94c..48ae472 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,22 @@ The RMQ benchmark harness rotates through several value arrays so results are less dependent on the global-minimum position. The `width` argument is the maximum query width, not an exact width. +To compare the new backend with `SegmentBTreeXL`, run both benchmark families +with a Google Benchmark filter. For example, after registering the new backend +as `rmq_linear` and `rmq_build_linear`: + +```sh +./build/release/bench_rmq \ + --benchmark_filter='^(rmq_linear|rmq_segment_btree_xl)/(4194304|16777216)/(64|4096|262144|4194304|16777216)$' + +./build/release/bench_rmq \ + --benchmark_filter='^(rmq_build_linear|rmq_build_segment_btree_xl)/(262144|4194304|16777216)$' +``` + +The first command compares query time for `2^22` and `2^24` input sizes across +the common RMQ widths. The second command compares construction time for the +same implementations. + ### RmM Tree ```sh From 1e883fe303632aa26d772229d85b6bd2213eb47e Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Wed, 10 Jun 2026 01:38:00 +0300 Subject: [PATCH 12/27] SDSL rmq backend --- CMakeLists.txt | 10 ++ .../optimization-experiment/EXAMPLES.md | 11 +- include/pixie/rmq.h | 61 +++++++++++ include/pixie/rmq/cartesian_tree_rmq.h | 24 +++- include/pixie/rmq/sdsl_sct_rmq.h | 103 ++++++++++++++++++ src/benchmarks/bench_rmq.cpp | 51 +++++++++ src/tests/rmq_tests.cpp | 61 +++++++++++ 7 files changed, 314 insertions(+), 7 deletions(-) create mode 100644 include/pixie/rmq/sdsl_sct_rmq.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f1f11fc..d932a40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -201,6 +201,10 @@ if(PIXIE_TESTS) gtest gtest_main ${PIXIE_DIAGNOSTICS_LIBS}) + if(PIXIE_THIRD_PARTY_BACKENDS) + target_include_directories(rmq_tests + PRIVATE ${sdsl_lite_SOURCE_DIR}/include) + endif() endif() # --------------------------------------------------------------------------- @@ -243,6 +247,12 @@ if(PIXIE_BENCHMARKS) target_link_libraries(bench_rmq benchmark ${PIXIE_DIAGNOSTICS_LIBS}) + if(PIXIE_THIRD_PARTY_BACKENDS) + target_include_directories(bench_rmq + PRIVATE ${sdsl_lite_SOURCE_DIR}/include) + target_compile_definitions(bench_rmq + PRIVATE PIXIE_THIRD_PARTY_BENCHMARKS) + endif() if(PIXIE_THIRD_PARTY_BACKENDS) add_executable(bench_rmm_sdsl diff --git a/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md b/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md index 91aa68f..1535efc 100644 --- a/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md +++ b/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md @@ -24,9 +24,11 @@ The `excess_min_128` experiment is a good template for future hot-path work: of applied to every short range. 6. **Mark unlikely cold dispatches**: if a promoted optimization is for a narrow special case, mark the branch unlikely when that matches expected workload. -7. **Record benchmark evidence near experimental code**: add a top-of-header - benchmark note in `/** ... */` form when future agents might confuse - experimental winners with production behavior. +7. **Persist benchmark evidence in the repository**: add a top-of-file + benchmark note in `/** ... */` or `/* ... */` form to the experimental + header, production header, benchmark file, or experiment log. Include metric + units, fixed decimal precision, JSON paths when available, and visible + separators between logical result blocks. ## Benchmark Pattern @@ -69,7 +71,8 @@ For experimental algorithms, use Doxygen block comments: ``` Keep long benchmark tables as a top-level `/** ... */` note when they describe -the whole experimental header rather than one symbol. +the whole experimental header rather than one symbol. Do not leave measurements +only in chat or `/tmp`; persist a top-of-file snapshot before finishing. ## Non-Obvious Lessons From This Session diff --git a/include/pixie/rmq.h b/include/pixie/rmq.h index 996a3d9..335d38f 100644 --- a/include/pixie/rmq.h +++ b/include/pixie/rmq.h @@ -1,5 +1,66 @@ #pragma once +/* + * RMQ benchmark snapshot, 2026-06-09. + * + * Command shape: + * taskset -c 0 ./build/release/bench_rmq + * --benchmark_filter='^(rmq_sparse_table|rmq_segment_tree|rmq_segment_btree_xl|rmq_cartesian_tree|rmq_sdsl_sct)/...' + * + * SDSL SCT comparison JSON: + * /tmp/rmq_sdsl_sct_query_20260609.json + * /tmp/rmq_sdsl_sct_build_20260609.json + * That pass covered sizes through 2^24; 2^26 SDSL SCT cells are not measured. + * + * Query CPU time. "max width" is the benchmark's maximum sampled query width. + * Sparse-table rows are intentionally not registered above 2^22. + * + * | N | max width | sparse table (ns) | segment tree (ns) | segment btree xl (ns) | cartesian tree (ns) | sdsl sct (ns) | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| + * | 2^10 | 64 | 12.2 | 39.2 | 36.2 | 75.6 | 118.9 | + * | 2^10 | 2^10 | 13.4 | 62.0 | 42.0 | 90.5 | 261.0 | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| + * | 2^14 | 64 | 15.6 | 47.5 | 41.8 | 95.0 | 138.7 | + * | 2^14 | 4096 | 17.9 | 103.7 | 45.3 | 113.5 | 452.3 | + * | 2^14 | 2^14 | 18.0 | 102.1 | 34.8 | 103.1 | 539.0 | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| + * | 2^18 | 64 | 72.0 | 121.0 | 57.0 | 97.0 | 142.9 | + * | 2^18 | 4096 | 51.8 | 256.0 | 67.8 | 138.1 | 560.3 | + * | 2^18 | 2^18 | 37.3 | 303.6 | 51.7 | 147.9 | 801.4 | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| + * | 2^22 | 64 | 97.5 | 184.5 | 177.0 | 133.8 | 246.0 | + * | 2^22 | 4096 | 76.3 | 318.5 | 165.1 | 235.0 | 788.0 | + * | 2^22 | 2^18 | 65.4 | 419.7 | 77.0 | 294.7 | 1086.1 | + * | 2^22 | 2^22 | 59.1 | 577.2 | 45.9 | 309.8 | 981.6 | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| + * | 2^24 | 64 | - | 205.9 | 180.1 | 230.2 | 447.1 | + * | 2^24 | 4096 | - | 380.6 | 302.9 | 389.0 | 1250.7 | + * | 2^24 | 2^18 | - | 634.2 | 121.6 | 472.9 | 1424.3 | + * | 2^24 | 2^22 | - | 565.7 | 53.2 | 426.8 | 1750.7 | + * | 2^24 | 2^24 | - | 686.9 | 39.2 | 389.0 | 1800.5 | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| + * | 2^26 | 64 | - | 314.0 | 248.0 | 378.0 | - | + * | 2^26 | 4096 | - | 636.0 | 315.0 | 549.0 | - | + * | 2^26 | 2^18 | - | 770.0 | 150.0 | 563.0 | - | + * | 2^26 | 2^22 | - | 880.0 | 71.3 | 676.0 | - | + * | 2^26 | 2^26 | - | 1051.0 | 56.7 | 505.0 | - | + * + * Build CPU time. Sparse-table build rows are intentionally not registered + * above 2^22. + * + * | N | sparse table (ms) | segment tree (ms) | segment btree xl (ms) | cartesian tree (ms) | sdsl sct (ms) | + * |-----:|------------------:|------------------:|----------------------:|--------------------:|--------------:| + * | 2^10 | 0.005 | 0.001 | 0.001 | 0.008 | 0.007 | + * |-----:|------------------:|------------------:|----------------------:|--------------------:|--------------:| + * | 2^14 | 1.091 | 0.015 | 0.017 | 0.043 | 0.239 | + * |-----:|------------------:|------------------:|----------------------:|--------------------:|--------------:| + * | 2^18 | 27.422 | 0.289 | 0.276 | 1.668 | 2.513 | + * |-----:|------------------:|------------------:|----------------------:|--------------------:|--------------:| + * | 2^22 | 855.613 | 57.580 | 5.675 | 29.491 | 40.839 | + * |-----:|------------------:|------------------:|----------------------:|--------------------:|--------------:| + * | 2^24 | - | 289.428 | 21.604 | 173.158 | 159.807 | + */ + #include #include #include diff --git a/include/pixie/rmq/cartesian_tree_rmq.h b/include/pixie/rmq/cartesian_tree_rmq.h index 6961ddf..d63778d 100644 --- a/include/pixie/rmq/cartesian_tree_rmq.h +++ b/include/pixie/rmq/cartesian_tree_rmq.h @@ -22,9 +22,27 @@ namespace pixie::rmq { * @brief General RMQ via the Ferrada-Navarro BP Cartesian-tree encoding. * * @details Builds the balanced-parentheses representation of the Cartesian-tree - * RMQ information directly from the indexed values. Queries use select/rank on - * BP positions plus a ±1 RMQ over BP prefix excess. The indexed values are not - * owned and must outlive this object. + * RMQ information directly from the indexed values. The construction scans the + * input right-to-left with a monotone stack and emits the BP sequence used by + * the Ferrada-Navarro formulation. Queries map the half-open value interval to + * positions in that BP sequence with `select0`, run a ±1 RMQ over BP prefix + * excess, and map the winning BP position back to an array index with `rank0`. + * + * The indexed values are not owned and must outlive this object. `arg_min()` + * answers from the BP indexes alone; `range_min()` still reads the external + * value span after the position is known. Equal values are handled stably: the + * smaller original position remains the first minimum. + * + * This implementation is a practical BP Cartesian-tree backend, not a fully + * compressed 2n + o(n)-bit object: it stores the 2n BP bits plus the supporting + * `BitVector` rank/select index and a `BpPlusMinusOneRmq` over BP excess. + * + * References: + * - Johannes Fischer, "Optimal Succinctness for Range Minimum Queries", + * LATIN 2010; arXiv:0812.2775. + * - Hector Ferrada and Gonzalo Navarro, "Improved Range Minimum Queries", + * Data Compression Conference 2016; Journal of Discrete Algorithms 43 + * (2017), doi:10.1016/j.jda.2016.09.002. * * @tparam T Value type in the indexed array. * @tparam Compare Strict weak ordering used to choose minima. diff --git a/include/pixie/rmq/sdsl_sct_rmq.h b/include/pixie/rmq/sdsl_sct_rmq.h new file mode 100644 index 0000000..af9521e --- /dev/null +++ b/include/pixie/rmq/sdsl_sct_rmq.h @@ -0,0 +1,103 @@ +#pragma once + +#ifndef SDSL_SUPPORT +#error "pixie/rmq/sdsl_sct_rmq.h requires SDSL_SUPPORT" +#endif + +#include + +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief Optional value-RMQ adapter over `sdsl::rmq_succinct_sct`. + * + * @details SDSL's SCT implementation builds a global Super-Cartesian-tree + * balanced-parentheses sequence (BPS-SCT) over the indexed values. It then + * answers inclusive range queries with balanced-parentheses navigation + * operations such as `select`, `find_close`, `rr_enclose`, and `rank`. + * + * SDSL's helper comments attribute the Super-Cartesian-tree BP construction to + * Ohlebusch and Gog in a compressed enhanced suffix-array context. That is not + * the canonical RMQ citation. For the succinct RMQ result behind this style of + * representation, see: + * + * Johannes Fischer, "Optimal Succinctness for Range Minimum Queries", + * LATIN 2010; arXiv:0812.2775. + * + * This adapter exposes the Pixie value-RMQ contract: half-open ranges + * `[left, right)`, invalid ranges returning `npos`, and first-minimum tie + * semantics. + * + * The SDSL structure does not retain values after construction, so this adapter + * keeps a non-owning span for `range_min()`. The indexed values must outlive + * the adapter. SDSL SCT supports min/max through a boolean template parameter, + * not arbitrary comparators; this wrapper currently exposes only + * `std::less`. + * + * @tparam T Value type in the indexed array. + * @tparam Compare Must be `std::less`. + * @tparam Index Interface compatibility parameter for benchmark templates. + */ +template , class Index = std::size_t> +class SdslSctRmq : public RmqBase, T> { + public: + static_assert(std::is_same_v>, + "SDSL SCT RMQ wrapper supports only std::less"); + + static constexpr std::size_t npos = + RmqBase, T>::npos; + + /** + * @brief Construct an empty SDSL SCT RMQ adapter. + */ + SdslSctRmq() = default; + + /** + * @brief Build the SDSL SCT index over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values keep the smaller index as the RMQ answer. + * + * @param values Values to index. + * @param compare Must be the default `std::less` comparator. + */ + explicit SdslSctRmq(std::span values, Compare compare = Compare()) + : values_(values), rmq_(&values_) { + (void)compare; + } + + /** + * @brief Return the number of indexed values. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + * + * @details SDSL's query is inclusive, so the adapter translates the Pixie + * half-open range to `[left, right - 1]`. + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size()) { + return npos; + } + return static_cast(rmq_(left, right - 1)); + } + + private: + std::span values_; + sdsl::rmq_succinct_sct rmq_; +}; + +} // namespace pixie::rmq diff --git a/src/benchmarks/bench_rmq.cpp b/src/benchmarks/bench_rmq.cpp index c5b5024..3c346d2 100644 --- a/src/benchmarks/bench_rmq.cpp +++ b/src/benchmarks/bench_rmq.cpp @@ -1,9 +1,14 @@ #include #include #include +#include #include #include +#ifdef PIXIE_THIRD_PARTY_BENCHMARKS +#include +#endif + #ifdef __linux__ #include #include @@ -16,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -1136,6 +1142,20 @@ void register_benchmarks() { std::int64_t, std::less, Index>>) ->Arg(static_cast(size)) ->Unit(benchmark::kMillisecond); + benchmark::RegisterBenchmark( + "rmq_build_cartesian_tree_rmm_btree", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); +#ifdef PIXIE_THIRD_PARTY_BENCHMARKS + benchmark::RegisterBenchmark( + "rmq_build_sdsl_sct", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); +#endif benchmark::RegisterBenchmark( "rmq_build_node_euler_btree", &run_value_rmq_build>) ->Arg(static_cast(size)) ->Unit(benchmark::kMillisecond); + benchmark::RegisterBenchmark( + "rmq_build_bp_rmm_btree", + &run_bp_rmq_build< + pixie::rmq::experimental::RmMBTreePlusMinusOneRmq>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); benchmark::RegisterBenchmark( "rmq_build_bp_one_interval_btree", &run_bp_rmq_build>) @@ -1226,6 +1252,22 @@ void register_benchmarks() { ->Args({static_cast(size), static_cast(width)}) ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_cartesian_tree_rmm_btree", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); +#ifdef PIXIE_THIRD_PARTY_BENCHMARKS + benchmark::RegisterBenchmark( + "rmq_sdsl_sct", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); +#endif benchmark::RegisterBenchmark( "rmq_node_euler_btree", &run_queriesArgs({static_cast(size), static_cast(width)}) ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_bp_rmm_btree", + &run_depth_queries< + pixie::rmq::experimental::RmMBTreePlusMinusOneRmq, + pixie::rmq::experimental::RmMBTreePlusMinusOneRmq< + Index>::kBlockSize>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); benchmark::RegisterBenchmark( "rmq_bp_one_interval_btree", &run_depth_queries< diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index e4689c5..acf6366 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -1,8 +1,13 @@ #include #include +#include #include #include +#ifdef SDSL_SUPPORT +#include +#endif + #include #include #include @@ -133,6 +138,17 @@ struct CartesianTreeCase { static MaxRmq make_max(std::span values) { return MaxRmq(values); } }; +struct ExperimentalCartesianTreeRmMBTreeCase { + using Rmq = pixie::rmq::experimental::CartesianTreeRmMBTreeRmq; + using MaxRmq = + pixie::rmq::experimental::CartesianTreeRmMBTreeRmq>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + struct NodeEulerBTreeCase { using Rmq = pixie::rmq::NodeEulerBTreeRmq; using MaxRmq = pixie::rmq::NodeEulerBTreeRmq>; @@ -202,6 +218,16 @@ struct BpPlusMinusOne64Case { } }; +struct ExperimentalRmMBTreePlusMinusOneCase { + using Rmq = pixie::rmq::experimental::RmMBTreePlusMinusOneRmq<>; + static constexpr std::size_t kBlockSize = Rmq::kBlockSize; + + static Rmq make(std::span bits, + std::size_t depth_count) { + return Rmq(bits, depth_count); + } +}; + struct OneIntervalBTreeCase { using Rmq = pixie::rmq::OneIntervalBTreeRmq<>; static constexpr std::size_t kBlockSize = Rmq::kBlockSize; @@ -289,6 +315,7 @@ class ValueRmqContractTest : public ::testing::Test {}; using ValueRmqCases = ::testing::Types values = {4, -3, 7, -3, 0, -8, -8, 2, 2}; + const pixie::rmq::SdslSctRmq rmq{std::span(values)}; + + check_all_ranges(rmq, std::span(values), std::less()); + EXPECT_EQ(rmq.arg_min(0, values.size()), 5u); + EXPECT_EQ(rmq.arg_min(5, 7), 5u); + EXPECT_EQ(rmq.arg_min(6, 7), 6u); + EXPECT_EQ(rmq.arg_min(3, 3), pixie::rmq::SdslSctRmq::npos); + EXPECT_EQ(rmq.arg_min(0, values.size() + 1), + pixie::rmq::SdslSctRmq::npos); +} + +TEST(RmqSdslSct, DifferentialRandom) { + std::mt19937_64 rng(123); + std::uniform_int_distribution value_dist(-20, 20); + for (std::size_t size = 1; size <= 129; size += 16) { + std::vector values(size); + std::generate(values.begin(), values.end(), + [&] { return value_dist(rng); }); + + const pixie::rmq::SdslSctRmq rmq{std::span(values)}; + check_all_ranges(rmq, std::span(values), std::less()); + } + + const std::vector empty_values; + const pixie::rmq::SdslSctRmq empty{std::span(empty_values)}; + EXPECT_TRUE(empty.empty()); + EXPECT_EQ(empty.arg_min(0, 0), pixie::rmq::SdslSctRmq::npos); +} +#endif + TEST(RmqNodeEulerBTree, BoundarySizesAroundLeavesAndInternalNodes) { using Rmq = pixie::rmq::NodeEulerBTreeRmq; constexpr std::size_t kLeaf = Rmq::kLeafSize; @@ -1041,6 +1101,7 @@ class DepthRmqContractTest : public ::testing::Test {}; using DepthRmqCases = ::testing::Types; TYPED_TEST_SUITE(DepthRmqContractTest, DepthRmqCases); From 8d6313ce35d6269a9f2c7427c684a4caf76f7ee5 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Wed, 10 Jun 2026 20:47:23 +0300 Subject: [PATCH 13/27] Implementation of two rmq variants without access to original array --- include/pixie/rmq.h | 95 +- .../cartesian_tree_rmm_btree_rmq.h | 373 ++++ .../cartesian_tree_segment_btree_xl_rmq.h | 1821 +++++++++++++++++ src/benchmarks/bench_rmq.cpp | 15 + src/tests/rmq_tests.cpp | 355 +++- 5 files changed, 2599 insertions(+), 60 deletions(-) create mode 100644 include/pixie/rmq/experimental/cartesian_tree_rmm_btree_rmq.h create mode 100644 include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h diff --git a/include/pixie/rmq.h b/include/pixie/rmq.h index 335d38f..ade35ba 100644 --- a/include/pixie/rmq.h +++ b/include/pixie/rmq.h @@ -1,64 +1,65 @@ #pragma once /* - * RMQ benchmark snapshot, 2026-06-09. + * RMQ benchmark snapshot, 2026-06-10. * * Command shape: * taskset -c 0 ./build/release/bench_rmq - * --benchmark_filter='^(rmq_sparse_table|rmq_segment_tree|rmq_segment_btree_xl|rmq_cartesian_tree|rmq_sdsl_sct)/...' + * --benchmark_filter='^(rmq_sparse_table|rmq_segment_tree|rmq_segment_btree_xl|rmq_cartesian_tree|rmq_cartesian_tree_segment_btree_xl|rmq_sdsl_sct)/' + * --benchmark_min_time=0.25s * - * SDSL SCT comparison JSON: - * /tmp/rmq_sdsl_sct_query_20260609.json - * /tmp/rmq_sdsl_sct_build_20260609.json - * That pass covered sizes through 2^24; 2^26 SDSL SCT cells are not measured. + * JSON: + * /tmp/rmq_value_query_table_20260610.json + * /tmp/rmq_value_build_table_20260610.json * * Query CPU time. "max width" is the benchmark's maximum sampled query width. - * Sparse-table rows are intentionally not registered above 2^22. + * Sparse-table rows are intentionally not registered above 2^22. "cartesian xl" + * is the experimental `rmq_cartesian_tree_segment_btree_xl` row. * - * | N | max width | sparse table (ns) | segment tree (ns) | segment btree xl (ns) | cartesian tree (ns) | sdsl sct (ns) | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| - * | 2^10 | 64 | 12.2 | 39.2 | 36.2 | 75.6 | 118.9 | - * | 2^10 | 2^10 | 13.4 | 62.0 | 42.0 | 90.5 | 261.0 | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| - * | 2^14 | 64 | 15.6 | 47.5 | 41.8 | 95.0 | 138.7 | - * | 2^14 | 4096 | 17.9 | 103.7 | 45.3 | 113.5 | 452.3 | - * | 2^14 | 2^14 | 18.0 | 102.1 | 34.8 | 103.1 | 539.0 | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| - * | 2^18 | 64 | 72.0 | 121.0 | 57.0 | 97.0 | 142.9 | - * | 2^18 | 4096 | 51.8 | 256.0 | 67.8 | 138.1 | 560.3 | - * | 2^18 | 2^18 | 37.3 | 303.6 | 51.7 | 147.9 | 801.4 | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| - * | 2^22 | 64 | 97.5 | 184.5 | 177.0 | 133.8 | 246.0 | - * | 2^22 | 4096 | 76.3 | 318.5 | 165.1 | 235.0 | 788.0 | - * | 2^22 | 2^18 | 65.4 | 419.7 | 77.0 | 294.7 | 1086.1 | - * | 2^22 | 2^22 | 59.1 | 577.2 | 45.9 | 309.8 | 981.6 | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| - * | 2^24 | 64 | - | 205.9 | 180.1 | 230.2 | 447.1 | - * | 2^24 | 4096 | - | 380.6 | 302.9 | 389.0 | 1250.7 | - * | 2^24 | 2^18 | - | 634.2 | 121.6 | 472.9 | 1424.3 | - * | 2^24 | 2^22 | - | 565.7 | 53.2 | 426.8 | 1750.7 | - * | 2^24 | 2^24 | - | 686.9 | 39.2 | 389.0 | 1800.5 | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|--------------:| - * | 2^26 | 64 | - | 314.0 | 248.0 | 378.0 | - | - * | 2^26 | 4096 | - | 636.0 | 315.0 | 549.0 | - | - * | 2^26 | 2^18 | - | 770.0 | 150.0 | 563.0 | - | - * | 2^26 | 2^22 | - | 880.0 | 71.3 | 676.0 | - | - * | 2^26 | 2^26 | - | 1051.0 | 56.7 | 505.0 | - | + * | N | max width | sparse table (ns) | segment tree (ns) | segment btree xl (ns) | cartesian tree (ns) | cartesian xl (ns) | sdsl sct (ns) | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^10 | 64 | 10.1 | 35.9 | 34.6 | 72.7 | 66.4 | 111.2 | + * | 2^10 | 2^10 | 13.1 | 58.8 | 41.0 | 90.0 | 118.6 | 231.3 | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^14 | 64 | 14.7 | 43.6 | 33.5 | 86.3 | 67.3 | 125.4 | + * | 2^14 | 4096 | 15.9 | 76.7 | 38.1 | 108.0 | 66.3 | 430.8 | + * | 2^14 | 2^14 | 15.8 | 93.5 | 33.7 | 103.6 | 49.2 | 532.1 | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^18 | 64 | 63.0 | 103.7 | 50.9 | 87.0 | 74.7 | 141.3 | + * | 2^18 | 4096 | 52.2 | 183.0 | 52.5 | 114.2 | 72.6 | 586.7 | + * | 2^18 | 2^18 | 40.9 | 299.1 | 46.0 | 148.5 | 27.6 | 791.0 | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^22 | 64 | 97.2 | 169.1 | 137.4 | 164.7 | 256.9 | 263.2 | + * | 2^22 | 4096 | 93.4 | 298.7 | 101.9 | 236.1 | 251.9 | 726.7 | + * | 2^22 | 2^18 | 68.0 | 437.3 | 82.2 | 344.1 | 86.4 | 1053.0 | + * | 2^22 | 2^22 | 60.7 | 485.3 | 45.5 | 225.6 | 22.3 | 1129.7 | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^24 | 64 | - | 208.0 | 166.6 | 326.0 | 480.4 | 549.6 | + * | 2^24 | 4096 | - | 357.2 | 222.2 | 755.7 | 535.4 | 864.8 | + * | 2^24 | 2^18 | - | 472.3 | 143.0 | 734.9 | 83.5 | 2824.2 | + * | 2^24 | 2^22 | - | 510.7 | 47.1 | 403.5 | 25.3 | 1169.8 | + * | 2^24 | 2^24 | - | 546.5 | 43.4 | 479.6 | 20.1 | 1239.6 | + * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^26 | 64 | - | 273.5 | 342.3 | 539.2 | 567.5 | 710.4 | + * | 2^26 | 4096 | - | 938.4 | 308.9 | 566.5 | 688.5 | 1701.6 | + * | 2^26 | 2^18 | - | 802.5 | 175.6 | 664.8 | 195.1 | 2609.4 | + * | 2^26 | 2^22 | - | 1690.7 | 64.0 | 625.2 | 45.2 | 2387.0 | + * | 2^26 | 2^26 | - | 1003.7 | 51.0 | 588.9 | 24.1 | 2152.0 | * * Build CPU time. Sparse-table build rows are intentionally not registered * above 2^22. * - * | N | sparse table (ms) | segment tree (ms) | segment btree xl (ms) | cartesian tree (ms) | sdsl sct (ms) | - * |-----:|------------------:|------------------:|----------------------:|--------------------:|--------------:| - * | 2^10 | 0.005 | 0.001 | 0.001 | 0.008 | 0.007 | - * |-----:|------------------:|------------------:|----------------------:|--------------------:|--------------:| - * | 2^14 | 1.091 | 0.015 | 0.017 | 0.043 | 0.239 | - * |-----:|------------------:|------------------:|----------------------:|--------------------:|--------------:| - * | 2^18 | 27.422 | 0.289 | 0.276 | 1.668 | 2.513 | - * |-----:|------------------:|------------------:|----------------------:|--------------------:|--------------:| - * | 2^22 | 855.613 | 57.580 | 5.675 | 29.491 | 40.839 | - * |-----:|------------------:|------------------:|----------------------:|--------------------:|--------------:| - * | 2^24 | - | 289.428 | 21.604 | 173.158 | 159.807 | + * | N | sparse table (ms) | segment tree (ms) | segment btree xl (ms) | cartesian tree (ms) | cartesian xl (ms) | sdsl sct (ms) | + * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^10 | 0.006 | 0.001 | 0.001 | 0.008 | 0.010 | 0.007 | + * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^14 | 1.081 | 0.015 | 0.017 | 0.048 | 0.063 | 0.230 | + * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^18 | 32.766 | 0.369 | 0.288 | 1.741 | 2.108 | 2.596 | + * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^22 | 903.196 | 78.183 | 5.401 | 29.116 | 33.755 | 41.024 | + * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^24 | - | 267.256 | 21.746 | 172.782 | 136.031 | 165.874 | */ #include diff --git a/include/pixie/rmq/experimental/cartesian_tree_rmm_btree_rmq.h b/include/pixie/rmq/experimental/cartesian_tree_rmm_btree_rmq.h new file mode 100644 index 0000000..05f4771 --- /dev/null +++ b/include/pixie/rmq/experimental/cartesian_tree_rmm_btree_rmq.h @@ -0,0 +1,373 @@ +#pragma once + +/** + * Benchmark snapshot: experimental rmM-backed Cartesian/BP RMQ, 2026-06-10. + * + * Command shape: + * taskset -c 0 ./build/release/bench_rmq + * --benchmark_filter='^(rmq_cartesian_tree|rmq_cartesian_tree_rmm_btree| + * rmq_bp_plus_minus_one|rmq_bp_rmm_btree)/...|^(rmq_build_...)/...' + * --benchmark_min_time=0.30s + * --benchmark_out=/tmp/rmq_rmm_btree_20260610.json + * + * Query CPU time, ns: + * | N | width | cartesian | cartesian+rmm | bp +/-1 | bp+rmm | + * |----------|----------|-----------|---------------|---------|---------| + * | 2^10 | 64 | 80.5 | 155.5 | 42.8 | 107.1 | + * |----------|----------|-----------|---------------|---------|---------| + * | 2^14 | 64 | 98.8 | 177.4 | 42.7 | 117.9 | + * | 2^14 | 4096 | 123.9 | 722.2 | 69.0 | 634.9 | + * |----------|----------|-----------|---------------|---------|---------| + * | 2^18 | 64 | 110.8 | 176.7 | 42.5 | 118.1 | + * | 2^18 | 4096 | 156.0 | 744.6 | 66.6 | 735.1 | + * | 2^18 | 2^18 | 165.8 | 1222.1 | 67.6 | 1204.5 | + * |----------|----------|-----------|---------------|---------|---------| + * | 2^22 | 64 | 199.1 | 328.6 | 46.5 | 131.7 | + * | 2^22 | 4096 | 327.0 | 1271.9 | 95.8 | 772.3 | + * | 2^22 | 2^22 | 312.8 | 1601.3 | 88.6 | 1498.6 | + * |----------|----------|-----------|---------------|---------|---------| + * | 2^24 | 64 | 272.5 | 396.2 | 77.0 | 202.9 | + * | 2^24 | 4096 | 500.1 | 1248.3 | 189.6 | 1038.2 | + * | 2^24 | 2^24 | 483.5 | 1832.2 | 126.7 | 1687.9 | + * + * Build CPU time, ms: + * | N | cartesian | cartesian+rmm | bp +/-1 | bp+rmm | + * |----------|-----------|---------------|---------|--------| + * | 2^10 | 0.006 | 0.008 | 0.000 | 0.005 | + * | 2^14 | 0.053 | 0.074 | 0.002 | 0.026 | + * | 2^18 | 1.770 | 2.243 | 0.047 | 0.377 | + * | 2^22 | 43.794 | 35.318 | 1.230 | 6.164 | + * | 2^24 | 192.036 | 155.852 | 20.232 | 23.921 | + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq::experimental { + +/** + * @brief Depth RMQ adapter over the experimental rmM btree. + * + * @details The input words encode adjacent depth deltas: bit 1 means +1 and + * bit 0 means -1. The indexed depth sequence has `depth_count` prefix + * positions, so the bit sequence has `depth_count - 1` bits. Queries return the + * first position of the minimum depth in a half-open depth-position range. + * + * `pixie::experimental::RmMBTree::range_min_query_pos()` works on inclusive + * bit ranges and reports extrema after consuming each bit. This adapter adds + * the missing left prefix boundary comparison, preserving first-minimum ties. + */ +template +class RmMBTreePlusMinusOneRmq { + public: + static_assert(std::is_unsigned_v, + "RmMBTreePlusMinusOneRmq index type must be unsigned"); + + static constexpr std::size_t npos = std::numeric_limits::max(); + static constexpr Index invalid_index = std::numeric_limits::max(); + static constexpr std::size_t kBlockSize = + pixie::experimental::RmMBTree::kBlockBits; + + RmMBTreePlusMinusOneRmq() = default; + + /** + * @brief Build an rmM-backed ±1 RMQ over external packed delta bits. + * + * @param bits Little-endian packed delta bits. + * @param depth_count Number of indexed depth positions. + * @throws std::length_error if @p Index cannot represent all positions. + * @throws std::invalid_argument if @p bits is shorter than required. + */ + RmMBTreePlusMinusOneRmq(std::span bits, + std::size_t depth_count) { + build(bits, depth_count); + } + + /** + * @brief Rebuild this adapter over external packed delta bits. + */ + void build(std::span bits, std::size_t depth_count) { + words_ = bits; + depth_count_ = depth_count; + rmm_ = RmM(); + + if (depth_count_ == 0) { + return; + } + if (depth_count_ > static_cast(invalid_index)) { + throw std::length_error("RMQ rmM btree index type is too small"); + } + rmm_ = RmM(words_, depth_count_ - 1); + } + + /** + * @brief Return the number of indexed depth positions. + */ + std::size_t size() const { return depth_count_; } + + /** + * @brief Whether the indexed depth sequence is empty. + */ + bool empty() const { return depth_count_ == 0; } + + /** + * @brief Return the first minimum depth position in [@p left, @p right). + * + * @details Empty or invalid ranges return `npos`. Equal minima return the + * smaller depth position. + */ + std::size_t arg_min(std::size_t left, std::size_t right) const { + if (left >= right || right > depth_count_) { + return npos; + } + if (right == left + 1) { + return left; + } + + const std::size_t bit_left = left; + const std::size_t bit_right = right - 2; + const int minimum_after_left = + rmm_.range_min_query_val(bit_left, bit_right); + if (minimum_after_left >= 0) { + return left; + } + + const std::size_t bit_position = + rmm_.range_min_query_pos(bit_left, bit_right); + return bit_position == RmM::npos ? npos : bit_position + 1; + } + + /** + * @brief Return the one-based rank-th closing parenthesis position. + */ + std::size_t select0(std::size_t rank) const { return rmm_.select0(rank); } + + /** + * @brief Count closing parentheses in the prefix [0, @p end_position). + */ + std::size_t rank0(std::size_t end_position) const { + return rmm_.rank0(end_position); + } + + private: + using RmM = pixie::experimental::RmMBTree; + + std::span words_; + std::size_t depth_count_ = 0; + RmM rmm_; +}; + +/** + * @brief Experimental Ferrada-Navarro Cartesian-tree RMQ using rmM support. + * + * @details This class keeps the same public value-RMQ contract as + * `CartesianTreeRmq`, but replaces the `BitVector` + `BpPlusMinusOneRmq` + * support layer with `RmMBTreePlusMinusOneRmq`, which in turn uses the + * experimental range min-max btree over the BP excess sequence. + * + * This is intentionally not included from `pixie/rmq.h`; it exists to benchmark + * the Ferrada-Navarro BP formula with an rmM-backed `rmq_D` primitive before + * deciding whether to promote or optimize it. + */ +template , + class Index = std::size_t, + std::size_t HighCacheLines = 4, + std::size_t LowFanout = 32> +class CartesianTreeRmMBTreeRmq + : public RmqBase, + T> { + public: + static_assert(std::is_unsigned_v, + "CartesianTreeRmMBTreeRmq index type must be unsigned"); + + static constexpr std::size_t npos = RmqBase< + CartesianTreeRmMBTreeRmq, + T>::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + + CartesianTreeRmMBTreeRmq() = default; + + /** + * @brief Build an experimental Cartesian-tree RMQ index over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values stay stable: the smaller index remains the first minimum. + */ + explicit CartesianTreeRmMBTreeRmq(std::span values, + Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + CartesianTreeRmMBTreeRmq(const CartesianTreeRmMBTreeRmq& other) + : values_(other.values_), + compare_(other.compare_), + bp_bits_(other.bp_bits_), + bp_bit_count_(other.bp_bit_count_) { + reset_bp_index(); + } + + CartesianTreeRmMBTreeRmq& operator=(const CartesianTreeRmMBTreeRmq& other) { + if (this == &other) { + return *this; + } + values_ = other.values_; + compare_ = other.compare_; + bp_bits_ = other.bp_bits_; + bp_bit_count_ = other.bp_bit_count_; + reset_bp_index(); + return *this; + } + + CartesianTreeRmMBTreeRmq(CartesianTreeRmMBTreeRmq&& other) noexcept + : values_(other.values_), + compare_(std::move(other.compare_)), + bp_bits_(std::move(other.bp_bits_)), + bp_bit_count_(other.bp_bit_count_) { + other.values_ = std::span(); + other.bp_bit_count_ = 0; + reset_bp_index(); + } + + CartesianTreeRmMBTreeRmq& operator=( + CartesianTreeRmMBTreeRmq&& other) noexcept { + if (this == &other) { + return *this; + } + values_ = other.values_; + compare_ = std::move(other.compare_); + bp_bits_ = std::move(other.bp_bits_); + bp_bit_count_ = other.bp_bit_count_; + other.values_ = std::span(); + other.bp_bit_count_ = 0; + reset_bp_index(); + return *this; + } + + /** + * @brief Return the number of indexed values. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size()) { + return npos; + } + const std::size_t first_close = bp_support_.select0(left + 1); + const std::size_t last_close = bp_support_.select0(right); + if (first_close == BpSupport::npos || last_close == BpSupport::npos || + first_close > last_close) { + return npos; + } + + const std::size_t shifted_min = + bp_support_.arg_min(first_close + 1, last_close + 2); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + const std::size_t answer = bp_support_.rank0(shifted_min) - 1; + return answer < values_.size() ? answer : npos; + } + + /** + * @brief Return the number of BP bits in the Cartesian-tree RMQ encoding. + */ + std::size_t bp_bit_count() const { return bp_bit_count_; } + + /** + * @brief Return the packed BP words used by the RMQ encoding. + */ + std::span bp_words() const { return bp_bits_; } + + private: + using BpSupport = RmMBTreePlusMinusOneRmq; + + void build() { + bp_bits_.clear(); + bp_bit_count_ = 0; + bp_support_ = BpSupport(); + + if (values_.empty()) { + return; + } + if (values_.size() > (static_cast(invalid_index) - 1) / 2) { + throw std::length_error("Cartesian rmM RMQ index type is too small"); + } + + bp_bit_count_ = 2 * values_.size(); + bp_bits_.assign((bp_bit_count_ + 63) / 64, 0); + build_bp_bits(); + reset_bp_index(); + } + + void build_bp_bits() { + const auto stack = std::make_unique_for_overwrite(values_.size()); + std::size_t stack_size = 0; + std::size_t write_position = bp_bit_count_; + + for (std::size_t i = values_.size(); i-- > 0;) { + while (stack_size != 0 && + !compare_(values_[stack[stack_size - 1]], values_[i])) { + --stack_size; + prepend_bp_bit(write_position, true); + } + stack[stack_size++] = static_cast(i); + prepend_bp_bit(write_position, false); + } + + while (write_position != 0) { + prepend_bp_bit(write_position, true); + } + } + + void prepend_bp_bit(std::size_t& write_position, bool bit) { + --write_position; + if (bit) { + bp_bits_[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); + } + } + + void reset_bp_index() { + bp_support_ = BpSupport(); + if (bp_bit_count_ == 0) { + return; + } + bp_support_ = + BpSupport(std::span(bp_bits_), bp_bit_count_ + 1); + } + + std::span values_; + Compare compare_; + std::vector bp_bits_; + std::size_t bp_bit_count_ = 0; + BpSupport bp_support_; +}; + +} // namespace pixie::rmq::experimental diff --git a/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h b/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h new file mode 100644 index 0000000..09d8917 --- /dev/null +++ b/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h @@ -0,0 +1,1821 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq::experimental { + +/** + * @brief SegmentBTreeXL-style RMQ backend for ±1 depth sequences. + * + * @details The input words encode adjacent depth deltas: bit 1 means +1 and + * bit 0 means -1. The indexed sequence has `depth_count` prefix positions, so + * the bit sequence has `depth_count - 1` deltas. Public ranges are half-open + * over depth positions, and ties return the smaller depth position. + * + * This backend mirrors the high-level query shape of `SegmentBTreeXL`: depth + * positions are split into leaves, leaves are grouped into a B-tree, and every + * internal node stores a local Cartesian/BP selector over the minima of its + * immediate children. The top one or two levels additionally keep sparse + * tables over child-minimum slots. Unlike value `SegmentBTreeXL`, leaves do not + * store another local Cartesian selector. A leaf query scans the original BP + * delta bits directly with the 128-bit excess primitives from `bits.h`. + * + * @tparam Index Unsigned integer type used for stored positions. + * @tparam LeafSize Number of depth positions per low-level leaf. + * @tparam Fanout Fanout used by the top high levels. + * @tparam MiddleFanout Fanout used below the high levels. + */ +template +class SegmentBTreeXLPlusMinusOneRmq { + public: + static_assert(std::is_unsigned_v, + "SegmentBTreeXLPlusMinusOneRmq index type must be unsigned"); + static_assert(LeafSize != 0 && LeafSize % 128 == 0, + "SegmentBTreeXLPlusMinusOneRmq leaf size must be a positive " + "multiple of 128"); + static_assert(Fanout > 1 && Fanout <= 256, + "SegmentBTreeXLPlusMinusOneRmq high fanout must be in " + "[2, 256]"); + static_assert(MiddleFanout > 1 && MiddleFanout <= 256, + "SegmentBTreeXLPlusMinusOneRmq middle fanout must be in " + "[2, 256]"); + + static constexpr std::size_t npos = std::numeric_limits::max(); + static constexpr Index invalid_index = std::numeric_limits::max(); + static constexpr std::size_t kLeafSize = LeafSize; + static constexpr std::size_t kFanout = Fanout; + static constexpr std::size_t kMiddleFanout = MiddleFanout; + + /** + * @brief Construct an empty ±1 RMQ index. + */ + SegmentBTreeXLPlusMinusOneRmq() = default; + + /** + * @brief Build a ±1 RMQ index over external packed delta bits. + * + * @param bits Little-endian packed delta bits. + * @param depth_count Number of indexed depth positions. + * @throws std::length_error if @p Index cannot represent all positions. + * @throws std::invalid_argument if @p bits is shorter than required. + */ + SegmentBTreeXLPlusMinusOneRmq(std::span bits, + std::size_t depth_count) { + build(bits, depth_count); + } + + /** + * @brief Rebuild this index over external packed delta bits. + */ + void build(std::span bits, std::size_t depth_count) { + input_bits_ = bits; + depth_count_ = depth_count; + build(); + } + + /** + * @brief Return the number of indexed depth positions. + */ + std::size_t size() const { return depth_count_; } + + /** + * @brief Whether the indexed depth sequence is empty. + */ + bool empty() const { return depth_count_ == 0; } + + /** + * @brief Return the first minimum depth position in [@p left, @p right). + * + * @details Empty or invalid ranges return `npos`. Equal depths return the + * smaller position. + */ + std::size_t arg_min(std::size_t left, std::size_t right) const { + if (left >= right || right > depth_count_ || level_sizes_.empty()) { + return npos; + } + if (left + 1 == right) { + return left; + } + + const std::size_t root_level = level_count() - 1; + if (left == 0 && right == depth_count_) { + return subtree_min_candidate(root_level, 0).position; + } + + const std::size_t left_leaf = leaf_for_position(left); + const std::size_t right_leaf = leaf_for_position(right - 1); + if (left_leaf == right_leaf) { + return leaf_range_min(left_leaf, left, right).position; + } + + const auto [level, node] = covering_node(left_leaf, right_leaf); + return query_node(level, node, left, right).position; + } + + /** + * @brief Return the one-based rank-th zero delta bit position. + * + * @details This is the select operation needed by the Cartesian wrapper to + * map value endpoints to close-parenthesis positions. High nodes keep + * per-child zero-count prefixes, so top-level descent avoids rescanning the + * 256-way high-node fanout. The returned position is a delta-bit position in + * `[0, size() - 1)`, or `npos` when @p rank is out of range. + */ + std::size_t select0(std::size_t rank) const { + if (rank == 0 || depth_count_ == 0 || level_sizes_.empty()) { + return npos; + } + const std::size_t root_level = level_count() - 1; + if (rank > subtree_zero_count(root_level, 0)) { + return npos; + } + return select0_in_node(root_level, 0, rank); + } + + private: + static constexpr std::size_t kLeafWords = LeafSize / 64; + static constexpr std::size_t kLeafChunks = LeafSize / 128; + static constexpr std::size_t kSelectorEntries = 256; + static constexpr std::size_t kSelectorBits = 2 * kSelectorEntries; + static constexpr std::size_t kSelectorWords = kSelectorBits / 64; + static constexpr std::size_t kHighSparseLevels = + static_cast(std::bit_width(Fanout)); + static constexpr std::size_t kHighSparseSlotsPerNode = + kHighSparseLevels * Fanout; + static constexpr bool kInvalidIndexEqualsNpos = + static_cast(invalid_index) == npos; + + struct DepthCandidate { + std::size_t position = npos; + std::int64_t depth = std::numeric_limits::max(); + }; + + struct LeafSummary { + std::int64_t base_depth = 0; + std::int64_t min_depth = 0; + Index min_offset = 0; + Index zero_count = 0; + }; + + struct HighChildMetadata { + std::size_t position_begin = 0; + std::size_t position_end = 0; + Index min_position = invalid_index; + std::int64_t min_depth = std::numeric_limits::max(); + }; + + class alignas(64) Bp512Selector { + public: + /** + * @brief Construct an empty packed BP selector. + */ + Bp512Selector() = default; + + /** + * @brief Build a stable local Cartesian-tree BP selector over entries. + * + * @details The comparator must return whether the left slot is strictly + * better than the right slot. Equal minima stay stable through the same + * right-to-left construction rule used by the value Cartesian tree. + */ + template + void build(std::size_t entry_count, EntryLess entry_less) { + if (entry_count > kSelectorEntries) { + throw std::length_error( + "SegmentBTreeXLPlusMinusOneRmq local selector too large"); + } + + bp_bits_.fill(0); + if (entry_count == 0) { + return; + } + + std::array stack{}; + std::size_t stack_size = 0; + std::size_t write_position = 2 * entry_count; + + for (std::size_t i = entry_count; i-- > 0;) { + while (stack_size != 0 && !entry_less(stack[stack_size - 1], i)) { + --stack_size; + prepend_bp_bit(write_position, true); + } + stack[stack_size++] = static_cast(i); + prepend_bp_bit(write_position, false); + } + + while (write_position != 0) { + prepend_bp_bit(write_position, true); + } + } + + /** + * @brief Return the first minimum slot in a local entry range. + */ + std::size_t arg_min(std::size_t slot_left, + std::size_t slot_right, + std::size_t entry_count) const { + if (slot_left >= slot_right || slot_right > entry_count || + entry_count > kSelectorEntries) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t bit_count = 2 * entry_count; + const std::size_t first_close = close_position(slot_left); + const std::size_t last_close = close_position(slot_right - 1); + if (first_close > last_close || last_close >= bit_count) { + return npos; + } + + const std::size_t shifted_min = + depth_arg_min(first_close + 1, last_close + 2, bit_count); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + + const std::size_t zero_rank = rank0_at(shifted_min, bit_count); + if (zero_rank == 0) { + return npos; + } + const std::size_t entry = zero_rank - 1; + return entry < entry_count ? entry : npos; + } + + private: + /** + * @brief Prepend one BP bit while building the sequence right-to-left. + */ + void prepend_bp_bit(std::size_t& write_position, bool bit) { + --write_position; + if (bit) { + bp_bits_[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); + } + } + + /** + * @brief Return the BP close position for a local entry slot. + */ + std::size_t close_position(std::size_t slot) const { + return select0_512(bp_bits_.data(), slot); + } + + /** + * @brief Count zero bits before @p position in the raw BP sequence. + */ + std::size_t rank0_at(std::size_t position, std::size_t bit_count) const { + position = std::min(position, bit_count); + return position - rank_512(bp_bits_.data(), position); + } + + /** + * @brief Return open-minus-close excess before a raw BP position. + */ + int prefix_excess(std::size_t position) const { + position = std::min(position, kSelectorBits); + const std::size_t ones = rank_512(bp_bits_.data(), position); + return static_cast(2 * ones) - static_cast(position); + } + + /** + * @brief Return the minimum BP-depth position in a selector depth range. + */ + std::size_t depth_arg_min(std::size_t left, + std::size_t right, + std::size_t bit_count) const { + const std::size_t depth_count = bit_count + 1; + if (left >= right || right > depth_count) { + return npos; + } + + std::size_t position = left; + int best_depth = prefix_excess(position); + std::size_t best_position = position; + + while (position < right) { + const std::size_t chunk_begin = (position / 128) * 128; + const std::size_t local_left = position - chunk_begin; + const std::size_t local_right = + std::min(right - 1, chunk_begin + 128) - chunk_begin; + + int candidate_depth; + std::size_t candidate_position; + if (chunk_begin >= bit_count) { + candidate_depth = prefix_excess(bit_count); + candidate_position = bit_count; + } else { + const std::size_t word = chunk_begin >> 6; + const ExcessResult result = + excess_min_128(bp_bits_.data() + word, local_left, local_right); + candidate_depth = prefix_excess(chunk_begin) + result.min_excess; + candidate_position = chunk_begin + result.offset; + } + + if (candidate_depth < best_depth) { + best_depth = candidate_depth; + best_position = candidate_position; + } + + position = chunk_begin + local_right + 1; + } + + return best_position; + } + + std::array bp_bits_{}; + }; + + static_assert(sizeof(Bp512Selector) == 64); + + /** + * @brief Return whether a stored position is one of the missing sentinels. + */ + bool missing_position(std::size_t position) const { + if constexpr (kInvalidIndexEqualsNpos) { + return position == npos; + } else { + return position == npos || + position == static_cast(invalid_index); + } + } + + /** + * @brief Choose the smaller-depth candidate, breaking ties by position. + */ + DepthCandidate better_candidate(DepthCandidate left, + DepthCandidate right) const { + if (missing_position(left.position)) { + return right; + } + if (missing_position(right.position)) { + return left; + } + if (right.depth < left.depth) { + return right; + } + if (left.depth < right.depth) { + return left; + } + return right.position < left.position ? right : left; + } + + /** + * @brief Return whether @p left is strictly better than @p right. + */ + bool strictly_better_candidate(DepthCandidate left, + DepthCandidate right) const { + if (missing_position(left.position)) { + return false; + } + if (missing_position(right.position)) { + return true; + } + if (left.depth != right.depth) { + return left.depth < right.depth; + } + return left.position < right.position; + } + + /** + * @brief Build all leaf summaries, internal selectors, and top sparse tables. + */ + void build() { + leaf_summaries_.clear(); + internal_selectors_.clear(); + internal_min_positions_.clear(); + internal_min_depths_.clear(); + high_child_metadata_.clear(); + high_sparse_min_slots_.clear(); + high_zero_prefixes_.clear(); + internal_level_offsets_.clear(); + high_level_offsets_.clear(); + level_sizes_.clear(); + level_position_spans_.clear(); + level_fanouts_.clear(); + high_level_begin_ = std::numeric_limits::max(); + + if (depth_count_ == 0) { + return; + } + if (depth_count_ > static_cast(invalid_index)) { + throw std::length_error( + "SegmentBTreeXLPlusMinusOneRmq index type is too small"); + } + if (depth_count_ > + static_cast(std::numeric_limits::max())) { + throw std::length_error( + "SegmentBTreeXLPlusMinusOneRmq depth range is too large"); + } + if (input_bits_.size() < (depth_count_ - 1 + 63) / 64) { + throw std::invalid_argument( + "SegmentBTreeXLPlusMinusOneRmq bit span is too small"); + } + + initialize_layout((depth_count_ + LeafSize - 1) / LeafSize); + build_leaves(); + for (std::size_t level = 1; level < level_count(); ++level) { + for (std::size_t node = 0; node < level_sizes_[level]; ++node) { + build_internal_node(level, node); + } + } + } + + /** + * @brief Compute B-tree level sizes, fanouts, and flat storage offsets. + */ + void initialize_layout(std::size_t leaf_count) { + level_sizes_.push_back(leaf_count); + level_position_spans_.push_back(LeafSize); + level_fanouts_.push_back(0); + + std::size_t current_count = leaf_count; + std::size_t current_span = LeafSize; + while (current_count > Fanout * Fanout) { + level_fanouts_.push_back(MiddleFanout); + current_count = ceil_div(current_count, MiddleFanout); + current_span = saturating_product(current_span, MiddleFanout); + level_sizes_.push_back(current_count); + level_position_spans_.push_back(current_span); + } + while (current_count > 1) { + level_fanouts_.push_back(Fanout); + current_count = ceil_div(current_count, Fanout); + current_span = saturating_product(current_span, Fanout); + level_sizes_.push_back(current_count); + level_position_spans_.push_back(current_span); + } + + leaf_summaries_.resize(level_sizes_[0]); + + internal_level_offsets_.assign(level_count(), 0); + high_level_offsets_.assign(level_count(), 0); + if (level_count() <= 1) { + return; + } + + const std::size_t root_level = level_count() - 1; + high_level_begin_ = root_level > 1 ? root_level - 1 : 1; + + std::size_t internal_count = 0; + for (std::size_t level = 1; level < level_count(); ++level) { + internal_level_offsets_[level] = internal_count; + internal_count += level_sizes_[level]; + } + internal_selectors_.resize(internal_count); + internal_min_positions_.resize(internal_count, invalid_index); + internal_min_depths_.resize(internal_count, + std::numeric_limits::max()); + internal_zero_counts_.resize(internal_count, 0); + + std::size_t high_node_count = 0; + for (std::size_t level = high_level_begin_; level < level_count(); + ++level) { + high_level_offsets_[level] = high_node_count; + high_node_count += level_sizes_[level]; + } + high_child_metadata_.resize(high_node_count * Fanout); + high_sparse_min_slots_.resize(high_node_count * kHighSparseSlotsPerNode); + high_zero_prefixes_.resize(high_node_count * (Fanout + 1)); + } + + /** + * @brief Build all leaf summaries while carrying the absolute base depth. + */ + void build_leaves() { + std::int64_t base_depth = 0; + for (std::size_t leaf = 0; leaf < leaf_summaries_.size(); ++leaf) { + const std::size_t count = entry_count(0, leaf); + DepthCandidate minimum = + scan_leaf_range_with_base(leaf, 0, count - 1, base_depth); + const std::size_t zero_count = + leaf_zero_count(leaf, next_leaf_delta_count(leaf)); + leaf_summaries_[leaf] = { + base_depth, + minimum.depth, + static_cast(minimum.position - node_position_begin(0, leaf)), + static_cast(zero_count), + }; + if (leaf + 1 < leaf_summaries_.size()) { + base_depth += leaf_excess(leaf, next_leaf_delta_count(leaf)); + } + } + } + + /** + * @brief Build one internal node selector and cached minimum. + */ + void build_internal_node(std::size_t level, std::size_t node) { + const std::size_t count = entry_count(level, node); + const std::size_t first_child = node * fanout_at_level(level); + const bool high_level = is_high_level(level); + const std::size_t high_flat = high_level ? high_flat_index(level, node) : 0; + + if (high_level) { + std::size_t zero_prefix = 0; + std::size_t* prefixes = mutable_high_zero_prefixes_begin(high_flat); + prefixes[0] = 0; + for (std::size_t slot = 0; slot < count; ++slot) { + const std::size_t child = first_child + slot; + const DepthCandidate child_min = + subtree_min_candidate(level - 1, child); + zero_prefix += subtree_zero_count(level - 1, child); + prefixes[slot + 1] = zero_prefix; + HighChildMetadata& metadata = + mutable_high_child_metadata_at(high_flat, slot); + metadata.position_begin = node_position_begin(level - 1, child); + metadata.position_end = node_position_end(level - 1, child); + metadata.min_position = static_cast(child_min.position); + metadata.min_depth = child_min.depth; + } + build_high_sparse_min_slots(level, node, count); + } + + Bp512Selector& selector = mutable_selector_at(level, node); + selector.build(count, [&](std::size_t left, std::size_t right) { + return strictly_better_candidate(child_min_candidate(level, node, left), + child_min_candidate(level, node, right)); + }); + + const std::size_t slot = selector.arg_min(0, count, count); + const DepthCandidate minimum = child_min_candidate(level, node, slot); + const std::size_t flat = internal_flat_index(level, node); + internal_min_positions_[flat] = static_cast(minimum.position); + internal_min_depths_[flat] = minimum.depth; + if (high_level) { + internal_zero_counts_[flat] = high_zero_prefixes_begin(high_flat)[count]; + } else { + std::size_t zero_count = 0; + for (std::size_t slot = 0; slot < count; ++slot) { + zero_count += subtree_zero_count(level - 1, first_child + slot); + } + internal_zero_counts_[flat] = zero_count; + } + } + + /** + * @brief Return `ceil(value / divisor)` for positive divisors. + */ + static std::size_t ceil_div(std::size_t value, std::size_t divisor) { + return (value + divisor - 1) / divisor; + } + + /** + * @brief Multiply two extents, saturating at `std::size_t` maximum. + */ + static std::size_t saturating_product(std::size_t left, std::size_t right) { + if (left != 0 && right > std::numeric_limits::max() / left) { + return std::numeric_limits::max(); + } + return left * right; + } + + /** + * @brief Read an input word or return zero past the available span. + */ + std::uint64_t word_or_zero(std::size_t word) const { + return word < input_bits_.size() ? input_bits_[word] : 0; + } + + /** + * @brief Load all delta words aligned to a leaf boundary. + */ + std::array leaf_bits(std::size_t leaf) const { + std::array bits{}; + const std::size_t first_word = leaf * kLeafWords; + for (std::size_t word = 0; word < kLeafWords; ++word) { + bits[word] = word_or_zero(first_word + word); + } + return bits; + } + + /** + * @brief Return total excess over the first @p delta_count bits of a leaf. + */ + std::int64_t leaf_excess(std::size_t leaf, std::size_t delta_count) const { + const auto bits = leaf_bits(leaf); + std::int64_t result = 0; + std::size_t remaining = std::min(delta_count, LeafSize); + for (std::size_t chunk = 0; chunk < kLeafChunks && remaining != 0; + ++chunk) { + const std::size_t chunk_count = std::min(128, remaining); + result += prefix_excess_128(bits.data() + 2 * chunk, chunk_count); + remaining -= chunk_count; + } + return result; + } + + /** + * @brief Return the number of zero delta bits in one leaf prefix. + */ + std::size_t leaf_zero_count(std::size_t leaf, std::size_t delta_count) const { + const auto bits = leaf_bits(leaf); + std::size_t result = 0; + std::size_t remaining = std::min(delta_count, LeafSize); + for (std::size_t word = 0; word < kLeafWords && remaining != 0; ++word) { + const std::size_t word_bits = std::min(64, remaining); + result += + word_bits - std::popcount(bits[word] & first_bits_mask(word_bits)); + remaining -= word_bits; + } + return result; + } + + /** + * @brief Count delta bits from a leaf start to the next leaf base depth. + */ + std::size_t next_leaf_delta_count(std::size_t leaf) const { + const std::size_t begin = node_position_begin(0, leaf); + const std::size_t next_begin = begin + LeafSize; + return std::min(next_begin, depth_count_ - 1) - begin; + } + + /** + * @brief Return the first minimum candidate inside one leaf local range. + */ + DepthCandidate scan_leaf_range_with_base(std::size_t leaf, + std::size_t left_offset, + std::size_t right_offset, + std::int64_t base_depth) const { + const std::size_t begin = node_position_begin(0, leaf); + const std::size_t count = entry_count(0, leaf); + if (count == 0 || left_offset > right_offset || left_offset >= count) { + return {}; + } + right_offset = std::min(right_offset, count - 1); + + const auto bits = leaf_bits(leaf); + DepthCandidate answer; + std::int64_t chunk_base_excess = 0; + for (std::size_t chunk = 0; chunk < kLeafChunks; ++chunk) { + const std::size_t chunk_begin = chunk * 128; + if (chunk_begin >= count || chunk_begin > right_offset) { + break; + } + + const std::size_t chunk_end = + std::min(count - 1, chunk_begin + 127); + if (left_offset > chunk_end) { + chunk_base_excess += prefix_excess_128(bits.data() + 2 * chunk, 128); + continue; + } + + const std::size_t local_left = + std::max(left_offset, chunk_begin) - chunk_begin; + const std::size_t local_right = + std::min(right_offset, chunk_end) - chunk_begin; + const ExcessResult result = + excess_min_128(bits.data() + 2 * chunk, local_left, local_right); + const std::size_t offset = chunk_begin + result.offset; + if (result.offset != npos && offset < count) { + answer = better_candidate( + answer, {begin + offset, + base_depth + chunk_base_excess + result.min_excess}); + } + + chunk_base_excess += prefix_excess_128(bits.data() + 2 * chunk, 128); + } + return answer; + } + + /** + * @brief Return the minimum candidate in a possibly partial leaf. + */ + DepthCandidate leaf_range_min(std::size_t leaf, + std::size_t left, + std::size_t right) const { + if (left >= right) { + return {}; + } + + const std::size_t begin = node_position_begin(0, leaf); + const std::size_t end = node_position_end(0, leaf); + if (left <= begin && end <= right) { + return subtree_min_candidate(0, leaf); + } + + const std::size_t slot_left = left - begin; + const std::size_t slot_right = right - begin; + return scan_leaf_range_with_base(leaf, slot_left, slot_right - 1, + leaf_summaries_[leaf].base_depth); + } + + /** + * @brief Find the lowest tree node that contains both endpoint leaves. + */ + std::pair covering_node( + std::size_t left_leaf, + std::size_t right_leaf) const { + std::size_t level = 0; + std::size_t left_node = left_leaf; + std::size_t right_node = right_leaf; + while (left_node != right_node) { + ++level; + const std::size_t fanout = fanout_at_level(level); + left_node /= fanout; + right_node /= fanout; + } + return {level, left_node}; + } + + /** + * @brief Return the leaf index containing a depth position. + */ + std::size_t leaf_for_position(std::size_t position) const { + return position / LeafSize; + } + + /** + * @brief Return the child index at @p child_level containing a position. + */ + std::size_t child_for_position(std::size_t child_level, + std::size_t position) const { + return position / level_position_spans_[child_level]; + } + + /** + * @brief Query a child-slot range of an internal node. + */ + DepthCandidate query_child_slots(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t left, + std::size_t right) const { + if (slot_left >= slot_right) { + return {}; + } + + const std::size_t count = entry_count(level, node); + const std::size_t slot = + slot_left + 1 == slot_right + ? slot_left + : selector_arg_min(level, node, slot_left, slot_right, count); + if (slot == npos) { + return {}; + } + + const std::size_t child_level = level - 1; + const std::size_t first_child = node * fanout_at_level(level); + const std::size_t child = first_child + slot; + const HighChildMetadata* high_children = + is_high_level(level) ? high_child_metadata_begin(level, node) : nullptr; + const DepthCandidate child_min = + high_children != nullptr ? high_child_min_candidate(level, node, slot) + : subtree_min_candidate(child_level, child); + const std::size_t child_begin = + high_children != nullptr ? high_children[slot].position_begin + : node_position_begin(child_level, child); + const std::size_t child_end = high_children != nullptr + ? high_children[slot].position_end + : node_position_end(child_level, child); + if ((left <= child_begin && child_end <= right) || + contains_position(left, right, child_min.position)) { + return child_min; + } + + const std::size_t last_slot = slot_right - 1; + const std::size_t left_child_begin = + high_children != nullptr + ? high_children[slot_left].position_begin + : node_position_begin(child_level, first_child + slot_left); + const std::size_t left_child_end = + high_children != nullptr + ? high_children[slot_left].position_end + : node_position_end(child_level, first_child + slot_left); + DepthCandidate answer = query_node(child_level, first_child + slot_left, + std::max(left, left_child_begin), + std::min(right, left_child_end)); + + if (slot_left != last_slot) { + const std::size_t right_child_begin = + high_children != nullptr + ? high_children[last_slot].position_begin + : node_position_begin(child_level, first_child + last_slot); + const std::size_t right_child_end = + high_children != nullptr + ? high_children[last_slot].position_end + : node_position_end(child_level, first_child + last_slot); + answer = better_candidate(answer, + query_node(child_level, first_child + last_slot, + std::max(left, right_child_begin), + std::min(right, right_child_end))); + } + + if (slot_left + 1 < last_slot) { + answer = better_candidate( + answer, + full_child_slot_range_min(level, node, slot_left + 1, last_slot)); + } + + return answer; + } + + /** + * @brief Return the best candidate among fully covered child slots. + */ + DepthCandidate full_child_slot_range_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right) const { + if (slot_left >= slot_right) { + return {}; + } + + const std::size_t slot = + slot_left + 1 == slot_right + ? slot_left + : selector_arg_min(level, node, slot_left, slot_right, + entry_count(level, node)); + if (slot == npos) { + return {}; + } + return child_min_candidate(level, node, slot); + } + + /** + * @brief Query a tree node for the minimum candidate in a depth range. + */ + DepthCandidate query_node(std::size_t level, + std::size_t node, + std::size_t left, + std::size_t right) const { + if (left >= right) { + return {}; + } + + const std::size_t begin = node_position_begin(level, node); + const std::size_t end = node_position_end(level, node); + if (left <= begin && end <= right) { + return subtree_min_candidate(level, node); + } + if (level == 0) { + return leaf_range_min(node, left, right); + } + + const std::size_t child_level = level - 1; + const std::size_t left_child = child_for_position(child_level, left); + const std::size_t right_child = child_for_position(child_level, right - 1); + const std::size_t first_child = node * fanout_at_level(level); + const std::size_t left_slot = left_child - first_child; + const std::size_t right_slot = right_child - first_child + 1; + return query_child_slots(level, node, left_slot, right_slot, left, right); + } + + /** + * @brief Return whether a position lies inside a half-open range. + */ + bool contains_position(std::size_t left, + std::size_t right, + std::size_t position) const { + return !missing_position(position) && left <= position && position < right; + } + + /** + * @brief Return the number of B-tree levels, including leaves. + */ + std::size_t level_count() const { return level_sizes_.size(); } + + /** + * @brief Return the number of entries in a leaf or child slots in a node. + */ + std::size_t entry_count(std::size_t level, std::size_t node) const { + if (level == 0) { + const std::size_t begin = node_position_begin(0, node); + return std::min(LeafSize, depth_count_ - begin); + } + const std::size_t first_child = node * fanout_at_level(level); + return std::min(fanout_at_level(level), + level_sizes_[level - 1] - first_child); + } + + /** + * @brief Return the first depth position covered by a node. + */ + std::size_t node_position_begin(std::size_t level, std::size_t node) const { + return node * level_position_spans_[level]; + } + + /** + * @brief Return one past the last depth position covered by a node. + */ + std::size_t node_position_end(std::size_t level, std::size_t node) const { + return std::min(depth_count_, node_position_begin(level, node) + + level_position_spans_[level]); + } + + /** + * @brief Return a node's cached subtree-minimum candidate. + */ + DepthCandidate subtree_min_candidate(std::size_t level, + std::size_t node) const { + if (level == 0) { + const LeafSummary& summary = leaf_summaries_[node]; + return {node_position_begin(0, node) + summary.min_offset, + summary.min_depth}; + } + const std::size_t flat = internal_flat_index(level, node); + return {static_cast(internal_min_positions_[flat]), + internal_min_depths_[flat]}; + } + + /** + * @brief Return a node's cached subtree zero count. + */ + std::size_t subtree_zero_count(std::size_t level, std::size_t node) const { + if (level == 0) { + return leaf_summaries_[node].zero_count; + } + return internal_zero_counts_[internal_flat_index(level, node)]; + } + + /** + * @brief Return a child slot's subtree-minimum candidate. + */ + DepthCandidate child_min_candidate(std::size_t level, + std::size_t node, + std::size_t slot) const { + if (is_high_level(level)) { + return high_child_min_candidate(level, node, slot); + } + return subtree_min_candidate(level - 1, + node * fanout_at_level(level) + slot); + } + + /** + * @brief Return a high-node child candidate using cached metadata. + */ + DepthCandidate high_child_min_candidate(std::size_t level, + std::size_t node, + std::size_t slot) const { + const HighChildMetadata& metadata = + high_child_metadata_at(high_flat_index(level, node), slot); + return {static_cast(metadata.min_position), + metadata.min_depth}; + } + + /** + * @brief Return an immutable internal-node BP selector. + */ + const Bp512Selector& selector_at(std::size_t level, std::size_t node) const { + return internal_selectors_[internal_flat_index(level, node)]; + } + + /** + * @brief Return a mutable internal-node BP selector while building. + */ + Bp512Selector& mutable_selector_at(std::size_t level, std::size_t node) { + return internal_selectors_[internal_flat_index(level, node)]; + } + + /** + * @brief Run the appropriate local selector for an internal node. + */ + std::size_t selector_arg_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t count) const { + if (is_high_level(level)) { + return high_sparse_arg_min(level, node, slot_left, slot_right, count); + } + return selector_at(level, node).arg_min(slot_left, slot_right, count); + } + + /** + * @brief Return whether a level uses the high-node layout. + */ + bool is_high_level(std::size_t level) const { + return level > 0 && level >= high_level_begin_ && level < level_count(); + } + + /** + * @brief Map an internal node to its flat storage index. + */ + std::size_t internal_flat_index(std::size_t level, std::size_t node) const { + return internal_level_offsets_[level] + node; + } + + /** + * @brief Return the fanout used to group children at a level. + */ + std::size_t fanout_at_level(std::size_t level) const { + return level_fanouts_[level]; + } + + /** + * @brief Map a high-level node to its flat high-node storage index. + */ + std::size_t high_flat_index(std::size_t level, std::size_t node) const { + return high_level_offsets_[level] + node; + } + + /** + * @brief Return the first child-metadata record for a high node. + */ + const HighChildMetadata* high_child_metadata_begin(std::size_t level, + std::size_t node) const { + return high_child_metadata_.data() + high_flat_index(level, node) * Fanout; + } + + /** + * @brief Return high-child metadata by flat high-node index and slot. + */ + const HighChildMetadata& high_child_metadata_at(std::size_t high_flat, + std::size_t slot) const { + return high_child_metadata_[high_flat * Fanout + slot]; + } + + /** + * @brief Return mutable high-child metadata while building. + */ + HighChildMetadata& mutable_high_child_metadata_at(std::size_t high_flat, + std::size_t slot) { + return high_child_metadata_[high_flat * Fanout + slot]; + } + + /** + * @brief Return mutable sparse-slot storage for one high node. + */ + std::uint8_t* mutable_high_sparse_min_slots_begin(std::size_t high_flat) { + return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; + } + + /** + * @brief Return sparse-slot storage for one high node. + */ + const std::uint8_t* high_sparse_min_slots_begin(std::size_t high_flat) const { + return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; + } + + /** + * @brief Return mutable zero-prefix storage for one high node. + */ + std::size_t* mutable_high_zero_prefixes_begin(std::size_t high_flat) { + return high_zero_prefixes_.data() + high_flat * (Fanout + 1); + } + + /** + * @brief Return zero-prefix storage for one high node. + */ + const std::size_t* high_zero_prefixes_begin(std::size_t high_flat) const { + return high_zero_prefixes_.data() + high_flat * (Fanout + 1); + } + + /** + * @brief Choose the better high-node child slot. + */ + std::size_t better_high_child_slot(std::size_t level, + std::size_t node, + std::size_t left_slot, + std::size_t right_slot) const { + const DepthCandidate left = + high_child_min_candidate(level, node, left_slot); + const DepthCandidate right = + high_child_min_candidate(level, node, right_slot); + return better_candidate(left, right).position == right.position ? right_slot + : left_slot; + } + + /** + * @brief Build sparse tables over high-node child minima. + */ + void build_high_sparse_min_slots(std::size_t level, + std::size_t node, + std::size_t count) { + const std::size_t high_flat = high_flat_index(level, node); + std::uint8_t* table = mutable_high_sparse_min_slots_begin(high_flat); + for (std::size_t slot = 0; slot < count; ++slot) { + table[slot] = static_cast(slot); + } + + for (std::size_t table_level = 1; table_level < kHighSparseLevels; + ++table_level) { + const std::size_t span = std::size_t{1} << table_level; + if (span > count) { + break; + } + const std::size_t half_span = span >> 1; + const std::uint8_t* previous = table + (table_level - 1) * Fanout; + std::uint8_t* current = table + table_level * Fanout; + for (std::size_t slot = 0; slot + span <= count; ++slot) { + current[slot] = static_cast(better_high_child_slot( + level, node, previous[slot], previous[slot + half_span])); + } + } + } + + /** + * @brief Return the best high-node child slot in a slot range. + */ + std::size_t high_sparse_arg_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t count) const { + if (slot_left >= slot_right || slot_right > count) { + return npos; + } + const std::size_t length = slot_right - slot_left; + if (length == 1) { + return slot_left; + } + + const std::size_t high_flat = high_flat_index(level, node); + const std::size_t table_level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << table_level; + const std::uint8_t* table = + high_sparse_min_slots_begin(high_flat) + table_level * Fanout; + return better_high_child_slot(level, node, table[slot_left], + table[slot_right - span]); + } + + /** + * @brief Descend the tree to select the rank-th zero bit. + */ + std::size_t select0_in_node(std::size_t level, + std::size_t node, + std::size_t rank) const { + if (level == 0) { + return select0_in_leaf(node, rank); + } + + const std::size_t count = entry_count(level, node); + std::size_t slot = 0; + if (is_high_level(level)) { + const std::size_t high_flat = high_flat_index(level, node); + const std::size_t* prefixes = high_zero_prefixes_begin(high_flat); + const std::size_t* first = prefixes + 1; + const std::size_t* last = prefixes + count + 1; + const std::size_t* selected = std::lower_bound(first, last, rank); + if (selected == last) { + return npos; + } + slot = static_cast(selected - first); + rank -= prefixes[slot]; + } else { + const std::size_t first_child = node * fanout_at_level(level); + for (; slot < count; ++slot) { + const std::size_t child_zeros = + subtree_zero_count(level - 1, first_child + slot); + if (rank <= child_zeros) { + break; + } + rank -= child_zeros; + } + if (slot == count) { + return npos; + } + } + + return select0_in_node(level - 1, node * fanout_at_level(level) + slot, + rank); + } + + /** + * @brief Select the rank-th zero bit inside one leaf. + */ + std::size_t select0_in_leaf(std::size_t leaf, std::size_t rank) const { + const std::size_t bit_begin = node_position_begin(0, leaf); + const std::size_t delta_count = next_leaf_delta_count(leaf); + const auto bits = leaf_bits(leaf); + std::size_t remaining = delta_count; + for (std::size_t word = 0; word < kLeafWords && remaining != 0; ++word) { + const std::size_t word_bits = std::min(64, remaining); + const std::uint64_t candidates = + (~bits[word]) & first_bits_mask(word_bits); + const std::size_t count = std::popcount(candidates); + if (rank <= count) { + return bit_begin + word * 64 + select_64(candidates, rank - 1); + } + rank -= count; + remaining -= word_bits; + } + return npos; + } + + std::span input_bits_; + std::size_t depth_count_ = 0; + std::vector leaf_summaries_; + std::vector internal_selectors_; + std::vector internal_min_positions_; + std::vector internal_min_depths_; + std::vector internal_zero_counts_; + std::vector high_child_metadata_; + std::vector high_sparse_min_slots_; + std::vector high_zero_prefixes_; + std::vector internal_level_offsets_; + std::vector high_level_offsets_; + std::vector level_sizes_; + std::vector level_position_spans_; + std::vector level_fanouts_; + std::size_t high_level_begin_ = std::numeric_limits::max(); +}; + +/** + * @brief Experimental Cartesian-tree value RMQ using SegmentBTreeXL-style LCA. + * + * @details This class keeps the same public value-RMQ contract as + * `CartesianTreeRmq`. It builds the same stable Ferrada-Navarro BP + * Cartesian-tree encoding, uses `BitVector` for close-parenthesis rank, keeps a + * dense close-select sample table for value endpoint mapping, and delegates the + * BP-depth minimum query to `SegmentBTreeXLPlusMinusOneRmq`. A single coarse + * value-level sparse table is checked first; it uses at least 4096-value blocks + * and grows the block width when needed so the top layer has at most 2^14 + * blocks. Wide queries whose padded block-cover minimum lies inside the + * requested range return from this top table without touching the global BP + * rank/select path. + * + * The implementation is intentionally kept in the experimental namespace and + * is not included from `pixie/rmq.h`. It is meant to compare the usual + * Cartesian-tree reduction against a SegmentBTreeXL-shaped ±1 RMQ backend. + */ +template , + class Index = std::size_t, + std::size_t LeafSize = 512, + std::size_t Fanout = 256, + std::size_t MiddleFanout = 192> +class CartesianTreeSegmentBTreeXLRmq + : public RmqBase, + T> { + public: + static_assert(std::is_unsigned_v, + "CartesianTreeSegmentBTreeXLRmq index type must be unsigned"); + + using Self = CartesianTreeSegmentBTreeXLRmq; + + static constexpr std::size_t npos = RmqBase::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + static constexpr std::size_t kMinTopSparseBlockSize = 4096; + static constexpr std::size_t kMaxTopSparseBlocks = std::size_t{1} << 14; + + /** + * @brief Construct an empty Cartesian-tree RMQ index. + */ + CartesianTreeSegmentBTreeXLRmq() = default; + + /** + * @brief Build an experimental Cartesian-tree RMQ index over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values stay stable: the smaller index remains the first minimum. + */ + explicit CartesianTreeSegmentBTreeXLRmq(std::span values, + Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + /** + * @brief Copy an RMQ index and rebuild internal non-owning views. + */ + CartesianTreeSegmentBTreeXLRmq(const CartesianTreeSegmentBTreeXLRmq& other) + : values_(other.values_), + compare_(other.compare_), + bp_bits_(other.bp_bits_), + bp_bit_count_(other.bp_bit_count_), + close_select_samples_(other.close_select_samples_), + top_sparse_candidates_(other.top_sparse_candidates_), + top_block_size_(other.top_block_size_), + top_block_count_(other.top_block_count_), + top_sparse_levels_(other.top_sparse_levels_) { + reset_bp_indexes(); + } + + /** + * @brief Copy-assign an RMQ index and rebuild internal non-owning views. + */ + CartesianTreeSegmentBTreeXLRmq& operator=( + const CartesianTreeSegmentBTreeXLRmq& other) { + if (this == &other) { + return *this; + } + values_ = other.values_; + compare_ = other.compare_; + bp_bits_ = other.bp_bits_; + bp_bit_count_ = other.bp_bit_count_; + close_select_samples_ = other.close_select_samples_; + top_sparse_candidates_ = other.top_sparse_candidates_; + top_block_size_ = other.top_block_size_; + top_block_count_ = other.top_block_count_; + top_sparse_levels_ = other.top_sparse_levels_; + reset_bp_indexes(); + return *this; + } + + /** + * @brief Move an RMQ index and rebuild internal non-owning views. + */ + CartesianTreeSegmentBTreeXLRmq( + CartesianTreeSegmentBTreeXLRmq&& other) noexcept + : values_(other.values_), + compare_(std::move(other.compare_)), + bp_bits_(std::move(other.bp_bits_)), + bp_bit_count_(other.bp_bit_count_), + close_select_samples_(std::move(other.close_select_samples_)), + top_sparse_candidates_(std::move(other.top_sparse_candidates_)), + top_block_size_(other.top_block_size_), + top_block_count_(other.top_block_count_), + top_sparse_levels_(other.top_sparse_levels_) { + other.values_ = std::span(); + other.bp_bit_count_ = 0; + other.top_block_size_ = kMinTopSparseBlockSize; + other.top_block_count_ = 0; + other.top_sparse_levels_ = 0; + reset_bp_indexes(); + } + + /** + * @brief Move-assign an RMQ index and rebuild internal non-owning views. + */ + CartesianTreeSegmentBTreeXLRmq& operator=( + CartesianTreeSegmentBTreeXLRmq&& other) noexcept { + if (this == &other) { + return *this; + } + values_ = other.values_; + compare_ = std::move(other.compare_); + bp_bits_ = std::move(other.bp_bits_); + bp_bit_count_ = other.bp_bit_count_; + close_select_samples_ = std::move(other.close_select_samples_); + top_sparse_candidates_ = std::move(other.top_sparse_candidates_); + top_block_size_ = other.top_block_size_; + top_block_count_ = other.top_block_count_; + top_sparse_levels_ = other.top_sparse_levels_; + other.values_ = std::span(); + other.bp_bit_count_ = 0; + other.top_block_size_ = kMinTopSparseBlockSize; + other.top_block_count_ = 0; + other.top_sparse_levels_ = 0; + reset_bp_indexes(); + return *this; + } + + /** + * @brief Return the number of indexed values. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size()) { + return npos; + } + if (right - left <= top_block_size_) { + return cartesian_arg_min(left, right); + } + const std::size_t top_answer = top_sparse_arg_min(left, right); + if (top_answer != npos) { + return top_answer; + } + return cartesian_arg_min(left, right); + } + + /** + * @brief Return the number of BP bits in the Cartesian-tree RMQ encoding. + */ + std::size_t bp_bit_count() const { return bp_bit_count_; } + + /** + * @brief Return the packed BP words used by the RMQ encoding. + */ + std::span bp_words() const { return bp_bits_; } + + /** + * @brief Return the top sparse-table block width chosen for a value count. + */ + static std::size_t top_sparse_block_size_for(std::size_t value_count) { + if (value_count == 0) { + return kMinTopSparseBlockSize; + } + return std::max(kMinTopSparseBlockSize, + ceil_div(value_count, kMaxTopSparseBlocks)); + } + + /** + * @brief Return the number of top sparse-table blocks for a value count. + */ + static std::size_t top_sparse_block_count_for(std::size_t value_count) { + if (value_count == 0) { + return 0; + } + return ceil_div(value_count, top_sparse_block_size_for(value_count)); + } + + /** + * @brief Return the current top sparse-table block width. + */ + std::size_t top_sparse_block_size() const { return top_block_size_; } + + /** + * @brief Return the current number of top sparse-table blocks. + */ + std::size_t top_sparse_block_count() const { return top_block_count_; } + + private: + using BpDepthRmq = + SegmentBTreeXLPlusMinusOneRmq; + static constexpr std::size_t kCloseSelectSampleFrequency = 256; + + struct TopCandidate { + Index position = invalid_index; + Index close_position = invalid_index; + }; + static_assert(sizeof(TopCandidate) == 2 * sizeof(Index)); + + /** + * @brief Return the first minimum position through the Cartesian BP + * reduction. + */ + std::size_t cartesian_arg_min(std::size_t left, std::size_t right) const { + const std::size_t first_close = cached_select0(left + 1); + const std::size_t last_close = cached_select0(right); + if (first_close == npos || last_close == npos || first_close > last_close) { + return npos; + } + + const std::size_t shifted_min = + bp_depth_rmq_.arg_min(first_close + 1, last_close + 2); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + const BitVector& bp_index = *bp_index_; + const std::size_t answer = bp_index.rank0(shifted_min) - 1; + return answer < values_.size() ? answer : npos; + } + + /** + * @brief Rebuild the BP Cartesian-tree representation and support indexes. + */ + void build() { + bp_bits_.clear(); + bp_bit_count_ = 0; + close_select_samples_.clear(); + top_sparse_candidates_.clear(); + top_block_size_ = kMinTopSparseBlockSize; + top_block_count_ = 0; + top_sparse_levels_ = 0; + reset_bp_indexes(); + + if (values_.empty()) { + return; + } + if (values_.size() > (static_cast(invalid_index) - 1) / 2) { + throw std::length_error( + "Cartesian SegmentBTreeXL RMQ index type is too small"); + } + + bp_bit_count_ = 2 * values_.size(); + bp_bits_.assign((bp_bit_count_ + 63) / 64, 0); + build_bp_bits(); + build_close_select_samples(); + build_top_sparse_table(); + reset_bp_indexes(); + } + + /** + * @brief Build the Ferrada-Navarro BP bits with a monotone stack. + */ + void build_bp_bits() { + const auto stack = std::make_unique_for_overwrite(values_.size()); + std::size_t stack_size = 0; + std::size_t write_position = bp_bit_count_; + + for (std::size_t i = values_.size(); i-- > 0;) { + while (stack_size != 0 && + !compare_(values_[stack[stack_size - 1]], values_[i])) { + --stack_size; + prepend_bp_bit(write_position, true); + } + stack[stack_size++] = static_cast(i); + prepend_bp_bit(write_position, false); + } + + while (write_position != 0) { + prepend_bp_bit(write_position, true); + } + } + + /** + * @brief Prepend one BP bit into the right-to-left construction buffer. + */ + void prepend_bp_bit(std::size_t& write_position, bool bit) { + --write_position; + if (bit) { + bp_bits_[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); + } + } + + /** + * @brief Rebuild indexes that store non-owning spans into `bp_bits_`. + */ + void reset_bp_indexes() { + bp_index_.reset(); + bp_depth_rmq_ = BpDepthRmq(); + if (bp_bit_count_ == 0) { + return; + } + const std::span words(bp_bits_); + bp_index_.emplace(words, bp_bit_count_); + bp_depth_rmq_ = BpDepthRmq(words, bp_bit_count_ + 1); + } + + /** + * @brief Build dense samples for close-parenthesis select queries. + */ + void build_close_select_samples() { + close_select_samples_.clear(); + close_select_samples_.reserve( + (values_.size() + kCloseSelectSampleFrequency - 1) / + kCloseSelectSampleFrequency); + + std::size_t zero_rank = 0; + for (std::size_t word = 0; word < bp_bits_.size(); ++word) { + const std::size_t word_begin = word * 64; + const std::size_t word_bits = + std::min(64, bp_bit_count_ - word_begin); + std::uint64_t zeros = (~bp_bits_[word]) & first_bits_mask(word_bits); + while (zeros != 0) { + const std::size_t offset = std::countr_zero(zeros); + ++zero_rank; + if ((zero_rank - 1) % kCloseSelectSampleFrequency == 0) { + close_select_samples_.push_back( + static_cast(word_begin + offset)); + } + zeros &= zeros - 1; + } + } + } + + /** + * @brief Build a single top sparse table over original-value block minima. + */ + void build_top_sparse_table() { + top_sparse_candidates_.clear(); + top_block_size_ = top_sparse_block_size_for(values_.size()); + top_block_count_ = ceil_div(values_.size(), top_block_size_); + top_sparse_levels_ = + top_block_count_ == 0 ? 0 : std::bit_width(top_block_count_); + if (top_block_count_ == 0) { + return; + } + + top_sparse_candidates_.assign(top_sparse_levels_ * top_block_count_, + TopCandidate{}); + for (std::size_t block = 0; block < top_block_count_; ++block) { + const std::size_t begin = block * top_block_size_; + const std::size_t end = std::min(values_.size(), begin + top_block_size_); + std::size_t minimum = begin; + for (std::size_t position = begin + 1; position < end; ++position) { + if (strictly_better_value_position(position, minimum)) { + minimum = position; + } + } + top_sparse_candidates_[block] = make_top_candidate(minimum); + } + + for (std::size_t level = 1; level < top_sparse_levels_; ++level) { + const std::size_t span = std::size_t{1} << level; + const std::size_t half_span = span >> 1; + TopCandidate* current = + top_sparse_candidates_.data() + level * top_block_count_; + const TopCandidate* previous = + top_sparse_candidates_.data() + (level - 1) * top_block_count_; + for (std::size_t block = 0; block + span <= top_block_count_; ++block) { + current[block] = + better_top_candidate(previous[block], previous[block + half_span]); + } + } + } + + /** + * @brief Return `ceil(value / divisor)` for positive divisors. + */ + static std::size_t ceil_div(std::size_t value, std::size_t divisor) { + return value == 0 ? 0 : 1 + (value - 1) / divisor; + } + + /** + * @brief Wrap an original value position with its Cartesian close position. + */ + TopCandidate make_top_candidate(std::size_t position) const { + if (position >= values_.size()) { + return {}; + } + return {static_cast(position), + static_cast(cached_select0(position + 1))}; + } + + /** + * @brief Return whether @p position is a valid stored original-value index. + */ + bool valid_value_position(std::size_t position) const { + return position != npos && + position != static_cast(invalid_index) && + position < values_.size(); + } + + /** + * @brief Return whether value position @p left is strictly better than @p + * right. + */ + bool strictly_better_value_position(std::size_t left, + std::size_t right) const { + if (!valid_value_position(left)) { + return false; + } + if (!valid_value_position(right)) { + return true; + } + if (compare_(values_[left], values_[right])) { + return true; + } + if (compare_(values_[right], values_[left])) { + return false; + } + return left < right; + } + + /** + * @brief Choose the better original-value candidate, preserving first ties. + */ + TopCandidate better_top_candidate(TopCandidate left, + TopCandidate right) const { + const std::size_t left_position = static_cast(left.position); + const std::size_t right_position = static_cast(right.position); + return strictly_better_value_position(right_position, left_position) ? right + : left; + } + + /** + * @brief Return the sparse-table candidate over a top-block range. + */ + TopCandidate top_sparse_block_arg_min(std::size_t block_left, + std::size_t block_right) const { + if (block_left >= block_right || block_right > top_block_count_ || + top_sparse_levels_ == 0) { + return {}; + } + const std::size_t length = block_right - block_left; + const std::size_t level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << level; + const TopCandidate* table = + top_sparse_candidates_.data() + level * top_block_count_; + return better_top_candidate(table[block_left], table[block_right - span]); + } + + /** + * @brief Return whether a candidate lies inside a half-open value range. + */ + bool top_candidate_inside(TopCandidate candidate, + std::size_t left, + std::size_t right) const { + const std::size_t position = static_cast(candidate.position); + return valid_value_position(position) && left <= position && + position < right; + } + + /** + * @brief Return the top-overlay answer, or `npos` when the BP path should + * run. + */ + std::size_t top_sparse_arg_min(std::size_t left, std::size_t right) const { + if (top_block_count_ <= 1) { + return npos; + } + + const std::size_t padded_block_left = left / top_block_size_; + const std::size_t padded_block_right = (right - 1) / top_block_size_ + 1; + if (padded_block_left + 1 >= padded_block_right) { + return npos; + } + + const TopCandidate padded = + top_sparse_block_arg_min(padded_block_left, padded_block_right); + if (top_candidate_inside(padded, left, right)) { + return static_cast(padded.position); + } + + const std::size_t first_full_block = + (left + top_block_size_ - 1) / top_block_size_; + const std::size_t full_block_right = right / top_block_size_; + if (first_full_block >= full_block_right) { + return npos; + } + + TopCandidate answer = + top_sparse_block_arg_min(first_full_block, full_block_right); + + const std::size_t left_border_end = first_full_block * top_block_size_; + if (left < left_border_end) { + answer = better_top_candidate( + answer, make_top_candidate(cartesian_arg_min(left, left_border_end))); + } + + const std::size_t right_border_begin = full_block_right * top_block_size_; + if (right_border_begin < right) { + answer = better_top_candidate( + answer, + make_top_candidate(cartesian_arg_min(right_border_begin, right))); + } + + return valid_value_position(static_cast(answer.position)) + ? static_cast(answer.position) + : npos; + } + + /** + * @brief Return the one-based rank-th close from the dense sample table. + */ + std::size_t cached_select0(std::size_t rank) const { + if (rank == 0 || rank > values_.size()) { + return npos; + } + const std::size_t sample_index = (rank - 1) / kCloseSelectSampleFrequency; + const std::size_t sample_rank = + sample_index * kCloseSelectSampleFrequency + 1; + if (sample_index >= close_select_samples_.size()) { + return npos; + } + return select0_from_position(close_select_samples_[sample_index], + rank - sample_rank + 1); + } + + /** + * @brief Select a zero bit by scanning forward from a sampled zero position. + */ + std::size_t select0_from_position(std::size_t position, + std::size_t rank) const { + std::size_t word = position >> 6; + std::size_t offset = position & 63; + while (word < bp_bits_.size()) { + const std::size_t word_begin = word * 64; + const std::size_t word_bits = + std::min(64, bp_bit_count_ - word_begin); + std::uint64_t zeros = (~bp_bits_[word]) & first_bits_mask(word_bits); + zeros &= ~first_bits_mask(offset); + const std::size_t count = std::popcount(zeros); + if (rank <= count) { + return word_begin + select_64(zeros, rank - 1); + } + rank -= count; + ++word; + offset = 0; + } + return npos; + } + + std::span values_; + Compare compare_; + std::vector bp_bits_; + std::size_t bp_bit_count_ = 0; + std::vector close_select_samples_; + std::vector top_sparse_candidates_; + std::size_t top_block_size_ = kMinTopSparseBlockSize; + std::size_t top_block_count_ = 0; + std::size_t top_sparse_levels_ = 0; + std::optional bp_index_; + BpDepthRmq bp_depth_rmq_; +}; + +} // namespace pixie::rmq::experimental diff --git a/src/benchmarks/bench_rmq.cpp b/src/benchmarks/bench_rmq.cpp index 3c346d2..5d22886 100644 --- a/src/benchmarks/bench_rmq.cpp +++ b/src/benchmarks/bench_rmq.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -1148,6 +1149,13 @@ void register_benchmarks() { std::int64_t, std::less, Index>>) ->Arg(static_cast(size)) ->Unit(benchmark::kMillisecond); + benchmark::RegisterBenchmark( + "rmq_build_cartesian_tree_segment_btree_xl", + &run_value_rmq_build< + pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq< + std::int64_t, std::less, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); #ifdef PIXIE_THIRD_PARTY_BENCHMARKS benchmark::RegisterBenchmark( "rmq_build_sdsl_sct", @@ -1259,6 +1267,13 @@ void register_benchmarks() { ->Args({static_cast(size), static_cast(width)}) ->Unit(benchmark::kNanosecond); + benchmark::RegisterBenchmark( + "rmq_cartesian_tree_segment_btree_xl", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); #ifdef PIXIE_THIRD_PARTY_BENCHMARKS benchmark::RegisterBenchmark( "rmq_sdsl_sct", diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index acf6366..210011e 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -85,6 +86,23 @@ bool packed_bit(std::span words, std::size_t position) { return ((words[position >> 6] >> (position & 63)) & 1u) != 0; } +std::size_t naive_select0(std::span words, + std::size_t bit_count, + std::size_t rank) { + if (rank == 0) { + return pixie::rmq::BpPlusMinusOneRmq<>::npos; + } + for (std::size_t position = 0; position < bit_count; ++position) { + if (!packed_bit(words, position)) { + --rank; + if (rank == 0) { + return position; + } + } + } + return pixie::rmq::BpPlusMinusOneRmq<>::npos; +} + template void expect_valid_bp_encoding(const Rmq& rmq, std::size_t value_count) { const std::size_t bit_count = rmq.bp_bit_count(); @@ -149,6 +167,16 @@ struct ExperimentalCartesianTreeRmMBTreeCase { static MaxRmq make_max(std::span values) { return MaxRmq(values); } }; +struct ExperimentalCartesianTreeSegmentBTreeXLCase { + using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq; + using MaxRmq = pixie::rmq::experimental:: + CartesianTreeSegmentBTreeXLRmq>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + struct NodeEulerBTreeCase { using Rmq = pixie::rmq::NodeEulerBTreeRmq; using MaxRmq = pixie::rmq::NodeEulerBTreeRmq>; @@ -228,6 +256,16 @@ struct ExperimentalRmMBTreePlusMinusOneCase { } }; +struct ExperimentalSegmentBTreeXLPlusMinusOneCase { + using Rmq = pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq<>; + static constexpr std::size_t kBlockSize = Rmq::kLeafSize; + + static Rmq make(std::span bits, + std::size_t depth_count) { + return Rmq(bits, depth_count); + } +}; + struct OneIntervalBTreeCase { using Rmq = pixie::rmq::OneIntervalBTreeRmq<>; static constexpr std::size_t kBlockSize = Rmq::kBlockSize; @@ -312,14 +350,16 @@ void check_one_interval_btree_ranges( template class ValueRmqContractTest : public ::testing::Test {}; -using ValueRmqCases = ::testing::Types; +using ValueRmqCases = + ::testing::Types; TYPED_TEST_SUITE(ValueRmqContractTest, ValueRmqCases); TYPED_TEST(ValueRmqContractTest, ExhaustiveSmallArray) { @@ -1099,11 +1139,13 @@ TEST(RmqExperimentalNodeEulerBTree, CopyAndMovePreserveSplitStorage) { template class DepthRmqContractTest : public ::testing::Test {}; -using DepthRmqCases = ::testing::Types; +using DepthRmqCases = + ::testing::Types; TYPED_TEST_SUITE(DepthRmqContractTest, DepthRmqCases); TYPED_TEST(DepthRmqContractTest, EmptyAndSingleDepthInputs) { @@ -1243,6 +1285,111 @@ TYPED_TEST(DepthRmqContractTest, DifferentialRandomWalks) { } } +TEST(RmqSegmentBTreeXLPlusMinusOne, DefaultBoundaryRanges) { + using Rmq = pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq<>; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + constexpr std::size_t kFanout = Rmq::kFanout; + constexpr std::size_t kHighBoundary = kLeaf * kFanout; + const std::vector sizes = { + kLeaf - 1, kLeaf, kLeaf + 1, + kHighBoundary - 1, kHighBoundary, kHighBoundary + 1, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector depths(size); + for (std::size_t i = 1; i < depths.size(); ++i) { + const bool up = (i % 7 == 0) || (i % 11 == 3) || (i % 19 == 5); + depths[i] = depths[i - 1] + (up ? 1 : -1); + } + + const std::vector bits = pack_depth_deltas(depths); + const Rmq rmq(bits, depths.size()); + const auto check_range = [&](std::size_t left, std::size_t right) { + ASSERT_LT(left, right); + ASSERT_LE(right, depths.size()); + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(depths), left, + right, std::less())) + << "range=[" << left << "," << right << ")"; + }; + + check_range(0, 1); + check_range(0, size); + check_range(size - 1, size); + for (const std::size_t boundary : {kLeaf, kHighBoundary}) { + if (boundary >= size) { + continue; + } + check_range(boundary - 1, boundary + 1); + check_range(boundary > 97 ? boundary - 97 : 0, + std::min(size, boundary + 101)); + check_range(boundary, size); + } + } +} + +TEST(RmqSegmentBTreeXLPlusMinusOne, Select0MatchesNaiveAcrossHighNodes) { + using Rmq = pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq<>; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + constexpr std::size_t kFanout = Rmq::kFanout; + std::vector depths(kLeaf * kFanout + 777); + for (std::size_t i = 1; i < depths.size(); ++i) { + const bool up = (i % 5 == 0) || (i % 17 == 3) || (i % 257 == 11); + depths[i] = depths[i - 1] + (up ? 1 : -1); + } + + const std::vector bits = pack_depth_deltas(depths); + const Rmq rmq(bits, depths.size()); + const std::size_t bit_count = depths.size() - 1; + std::size_t zero_count = 0; + for (std::size_t position = 0; position < bit_count; ++position) { + zero_count += packed_bit(bits, position) ? 0 : 1; + } + + const std::vector ranks = { + 1, 2, 63, 64, 65, zero_count / 4, zero_count / 2, zero_count - 1, + zero_count, + }; + for (const std::size_t rank : ranks) { + ASSERT_GT(rank, 0u); + ASSERT_LE(rank, zero_count); + EXPECT_EQ( + rmq.select0(rank), + naive_select0(std::span(bits), bit_count, rank)) + << "rank=" << rank; + } + EXPECT_EQ(rmq.select0(0), Rmq::npos); + EXPECT_EQ(rmq.select0(zero_count + 1), Rmq::npos); +} + +TEST(RmqSegmentBTreeXLPlusMinusOne, SmallFanoutExercisesMiddleAndHighLevels) { + using Rmq = + pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq; + constexpr std::size_t kBoundary = + Rmq::kLeafSize * Rmq::kFanout * Rmq::kFanout; + std::vector depths(kBoundary + 3 * Rmq::kLeafSize + 17); + for (std::size_t i = 1; i < depths.size(); ++i) { + const bool up = (i % 13 == 0) || (i % 17 == 1) || (i % 29 == 7); + depths[i] = depths[i - 1] + (up ? 1 : -1); + } + + const std::vector bits = pack_depth_deltas(depths); + const Rmq rmq(bits, depths.size()); + const std::vector> ranges = { + {0, depths.size()}, + {kBoundary - 2, kBoundary + 2}, + {kBoundary - Rmq::kLeafSize - 3, kBoundary + Rmq::kLeafSize + 5}, + {Rmq::kLeafSize, kBoundary + 17}, + {kBoundary + 1, depths.size()}, + {depths.size() - Rmq::kLeafSize, depths.size()}, + }; + check_depth_ranges( + rmq, std::span(depths), + std::span>(ranges)); +} + TEST(RmqOneIntervalBTree, FusedBoundaryTiesKeepFirstPosition) { constexpr std::size_t kBlock = pixie::rmq::OneIntervalBTreeRmq<>::kBlockSize; std::vector depths(4 * kBlock + 9); @@ -1337,6 +1484,188 @@ TEST(RmqOneIntervalBTree, HighNodeBoundaryRanges) { std::span>(ranges)); } +TEST(RmqCartesianTreeSegmentBTreeXL, BoundarySizesAndBpEncoding) { + using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq; + const std::vector sizes = { + 1u, 255u, 256u, 257u, 511u, 512u, 513u, 8191u, 8192u, 8193u, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 37 + i / 5) % 53); + if (i % 17 == 0 || i % 257 == 3) { + values[i] = -4; + } + } + + const Rmq rmq(values); + expect_valid_bp_encoding(rmq, values.size()); + + const std::vector> ranges = { + {0, 1}, + {0, size}, + {size - 1, size}, + {size / 3, std::min(size, size / 3 + 19)}, + {size / 2, std::min(size, size / 2 + 37)}, + }; + for (const auto [left, right] : ranges) { + ASSERT_LT(left, right); + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())) + << "range=[" << left << "," << right << ")"; + } + } +} + +TEST(RmqCartesianTreeSegmentBTreeXL, DuplicateHeavyRandomDifferentialTo8193) { + using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq; + std::mt19937_64 rng(20260610); + std::uniform_int_distribution value_dist(-3, 3); + const std::vector sizes = { + 1, 2, 17, 255, 256, 257, 1024, 4096, 8191, 8192, 8193, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = value_dist(rng); + if ((i % 11) < 6) { + values[i] = 0; + } + } + + const Rmq rmq(values); + std::uniform_int_distribution width_dist(1, size); + for (std::size_t query = 0; query < 2000; ++query) { + const std::size_t width = width_dist(rng); + std::uniform_int_distribution left_dist(0, size - width); + const std::size_t left = left_dist(rng); + const std::size_t right = left + width; + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } + } +} + +TEST(RmqCartesianTreeSegmentBTreeXL, TopSparseOverlayBoundaryRanges) { + using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq; + constexpr std::size_t kTopBlock = 4096; + std::vector values(3 * kTopBlock + 77, 1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = 1000 + static_cast((i * 17 + i / 7) % 89); + } + values[0] = -1000; + values[211] = -700; + values[kTopBlock + 123] = -900; + values[2 * kTopBlock + 300] = -800; + + const Rmq original(values); + Rmq copied(original); + Rmq assigned; + assigned = copied; + Rmq moved(std::move(copied)); + + const std::vector> ranges = { + {0, kTopBlock + 100}, + {100, 2 * kTopBlock + 100}, + {kTopBlock - 20, 2 * kTopBlock + 20}, + {333, kTopBlock + 10}, + {kTopBlock + 1, kTopBlock + 2000}, + {2 * kTopBlock + 10, values.size()}, + {0, values.size()}, + }; + + for (const auto [left, right] : ranges) { + ASSERT_LT(left, right); + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(original.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(assigned.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(moved.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqCartesianTreeSegmentBTreeXL, TopSparseOverlayCapsBlockCount) { + using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq; + constexpr std::size_t kMinBlock = Rmq::kMinTopSparseBlockSize; + constexpr std::size_t kMaxBlocks = Rmq::kMaxTopSparseBlocks; + constexpr std::size_t kLargestFixedBlockInput = kMinBlock * kMaxBlocks; + + EXPECT_EQ(Rmq::top_sparse_block_size_for(0), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(0), 0u); + EXPECT_EQ(Rmq::top_sparse_block_size_for(1), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(1), 1u); + EXPECT_EQ(Rmq::top_sparse_block_size_for(kLargestFixedBlockInput), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(kLargestFixedBlockInput), + kMaxBlocks); + EXPECT_GT(Rmq::top_sparse_block_size_for(kLargestFixedBlockInput + 1), + kMinBlock); + EXPECT_LE(Rmq::top_sparse_block_count_for(kLargestFixedBlockInput + 1), + kMaxBlocks); + + const std::size_t huge = std::numeric_limits::max() / 4; + EXPECT_LE(Rmq::top_sparse_block_count_for(huge), kMaxBlocks); + + std::vector values(3 * kMinBlock + 7, 0); + const Rmq rmq(values); + EXPECT_EQ(rmq.top_sparse_block_size(), kMinBlock); + EXPECT_EQ(rmq.top_sparse_block_count(), 4u); +} + +TEST(RmqCartesianTreeSegmentBTreeXL, TopSparseOverlayComparatorMaximum) { + using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq< + int, std::greater>; + constexpr std::size_t kTopBlock = 4096; + std::vector values(2 * kTopBlock + 33, -1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = -1000 + static_cast((i * 13 + i / 5) % 71); + } + values[0] = 1000; + values[kTopBlock + 17] = 900; + values[kTopBlock + 18] = 900; + + const Rmq rmq(values); + const std::vector> ranges = { + {1, values.size()}, + {kTopBlock, kTopBlock + 100}, + {kTopBlock + 18, values.size()}, + {0, values.size()}, + }; + + for (const auto [left, right] : ranges) { + ASSERT_LT(left, right); + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::greater())) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqCartesianTreeSegmentBTreeXL, ComparatorCopyAndMove) { + using MaxRmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq< + int, std::greater>; + const std::vector values = {1, 8, 8, 7, 8, 3, 8, 4, 4, 8}; + const MaxRmq original(values); + MaxRmq copied(original); + MaxRmq assigned; + assigned = copied; + MaxRmq moved(std::move(copied)); + + check_all_ranges(original, std::span(values), std::greater()); + check_all_ranges(assigned, std::span(values), std::greater()); + check_all_ranges(moved, std::span(values), std::greater()); + EXPECT_EQ(original.arg_min(0, values.size()), 1u); +} + TEST(RmqCartesianTree, DuplicateHeavyArrays) { const std::vector> cases = { {5, 5, 5, 5, 5, 5, 5}, From bd4761b930453e8c8fc3f63255a9d948864eed74 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Thu, 11 Jun 2026 01:59:28 +0300 Subject: [PATCH 14/27] Memory normalization and benchmarks --- .../optimization-experiment/EXAMPLES.md | 113 ++++- include/pixie/bitvector.h | 13 + include/pixie/cache_line.h | 5 + include/pixie/memory_usage.h | 51 +++ include/pixie/rmq.h | 105 +++-- include/pixie/rmq/bp_plus_minus_one_rmq.h | 13 + include/pixie/rmq/cartesian_tree_rmq.h | 14 + .../cartesian_tree_segment_btree_xl_rmq.h | 403 ++++++------------ include/pixie/rmq/rmq_base.h | 19 + include/pixie/rmq/sdsl_sct_rmq.h | 12 + include/pixie/rmq/segment_btree_xl.h | 25 ++ include/pixie/rmq/segment_tree.h | 11 + include/pixie/rmq/sparse_table.h | 16 + 13 files changed, 487 insertions(+), 313 deletions(-) create mode 100644 include/pixie/memory_usage.h diff --git a/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md b/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md index 1535efc..4c2e3bb 100644 --- a/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md +++ b/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md @@ -27,8 +27,9 @@ The `excess_min_128` experiment is a good template for future hot-path work: 7. **Persist benchmark evidence in the repository**: add a top-of-file benchmark note in `/** ... */` or `/* ... */` form to the experimental header, production header, benchmark file, or experiment log. Include metric - units, fixed decimal precision, JSON paths when available, and visible - separators between logical result blocks. + units, fixed decimal precision, and visible separators between logical + result blocks. Do not include machine-local JSON paths such as `/tmp/...` in + source comments. ## Benchmark Pattern @@ -53,6 +54,114 @@ For final comparison tables, include at least: - a random or mixed workload row - the rows that justified any narrowed production dispatch condition +## Example: RMQ Optimization Report + +The current RMQ work is the preferred template for larger optimization +experiments. It records the command shape, units, precision, and the main +tradeoff dimensions: query latency, build time, absolute memory, and normalized +memory. + +Keep this kind of report at the top of the relevant header or experiment log: + +```text +RMQ benchmark snapshot, 2026-06-11. + +Query command: + taskset -c 0 ./build/release/bench_rmq + --benchmark_filter='^(rmq_sparse_table|rmq_segment_tree|rmq_segment_btree_xl|rmq_cartesian_tree|rmq_cartesian_tree_segment_btree_xl|rmq_sdsl_sct)/' + --benchmark_min_time=0.25s + +Build/memory command: + taskset -c 0 ./build/release/bench_rmq + --benchmark_filter='^(rmq_build_sparse_table|rmq_build_segment_tree|rmq_build_segment_btree_xl|rmq_build_cartesian_tree|rmq_build_cartesian_tree_segment_btree_xl|rmq_build_sdsl_sct)/' + --benchmark_min_time=0.20s + +Query CPU time. "max width" is the benchmark's maximum sampled query width. +Sparse-table rows are intentionally not registered above 2^22. "cartesian xl" +is the experimental `rmq_cartesian_tree_segment_btree_xl` row. + +| N | max width | sparse table (ns) | segment tree (ns) | segment btree xl (ns) | cartesian tree (ns) | cartesian xl (ns) | sdsl sct (ns) | +|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^10 | 64 | 10.1 | 35.9 | 34.6 | 72.7 | 66.4 | 111.2 | +| 2^10 | 2^10 | 13.1 | 58.8 | 41.0 | 90.0 | 118.6 | 231.3 | +|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^14 | 64 | 14.7 | 43.6 | 33.5 | 86.3 | 67.3 | 125.4 | +| 2^14 | 4096 | 15.9 | 76.7 | 38.1 | 108.0 | 66.3 | 430.8 | +| 2^14 | 2^14 | 15.8 | 93.5 | 33.7 | 103.6 | 49.2 | 532.1 | +|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^18 | 64 | 63.0 | 103.7 | 50.9 | 87.0 | 74.7 | 141.3 | +| 2^18 | 4096 | 52.2 | 183.0 | 52.5 | 114.2 | 72.6 | 586.7 | +| 2^18 | 2^18 | 40.9 | 299.1 | 46.0 | 148.5 | 27.6 | 791.0 | +|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^22 | 64 | 97.2 | 169.1 | 137.4 | 164.7 | 256.9 | 263.2 | +| 2^22 | 4096 | 93.4 | 298.7 | 101.9 | 236.1 | 251.9 | 726.7 | +| 2^22 | 2^18 | 68.0 | 437.3 | 82.2 | 344.1 | 86.4 | 1053.0 | +| 2^22 | 2^22 | 60.7 | 485.3 | 45.5 | 225.6 | 22.3 | 1129.7 | +|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^24 | 64 | - | 208.0 | 166.6 | 326.0 | 480.4 | 549.6 | +| 2^24 | 4096 | - | 357.2 | 222.2 | 755.7 | 535.4 | 864.8 | +| 2^24 | 2^18 | - | 472.3 | 143.0 | 734.9 | 83.5 | 2824.2 | +| 2^24 | 2^22 | - | 510.7 | 47.1 | 403.5 | 25.3 | 1169.8 | +| 2^24 | 2^24 | - | 546.5 | 43.4 | 479.6 | 20.1 | 1239.6 | +|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^26 | 64 | - | 273.5 | 342.3 | 539.2 | 567.5 | 710.4 | +| 2^26 | 4096 | - | 938.4 | 308.9 | 566.5 | 688.5 | 1701.6 | +| 2^26 | 2^18 | - | 802.5 | 175.6 | 664.8 | 195.1 | 2609.4 | +| 2^26 | 2^22 | - | 1690.7 | 64.0 | 625.2 | 45.2 | 2387.0 | +| 2^26 | 2^26 | - | 1003.7 | 51.0 | 588.9 | 24.1 | 2152.0 | + +Build CPU time. Sparse-table build rows are intentionally not registered +above 2^22. + +| N | sparse table (ms) | segment tree (ms) | segment btree xl (ms) | cartesian tree (ms) | cartesian xl (ms) | sdsl sct (ms) | +|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^10 | 0.005 | 0.001 | 0.001 | 0.008 | 0.009 | 0.007 | +|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^14 | 1.096 | 0.015 | 0.018 | 0.045 | 0.058 | 0.162 | +|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^18 | 28.405 | 0.292 | 0.301 | 1.686 | 1.985 | 2.484 | +|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^22 | 860.619 | 92.812 | 6.077 | 29.128 | 31.552 | 40.262 | +|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^24 | - | 426.733 | 21.802 | 166.533 | 124.868 | 158.091 | +|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| +| 2^26 | - | 1432.849 | 86.641 | 852.740 | 518.439 | 647.360 | + +Owned auxiliary memory. Benchmarks use 64-bit signed integer values and +64-bit indexes. The external input values are not owned by RMQ indexes and +are excluded. + +| N | sparse table (MiB) | segment tree (MiB) | segment btree xl (MiB) | cartesian tree (MiB) | cartesian xl (MiB) | sdsl sct (MiB) | +|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| +| 2^10 | 0.071 | 0.016 | 0.009 | 0.004 | 0.016 | 0.001 | +|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| +| 2^14 | 1.626 | 0.250 | 0.011 | 0.025 | 0.022 | 0.005 | +|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| +| 2^18 | 34.001 | 4.000 | 0.065 | 0.474 | 0.186 | 0.079 | +|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| +| 2^22 | 672.001 | 64.000 | 0.801 | 9.539 | 2.632 | 1.262 | +|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| +| 2^24 | - | 256.000 | 3.155 | 42.148 | 7.525 | 5.051 | +|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| +| 2^26 | - | 1024.000 | 8.340 | 184.582 | 30.530 | 20.224 | + +Owned auxiliary memory normalized by indexed value count (bits/value). + +| N | sparse table | segment tree | segment btree xl | cartesian tree | cartesian xl | sdsl sct | +|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| +| 2^10 | 579.188 | 128.438 | 71.438 | 30.750 | 127.500 | 7.312 | +|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| +| 2^14 | 832.262 | 128.027 | 5.434 | 12.562 | 11.102 | 2.770 | +|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| +| 2^18 | 1088.020 | 128.002 | 2.089 | 15.175 | 5.957 | 2.534 | +|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| +| 2^22 | 1344.002 | 128.000 | 1.603 | 19.079 | 5.264 | 2.524 | +|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| +| 2^24 | - | 128.000 | 1.577 | 21.074 | 3.763 | 2.526 | +|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| +| 2^26 | - | 128.000 | 1.042 | 23.073 | 3.816 | 2.528 | +``` + ## Documentation Pattern For experimental algorithms, use Doxygen block comments: diff --git a/include/pixie/bitvector.h b/include/pixie/bitvector.h index 8a27553..28840a4 100644 --- a/include/pixie/bitvector.h +++ b/include/pixie/bitvector.h @@ -537,6 +537,19 @@ class BitVector { */ size_t size() const { return num_bits_; } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this rank/select object and its owned aligned metadata + * buffers. The external source bit words are not owned and are excluded. + */ + size_t memory_usage_bytes() const { + return sizeof(*this) + super_block_rank_.allocated_bytes() + + basic_block_rank_.allocated_bytes() + + select1_samples_.allocated_bytes() + + select0_samples_.allocated_bytes(); + } + /** * @brief Returns the bit at the given position. * @param pos Bit diff --git a/include/pixie/cache_line.h b/include/pixie/cache_line.h index 82f5f23..dcae1f7 100644 --- a/include/pixie/cache_line.h +++ b/include/pixie/cache_line.h @@ -65,6 +65,11 @@ class AlignedStorage { return data_.size() * kAlignedStorageLineBytes; } + /** @brief Bytes reserved by the underlying aligned cache-line buffer. */ + std::size_t allocated_bytes() const { + return data_.capacity() * kAlignedStorageLineBytes; + } + /** @brief Mutable view as cache lines. */ std::span AsLines() { return data_; } /** @brief Const view as cache lines. */ diff --git a/include/pixie/memory_usage.h b/include/pixie/memory_usage.h new file mode 100644 index 0000000..9be316d --- /dev/null +++ b/include/pixie/memory_usage.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include + +namespace pixie { + +/** + * @brief Return bytes reserved by a vector's heap buffer. + * + * @details This intentionally uses capacity rather than size because the goal + * is to report practical owned memory after construction. The vector control + * block itself is counted by the owning object's `sizeof(*this)`. + */ +template +std::size_t vector_capacity_bytes(const std::vector& values) { + return values.capacity() * sizeof(T); +} + +/** + * @brief Concept for types that expose total owned memory usage in bytes. + */ +template +concept HasMemoryUsageBytes = requires(const T& value) { + { value.memory_usage_bytes() } -> std::convertible_to; +}; + +/** + * @brief Return heap bytes owned below an inline nested object. + * + * @details `outer.sizeof(*this)` already counts the inline nested object + * storage. This helper subtracts that inline storage from the nested object's + * total memory usage and leaves only buffers owned below it. + */ +template +std::size_t nested_owned_memory_bytes(const T& value) { + const std::size_t total = value.memory_usage_bytes(); + return total > sizeof(T) ? total - sizeof(T) : 0; +} + +/** + * @brief Return heap bytes owned below an engaged optional nested object. + */ +template +std::size_t optional_nested_owned_memory_bytes(const std::optional& value) { + return value.has_value() ? nested_owned_memory_bytes(*value) : 0; +} + +} // namespace pixie diff --git a/include/pixie/rmq.h b/include/pixie/rmq.h index ade35ba..2c7bdc0 100644 --- a/include/pixie/rmq.h +++ b/include/pixie/rmq.h @@ -1,65 +1,104 @@ #pragma once /* - * RMQ benchmark snapshot, 2026-06-10. + * RMQ benchmark snapshot, 2026-06-11. * - * Command shape: + * Query command: * taskset -c 0 ./build/release/bench_rmq * --benchmark_filter='^(rmq_sparse_table|rmq_segment_tree|rmq_segment_btree_xl|rmq_cartesian_tree|rmq_cartesian_tree_segment_btree_xl|rmq_sdsl_sct)/' * --benchmark_min_time=0.25s * - * JSON: - * /tmp/rmq_value_query_table_20260610.json - * /tmp/rmq_value_build_table_20260610.json + * Build/memory command: + * taskset -c 0 ./build/release/bench_rmq + * --benchmark_filter='^(rmq_build_sparse_table|rmq_build_segment_tree|rmq_build_segment_btree_xl|rmq_build_cartesian_tree|rmq_build_cartesian_tree_segment_btree_xl|rmq_build_sdsl_sct)/' + * --benchmark_min_time=0.20s * * Query CPU time. "max width" is the benchmark's maximum sampled query width. * Sparse-table rows are intentionally not registered above 2^22. "cartesian xl" - * is the experimental `rmq_cartesian_tree_segment_btree_xl` row. + * is the experimental `rmq_cartesian_tree_segment_btree_xl` row. The + * cartesian xl column was refreshed after removing leaf summaries, unused top + * sparse-table close positions, and the close-select sample cache. * * | N | max width | sparse table (ns) | segment tree (ns) | segment btree xl (ns) | cartesian tree (ns) | cartesian xl (ns) | sdsl sct (ns) | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^10 | 64 | 10.1 | 35.9 | 34.6 | 72.7 | 66.4 | 111.2 | - * | 2^10 | 2^10 | 13.1 | 58.8 | 41.0 | 90.0 | 118.6 | 231.3 | + * | 2^10 | 64 | 10.1 | 37.6 | 35.0 | 71.4 | 90.5 | 120.6 | + * | 2^10 | 2^10 | 13.3 | 62.0 | 42.0 | 93.2 | 144.6 | 255.8 | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^14 | 64 | 14.7 | 43.6 | 33.5 | 86.3 | 67.3 | 125.4 | - * | 2^14 | 4096 | 15.9 | 76.7 | 38.1 | 108.0 | 66.3 | 430.8 | - * | 2^14 | 2^14 | 15.8 | 93.5 | 33.7 | 103.6 | 49.2 | 532.1 | + * | 2^14 | 64 | 17.7 | 53.6 | 35.4 | 93.6 | 103.8 | 130.5 | + * | 2^14 | 4096 | 16.1 | 83.8 | 39.6 | 111.7 | 95.1 | 433.0 | + * | 2^14 | 2^14 | 18.9 | 121.1 | 34.6 | 107.8 | 60.7 | 534.9 | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^18 | 64 | 63.0 | 103.7 | 50.9 | 87.0 | 74.7 | 141.3 | - * | 2^18 | 4096 | 52.2 | 183.0 | 52.5 | 114.2 | 72.6 | 586.7 | - * | 2^18 | 2^18 | 40.9 | 299.1 | 46.0 | 148.5 | 27.6 | 791.0 | + * | 2^18 | 64 | 71.8 | 119.1 | 56.4 | 91.2 | 99.0 | 144.4 | + * | 2^18 | 4096 | 53.2 | 239.0 | 63.8 | 120.5 | 98.2 | 535.8 | + * | 2^18 | 2^18 | 47.6 | 336.2 | 45.1 | 126.3 | 26.8 | 737.6 | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^22 | 64 | 97.2 | 169.1 | 137.4 | 164.7 | 256.9 | 263.2 | - * | 2^22 | 4096 | 93.4 | 298.7 | 101.9 | 236.1 | 251.9 | 726.7 | - * | 2^22 | 2^18 | 68.0 | 437.3 | 82.2 | 344.1 | 86.4 | 1053.0 | - * | 2^22 | 2^22 | 60.7 | 485.3 | 45.5 | 225.6 | 22.3 | 1129.7 | + * | 2^22 | 64 | 103.0 | 192.4 | 140.3 | 128.7 | 143.9 | 231.9 | + * | 2^22 | 4096 | 83.6 | 341.8 | 166.6 | 346.8 | 174.6 | 741.0 | + * | 2^22 | 2^18 | 73.9 | 477.8 | 67.3 | 372.1 | 41.6 | 951.0 | + * | 2^22 | 2^22 | 64.3 | 506.0 | 43.1 | 254.3 | 21.0 | 939.4 | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^24 | 64 | - | 208.0 | 166.6 | 326.0 | 480.4 | 549.6 | - * | 2^24 | 4096 | - | 357.2 | 222.2 | 755.7 | 535.4 | 864.8 | - * | 2^24 | 2^18 | - | 472.3 | 143.0 | 734.9 | 83.5 | 2824.2 | - * | 2^24 | 2^22 | - | 510.7 | 47.1 | 403.5 | 25.3 | 1169.8 | - * | 2^24 | 2^24 | - | 546.5 | 43.4 | 479.6 | 20.1 | 1239.6 | + * | 2^24 | 64 | - | 257.3 | 179.7 | 237.2 | 241.9 | 455.3 | + * | 2^24 | 4096 | - | 462.3 | 321.1 | 606.0 | 517.7 | 1254.6 | + * | 2^24 | 2^18 | - | 538.2 | 138.9 | 530.7 | 91.4 | 1855.2 | + * | 2^24 | 2^22 | - | 638.3 | 58.2 | 462.0 | 24.4 | 1512.0 | + * | 2^24 | 2^24 | - | 663.9 | 48.7 | 416.0 | 21.0 | 1562.5 | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^26 | 64 | - | 273.5 | 342.3 | 539.2 | 567.5 | 710.4 | - * | 2^26 | 4096 | - | 938.4 | 308.9 | 566.5 | 688.5 | 1701.6 | - * | 2^26 | 2^18 | - | 802.5 | 175.6 | 664.8 | 195.1 | 2609.4 | - * | 2^26 | 2^22 | - | 1690.7 | 64.0 | 625.2 | 45.2 | 2387.0 | - * | 2^26 | 2^26 | - | 1003.7 | 51.0 | 588.9 | 24.1 | 2152.0 | + * | 2^26 | 64 | - | 299.1 | 274.2 | 342.3 | 362.2 | 643.3 | + * | 2^26 | 4096 | - | 519.7 | 319.1 | 608.6 | 614.7 | 1591.8 | + * | 2^26 | 2^18 | - | 742.8 | 218.2 | 569.2 | 154.6 | 2465.1 | + * | 2^26 | 2^22 | - | 1066.8 | 64.7 | 596.5 | 41.0 | 2187.8 | + * | 2^26 | 2^26 | - | 1010.0 | 62.1 | 665.6 | 21.5 | 2080.6 | * * Build CPU time. Sparse-table build rows are intentionally not registered * above 2^22. * * | N | sparse table (ms) | segment tree (ms) | segment btree xl (ms) | cartesian tree (ms) | cartesian xl (ms) | sdsl sct (ms) | * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^10 | 0.006 | 0.001 | 0.001 | 0.008 | 0.010 | 0.007 | + * | 2^10 | 0.007 | 0.001 | 0.001 | 0.006 | 0.007 | 0.007 | + * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| + * | 2^14 | 1.118 | 0.015 | 0.017 | 0.040 | 0.051 | 0.164 | * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^14 | 1.081 | 0.015 | 0.017 | 0.048 | 0.063 | 0.230 | + * | 2^18 | 33.542 | 0.378 | 0.303 | 1.741 | 1.924 | 2.585 | * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^18 | 32.766 | 0.369 | 0.288 | 1.741 | 2.108 | 2.596 | + * | 2^22 | 934.032 | 65.087 | 5.548 | 29.030 | 30.147 | 40.664 | * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^22 | 903.196 | 78.183 | 5.401 | 29.116 | 33.755 | 41.024 | + * | 2^24 | - | 289.874 | 20.924 | 172.132 | 120.261 | 162.269 | * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^24 | - | 267.256 | 21.746 | 172.782 | 136.031 | 165.874 | + * | 2^26 | - | 1181.042 | 86.430 | 727.667 | 458.843 | 670.310 | + * + * Owned auxiliary memory. Benchmarks use 64-bit signed integer values and + * 64-bit indexes. The external input values are not owned by RMQ indexes and + * are excluded. + * + * | N | sparse table (MiB) | segment tree (MiB) | segment btree xl (MiB) | cartesian tree (MiB) | cartesian xl (MiB) | sdsl sct (MiB) | + * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| + * | 2^10 | 0.071 | 0.016 | 0.009 | 0.004 | 0.014 | 0.001 | + * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| + * | 2^14 | 1.626 | 0.250 | 0.011 | 0.025 | 0.017 | 0.005 | + * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| + * | 2^18 | 34.001 | 4.000 | 0.065 | 0.474 | 0.132 | 0.079 | + * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| + * | 2^22 | 672.001 | 64.000 | 0.801 | 9.539 | 1.791 | 1.262 | + * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| + * | 2^24 | - | 256.000 | 3.155 | 42.148 | 4.611 | 5.051 | + * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| + * | 2^26 | - | 1024.000 | 8.340 | 184.582 | 18.631 | 20.224 | + * + * Owned auxiliary memory normalized by indexed value count (bits/value). + * + * | N | sparse table | segment tree | segment btree xl | cartesian tree | cartesian xl | sdsl sct | + * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| + * | 2^10 | 579.188 | 128.438 | 71.438 | 30.750 | 112.312 | 7.312 | + * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| + * | 2^14 | 832.262 | 128.027 | 5.434 | 12.562 | 8.938 | 2.770 | + * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| + * | 2^18 | 1088.020 | 128.002 | 2.089 | 15.175 | 4.228 | 2.534 | + * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| + * | 2^22 | 1344.002 | 128.000 | 1.603 | 19.079 | 3.582 | 2.524 | + * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| + * | 2^24 | - | 128.000 | 1.577 | 21.074 | 2.305 | 2.526 | + * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| + * | 2^26 | - | 128.000 | 1.042 | 23.073 | 2.329 | 2.528 | */ #include diff --git a/include/pixie/rmq/bp_plus_minus_one_rmq.h b/include/pixie/rmq/bp_plus_minus_one_rmq.h index e14e28f..2aa0df9 100644 --- a/include/pixie/rmq/bp_plus_minus_one_rmq.h +++ b/include/pixie/rmq/bp_plus_minus_one_rmq.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -134,6 +135,18 @@ class BpPlusMinusOneRmq { */ bool empty() const { return depth_count_ == 0; } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this ±1 RMQ object, block summaries, and the macro sparse + * table over those summaries. The external packed delta bits are not owned + * and are excluded. + */ + std::size_t memory_usage_bytes() const { + return sizeof(*this) + pixie::vector_capacity_bytes(block_summaries_) + + pixie::nested_owned_memory_bytes(macro_rmq_); + } + /** * @brief Return the first minimum depth position in [@p left, @p right). * diff --git a/include/pixie/rmq/cartesian_tree_rmq.h b/include/pixie/rmq/cartesian_tree_rmq.h index d63778d..f1261b2 100644 --- a/include/pixie/rmq/cartesian_tree_rmq.h +++ b/include/pixie/rmq/cartesian_tree_rmq.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -206,6 +207,19 @@ class CartesianTreeRmq */ std::span bp_words() const { return bp_bits_; } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this Cartesian-tree object, its packed BP words, and the + * nested BP rank/select and ±1 RMQ indexes. The external input values are not + * owned and are excluded. + */ + std::size_t memory_usage_bytes_impl() const { + return sizeof(*this) + pixie::vector_capacity_bytes(bp_bits_) + + pixie::optional_nested_owned_memory_bytes(bp_index_) + + pixie::nested_owned_memory_bytes(bp_depth_rmq_); + } + private: /** * @brief Rebuild the direct BP Cartesian-tree RMQ representation. diff --git a/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h b/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h index 09d8917..1bf3d87 100644 --- a/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h +++ b/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -34,8 +35,10 @@ namespace pixie::rmq::experimental { * internal node stores a local Cartesian/BP selector over the minima of its * immediate children. The top one or two levels additionally keep sparse * tables over child-minimum slots. Unlike value `SegmentBTreeXL`, leaves do not - * store another local Cartesian selector. A leaf query scans the original BP - * delta bits directly with the 128-bit excess primitives from `bits.h`. + * store another local Cartesian selector or cached minima. Leaf minima are + * recomputed from the original BP delta bits with the 128-bit excess + * primitives from `bits.h`, using rank support to recover the absolute base + * depth at the leaf start. * * @tparam Index Unsigned integer type used for stored positions. * @tparam LeafSize Number of depth positions per low-level leaf. @@ -84,12 +87,37 @@ class SegmentBTreeXLPlusMinusOneRmq { build(bits, depth_count); } + /** + * @brief Build a ±1 RMQ index over external packed bits and rank support. + * + * @details The supplied rank index is non-owning and must outlive this RMQ + * object. This is used by Cartesian XL to reuse its BP rank/select index. + */ + SegmentBTreeXLPlusMinusOneRmq(std::span bits, + std::size_t depth_count, + const BitVector& rank_index) { + build(bits, depth_count, rank_index); + } + /** * @brief Rebuild this index over external packed delta bits. */ void build(std::span bits, std::size_t depth_count) { input_bits_ = bits; depth_count_ = depth_count; + external_rank_index_ = nullptr; + build(); + } + + /** + * @brief Rebuild this index using non-owning rank support for the same bits. + */ + void build(std::span bits, + std::size_t depth_count, + const BitVector& rank_index) { + input_bits_ = bits; + depth_count_ = depth_count; + external_rank_index_ = &rank_index; build(); } @@ -103,6 +131,30 @@ class SegmentBTreeXLPlusMinusOneRmq { */ bool empty() const { return depth_count_ == 0; } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this ±1 RMQ object and all selector/metadata buffers. The + * external packed delta bits are not owned and are excluded. + */ + std::size_t memory_usage_bytes() const { + std::size_t bytes = sizeof(*this); + if (external_rank_index_ == nullptr) { + bytes += pixie::optional_nested_owned_memory_bytes(owned_rank_index_); + } + bytes += pixie::vector_capacity_bytes(internal_selectors_); + bytes += pixie::vector_capacity_bytes(internal_min_positions_); + bytes += pixie::vector_capacity_bytes(internal_min_depths_); + bytes += pixie::vector_capacity_bytes(high_child_metadata_); + bytes += pixie::vector_capacity_bytes(high_sparse_min_slots_); + bytes += pixie::vector_capacity_bytes(internal_level_offsets_); + bytes += pixie::vector_capacity_bytes(high_level_offsets_); + bytes += pixie::vector_capacity_bytes(level_sizes_); + bytes += pixie::vector_capacity_bytes(level_position_spans_); + bytes += pixie::vector_capacity_bytes(level_fanouts_); + return bytes; + } + /** * @brief Return the first minimum depth position in [@p left, @p right). * @@ -136,20 +188,17 @@ class SegmentBTreeXLPlusMinusOneRmq { * @brief Return the one-based rank-th zero delta bit position. * * @details This is the select operation needed by the Cartesian wrapper to - * map value endpoints to close-parenthesis positions. High nodes keep - * per-child zero-count prefixes, so top-level descent avoids rescanning the - * 256-way high-node fanout. The returned position is a delta-bit position in - * `[0, size() - 1)`, or `npos` when @p rank is out of range. + * map value endpoints to close-parenthesis positions. The returned position + * is a delta-bit position in `[0, size() - 1)`, or `npos` when @p rank is out + * of range. */ std::size_t select0(std::size_t rank) const { - if (rank == 0 || depth_count_ == 0 || level_sizes_.empty()) { + if (rank == 0 || depth_count_ == 0 || rank_index_or_null() == nullptr) { return npos; } - const std::size_t root_level = level_count() - 1; - if (rank > subtree_zero_count(root_level, 0)) { - return npos; - } - return select0_in_node(root_level, 0, rank); + const std::size_t delta_count = depth_count_ - 1; + const std::size_t position = rank_index().select0(rank); + return position < delta_count ? position : npos; } private: @@ -170,13 +219,6 @@ class SegmentBTreeXLPlusMinusOneRmq { std::int64_t depth = std::numeric_limits::max(); }; - struct LeafSummary { - std::int64_t base_depth = 0; - std::int64_t min_depth = 0; - Index min_offset = 0; - Index zero_count = 0; - }; - struct HighChildMetadata { std::size_t position_begin = 0; std::size_t position_end = 0; @@ -399,16 +441,15 @@ class SegmentBTreeXLPlusMinusOneRmq { } /** - * @brief Build all leaf summaries, internal selectors, and top sparse tables. + * @brief Build rank support, internal selectors, and top sparse tables. */ void build() { - leaf_summaries_.clear(); + owned_rank_index_.reset(); internal_selectors_.clear(); internal_min_positions_.clear(); internal_min_depths_.clear(); high_child_metadata_.clear(); high_sparse_min_slots_.clear(); - high_zero_prefixes_.clear(); internal_level_offsets_.clear(); high_level_offsets_.clear(); level_sizes_.clear(); @@ -433,8 +474,15 @@ class SegmentBTreeXLPlusMinusOneRmq { "SegmentBTreeXLPlusMinusOneRmq bit span is too small"); } + const std::size_t delta_count = depth_count_ - 1; + if (external_rank_index_ == nullptr) { + owned_rank_index_.emplace(input_bits_, delta_count); + } else if (external_rank_index_->size() < delta_count) { + throw std::invalid_argument( + "SegmentBTreeXLPlusMinusOneRmq external rank index is too small"); + } + initialize_layout((depth_count_ + LeafSize - 1) / LeafSize); - build_leaves(); for (std::size_t level = 1; level < level_count(); ++level) { for (std::size_t node = 0; node < level_sizes_[level]; ++node) { build_internal_node(level, node); @@ -466,9 +514,6 @@ class SegmentBTreeXLPlusMinusOneRmq { level_sizes_.push_back(current_count); level_position_spans_.push_back(current_span); } - - leaf_summaries_.resize(level_sizes_[0]); - internal_level_offsets_.assign(level_count(), 0); high_level_offsets_.assign(level_count(), 0); if (level_count() <= 1) { @@ -487,7 +532,6 @@ class SegmentBTreeXLPlusMinusOneRmq { internal_min_positions_.resize(internal_count, invalid_index); internal_min_depths_.resize(internal_count, std::numeric_limits::max()); - internal_zero_counts_.resize(internal_count, 0); std::size_t high_node_count = 0; for (std::size_t level = high_level_begin_; level < level_count(); @@ -497,30 +541,6 @@ class SegmentBTreeXLPlusMinusOneRmq { } high_child_metadata_.resize(high_node_count * Fanout); high_sparse_min_slots_.resize(high_node_count * kHighSparseSlotsPerNode); - high_zero_prefixes_.resize(high_node_count * (Fanout + 1)); - } - - /** - * @brief Build all leaf summaries while carrying the absolute base depth. - */ - void build_leaves() { - std::int64_t base_depth = 0; - for (std::size_t leaf = 0; leaf < leaf_summaries_.size(); ++leaf) { - const std::size_t count = entry_count(0, leaf); - DepthCandidate minimum = - scan_leaf_range_with_base(leaf, 0, count - 1, base_depth); - const std::size_t zero_count = - leaf_zero_count(leaf, next_leaf_delta_count(leaf)); - leaf_summaries_[leaf] = { - base_depth, - minimum.depth, - static_cast(minimum.position - node_position_begin(0, leaf)), - static_cast(zero_count), - }; - if (leaf + 1 < leaf_summaries_.size()) { - base_depth += leaf_excess(leaf, next_leaf_delta_count(leaf)); - } - } } /** @@ -532,16 +552,16 @@ class SegmentBTreeXLPlusMinusOneRmq { const bool high_level = is_high_level(level); const std::size_t high_flat = high_level ? high_flat_index(level, node) : 0; + std::array child_minima{}; + for (std::size_t slot = 0; slot < count; ++slot) { + child_minima[slot] = + subtree_min_candidate(level - 1, first_child + slot); + } + if (high_level) { - std::size_t zero_prefix = 0; - std::size_t* prefixes = mutable_high_zero_prefixes_begin(high_flat); - prefixes[0] = 0; for (std::size_t slot = 0; slot < count; ++slot) { const std::size_t child = first_child + slot; - const DepthCandidate child_min = - subtree_min_candidate(level - 1, child); - zero_prefix += subtree_zero_count(level - 1, child); - prefixes[slot + 1] = zero_prefix; + const DepthCandidate child_min = child_minima[slot]; HighChildMetadata& metadata = mutable_high_child_metadata_at(high_flat, slot); metadata.position_begin = node_position_begin(level - 1, child); @@ -554,24 +574,15 @@ class SegmentBTreeXLPlusMinusOneRmq { Bp512Selector& selector = mutable_selector_at(level, node); selector.build(count, [&](std::size_t left, std::size_t right) { - return strictly_better_candidate(child_min_candidate(level, node, left), - child_min_candidate(level, node, right)); + return strictly_better_candidate(child_minima[left], + child_minima[right]); }); const std::size_t slot = selector.arg_min(0, count, count); - const DepthCandidate minimum = child_min_candidate(level, node, slot); + const DepthCandidate minimum = child_minima[slot]; const std::size_t flat = internal_flat_index(level, node); internal_min_positions_[flat] = static_cast(minimum.position); internal_min_depths_[flat] = minimum.depth; - if (high_level) { - internal_zero_counts_[flat] = high_zero_prefixes_begin(high_flat)[count]; - } else { - std::size_t zero_count = 0; - for (std::size_t slot = 0; slot < count; ++slot) { - zero_count += subtree_zero_count(level - 1, first_child + slot); - } - internal_zero_counts_[flat] = zero_count; - } } /** @@ -611,44 +622,28 @@ class SegmentBTreeXLPlusMinusOneRmq { } /** - * @brief Return total excess over the first @p delta_count bits of a leaf. + * @brief Return the active BP rank/select support, if one exists. */ - std::int64_t leaf_excess(std::size_t leaf, std::size_t delta_count) const { - const auto bits = leaf_bits(leaf); - std::int64_t result = 0; - std::size_t remaining = std::min(delta_count, LeafSize); - for (std::size_t chunk = 0; chunk < kLeafChunks && remaining != 0; - ++chunk) { - const std::size_t chunk_count = std::min(128, remaining); - result += prefix_excess_128(bits.data() + 2 * chunk, chunk_count); - remaining -= chunk_count; - } - return result; + const BitVector* rank_index_or_null() const { + return external_rank_index_ != nullptr + ? external_rank_index_ + : (owned_rank_index_ ? &*owned_rank_index_ : nullptr); } /** - * @brief Return the number of zero delta bits in one leaf prefix. + * @brief Return the active BP rank/select support. */ - std::size_t leaf_zero_count(std::size_t leaf, std::size_t delta_count) const { - const auto bits = leaf_bits(leaf); - std::size_t result = 0; - std::size_t remaining = std::min(delta_count, LeafSize); - for (std::size_t word = 0; word < kLeafWords && remaining != 0; ++word) { - const std::size_t word_bits = std::min(64, remaining); - result += - word_bits - std::popcount(bits[word] & first_bits_mask(word_bits)); - remaining -= word_bits; - } - return result; - } + const BitVector& rank_index() const { return *rank_index_or_null(); } /** - * @brief Count delta bits from a leaf start to the next leaf base depth. + * @brief Return the absolute open-minus-close depth at a BP depth position. */ - std::size_t next_leaf_delta_count(std::size_t leaf) const { - const std::size_t begin = node_position_begin(0, leaf); - const std::size_t next_begin = begin + LeafSize; - return std::min(next_begin, depth_count_ - 1) - begin; + std::int64_t depth_at_position(std::size_t position) const { + const std::size_t delta_count = depth_count_ == 0 ? 0 : depth_count_ - 1; + position = std::min(position, delta_count); + const std::uint64_t ones = rank_index().rank(position); + return static_cast(ones) - + static_cast(position - ones); } /** @@ -710,15 +705,10 @@ class SegmentBTreeXLPlusMinusOneRmq { } const std::size_t begin = node_position_begin(0, leaf); - const std::size_t end = node_position_end(0, leaf); - if (left <= begin && end <= right) { - return subtree_min_candidate(0, leaf); - } - const std::size_t slot_left = left - begin; const std::size_t slot_right = right - begin; return scan_leaf_range_with_base(leaf, slot_left, slot_right - 1, - leaf_summaries_[leaf].base_depth); + depth_at_position(begin)); } /** @@ -931,25 +921,14 @@ class SegmentBTreeXLPlusMinusOneRmq { DepthCandidate subtree_min_candidate(std::size_t level, std::size_t node) const { if (level == 0) { - const LeafSummary& summary = leaf_summaries_[node]; - return {node_position_begin(0, node) + summary.min_offset, - summary.min_depth}; + return leaf_range_min(node, node_position_begin(0, node), + node_position_end(0, node)); } const std::size_t flat = internal_flat_index(level, node); return {static_cast(internal_min_positions_[flat]), internal_min_depths_[flat]}; } - /** - * @brief Return a node's cached subtree zero count. - */ - std::size_t subtree_zero_count(std::size_t level, std::size_t node) const { - if (level == 0) { - return leaf_summaries_[node].zero_count; - } - return internal_zero_counts_[internal_flat_index(level, node)]; - } - /** * @brief Return a child slot's subtree-minimum candidate. */ @@ -1069,20 +1048,6 @@ class SegmentBTreeXLPlusMinusOneRmq { return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; } - /** - * @brief Return mutable zero-prefix storage for one high node. - */ - std::size_t* mutable_high_zero_prefixes_begin(std::size_t high_flat) { - return high_zero_prefixes_.data() + high_flat * (Fanout + 1); - } - - /** - * @brief Return zero-prefix storage for one high node. - */ - const std::size_t* high_zero_prefixes_begin(std::size_t high_flat) const { - return high_zero_prefixes_.data() + high_flat * (Fanout + 1); - } - /** * @brief Choose the better high-node child slot. */ @@ -1151,80 +1116,15 @@ class SegmentBTreeXLPlusMinusOneRmq { table[slot_right - span]); } - /** - * @brief Descend the tree to select the rank-th zero bit. - */ - std::size_t select0_in_node(std::size_t level, - std::size_t node, - std::size_t rank) const { - if (level == 0) { - return select0_in_leaf(node, rank); - } - - const std::size_t count = entry_count(level, node); - std::size_t slot = 0; - if (is_high_level(level)) { - const std::size_t high_flat = high_flat_index(level, node); - const std::size_t* prefixes = high_zero_prefixes_begin(high_flat); - const std::size_t* first = prefixes + 1; - const std::size_t* last = prefixes + count + 1; - const std::size_t* selected = std::lower_bound(first, last, rank); - if (selected == last) { - return npos; - } - slot = static_cast(selected - first); - rank -= prefixes[slot]; - } else { - const std::size_t first_child = node * fanout_at_level(level); - for (; slot < count; ++slot) { - const std::size_t child_zeros = - subtree_zero_count(level - 1, first_child + slot); - if (rank <= child_zeros) { - break; - } - rank -= child_zeros; - } - if (slot == count) { - return npos; - } - } - - return select0_in_node(level - 1, node * fanout_at_level(level) + slot, - rank); - } - - /** - * @brief Select the rank-th zero bit inside one leaf. - */ - std::size_t select0_in_leaf(std::size_t leaf, std::size_t rank) const { - const std::size_t bit_begin = node_position_begin(0, leaf); - const std::size_t delta_count = next_leaf_delta_count(leaf); - const auto bits = leaf_bits(leaf); - std::size_t remaining = delta_count; - for (std::size_t word = 0; word < kLeafWords && remaining != 0; ++word) { - const std::size_t word_bits = std::min(64, remaining); - const std::uint64_t candidates = - (~bits[word]) & first_bits_mask(word_bits); - const std::size_t count = std::popcount(candidates); - if (rank <= count) { - return bit_begin + word * 64 + select_64(candidates, rank - 1); - } - rank -= count; - remaining -= word_bits; - } - return npos; - } - std::span input_bits_; std::size_t depth_count_ = 0; - std::vector leaf_summaries_; + std::optional owned_rank_index_; + const BitVector* external_rank_index_ = nullptr; std::vector internal_selectors_; std::vector internal_min_positions_; std::vector internal_min_depths_; - std::vector internal_zero_counts_; std::vector high_child_metadata_; std::vector high_sparse_min_slots_; - std::vector high_zero_prefixes_; std::vector internal_level_offsets_; std::vector high_level_offsets_; std::vector level_sizes_; @@ -1306,7 +1206,6 @@ class CartesianTreeSegmentBTreeXLRmq compare_(other.compare_), bp_bits_(other.bp_bits_), bp_bit_count_(other.bp_bit_count_), - close_select_samples_(other.close_select_samples_), top_sparse_candidates_(other.top_sparse_candidates_), top_block_size_(other.top_block_size_), top_block_count_(other.top_block_count_), @@ -1326,7 +1225,6 @@ class CartesianTreeSegmentBTreeXLRmq compare_ = other.compare_; bp_bits_ = other.bp_bits_; bp_bit_count_ = other.bp_bit_count_; - close_select_samples_ = other.close_select_samples_; top_sparse_candidates_ = other.top_sparse_candidates_; top_block_size_ = other.top_block_size_; top_block_count_ = other.top_block_count_; @@ -1344,7 +1242,6 @@ class CartesianTreeSegmentBTreeXLRmq compare_(std::move(other.compare_)), bp_bits_(std::move(other.bp_bits_)), bp_bit_count_(other.bp_bit_count_), - close_select_samples_(std::move(other.close_select_samples_)), top_sparse_candidates_(std::move(other.top_sparse_candidates_)), top_block_size_(other.top_block_size_), top_block_count_(other.top_block_count_), @@ -1369,7 +1266,6 @@ class CartesianTreeSegmentBTreeXLRmq compare_ = std::move(other.compare_); bp_bits_ = std::move(other.bp_bits_); bp_bit_count_ = other.bp_bit_count_; - close_select_samples_ = std::move(other.close_select_samples_); top_sparse_candidates_ = std::move(other.top_sparse_candidates_); top_block_size_ = other.top_block_size_; top_block_count_ = other.top_block_count_; @@ -1451,24 +1347,36 @@ class CartesianTreeSegmentBTreeXLRmq */ std::size_t top_sparse_block_count() const { return top_block_count_; } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this value-RMQ object, packed Cartesian BP words, the top + * sparse overlay, and nested BP rank/select and ±1 RMQ indexes. The external + * input values are not owned and are excluded. + */ + std::size_t memory_usage_bytes_impl() const { + return sizeof(*this) + pixie::vector_capacity_bytes(bp_bits_) + + pixie::vector_capacity_bytes(top_sparse_candidates_) + + pixie::optional_nested_owned_memory_bytes(bp_index_) + + pixie::nested_owned_memory_bytes(bp_depth_rmq_); + } + private: using BpDepthRmq = SegmentBTreeXLPlusMinusOneRmq; - static constexpr std::size_t kCloseSelectSampleFrequency = 256; struct TopCandidate { Index position = invalid_index; - Index close_position = invalid_index; }; - static_assert(sizeof(TopCandidate) == 2 * sizeof(Index)); + static_assert(sizeof(TopCandidate) == sizeof(Index)); /** * @brief Return the first minimum position through the Cartesian BP * reduction. */ std::size_t cartesian_arg_min(std::size_t left, std::size_t right) const { - const std::size_t first_close = cached_select0(left + 1); - const std::size_t last_close = cached_select0(right); + const std::size_t first_close = select_close_position(left + 1); + const std::size_t last_close = select_close_position(right); if (first_close == npos || last_close == npos || first_close > last_close) { return npos; } @@ -1489,7 +1397,6 @@ class CartesianTreeSegmentBTreeXLRmq void build() { bp_bits_.clear(); bp_bit_count_ = 0; - close_select_samples_.clear(); top_sparse_candidates_.clear(); top_block_size_ = kMinTopSparseBlockSize; top_block_count_ = 0; @@ -1507,7 +1414,6 @@ class CartesianTreeSegmentBTreeXLRmq bp_bit_count_ = 2 * values_.size(); bp_bits_.assign((bp_bit_count_ + 63) / 64, 0); build_bp_bits(); - build_close_select_samples(); build_top_sparse_table(); reset_bp_indexes(); } @@ -1557,34 +1463,7 @@ class CartesianTreeSegmentBTreeXLRmq } const std::span words(bp_bits_); bp_index_.emplace(words, bp_bit_count_); - bp_depth_rmq_ = BpDepthRmq(words, bp_bit_count_ + 1); - } - - /** - * @brief Build dense samples for close-parenthesis select queries. - */ - void build_close_select_samples() { - close_select_samples_.clear(); - close_select_samples_.reserve( - (values_.size() + kCloseSelectSampleFrequency - 1) / - kCloseSelectSampleFrequency); - - std::size_t zero_rank = 0; - for (std::size_t word = 0; word < bp_bits_.size(); ++word) { - const std::size_t word_begin = word * 64; - const std::size_t word_bits = - std::min(64, bp_bit_count_ - word_begin); - std::uint64_t zeros = (~bp_bits_[word]) & first_bits_mask(word_bits); - while (zeros != 0) { - const std::size_t offset = std::countr_zero(zeros); - ++zero_rank; - if ((zero_rank - 1) % kCloseSelectSampleFrequency == 0) { - close_select_samples_.push_back( - static_cast(word_begin + offset)); - } - zeros &= zeros - 1; - } - } + bp_depth_rmq_ = BpDepthRmq(words, bp_bit_count_ + 1, *bp_index_); } /** @@ -1636,14 +1515,13 @@ class CartesianTreeSegmentBTreeXLRmq } /** - * @brief Wrap an original value position with its Cartesian close position. + * @brief Wrap an original value position as a top sparse-table candidate. */ TopCandidate make_top_candidate(std::size_t position) const { if (position >= values_.size()) { return {}; } - return {static_cast(position), - static_cast(cached_select0(position + 1))}; + return {static_cast(position)}; } /** @@ -1765,51 +1643,20 @@ class CartesianTreeSegmentBTreeXLRmq } /** - * @brief Return the one-based rank-th close from the dense sample table. + * @brief Return the one-based rank-th Cartesian close parenthesis. */ - std::size_t cached_select0(std::size_t rank) const { - if (rank == 0 || rank > values_.size()) { - return npos; - } - const std::size_t sample_index = (rank - 1) / kCloseSelectSampleFrequency; - const std::size_t sample_rank = - sample_index * kCloseSelectSampleFrequency + 1; - if (sample_index >= close_select_samples_.size()) { + std::size_t select_close_position(std::size_t rank) const { + if (rank == 0 || rank > values_.size() || !bp_index_) { return npos; } - return select0_from_position(close_select_samples_[sample_index], - rank - sample_rank + 1); - } - - /** - * @brief Select a zero bit by scanning forward from a sampled zero position. - */ - std::size_t select0_from_position(std::size_t position, - std::size_t rank) const { - std::size_t word = position >> 6; - std::size_t offset = position & 63; - while (word < bp_bits_.size()) { - const std::size_t word_begin = word * 64; - const std::size_t word_bits = - std::min(64, bp_bit_count_ - word_begin); - std::uint64_t zeros = (~bp_bits_[word]) & first_bits_mask(word_bits); - zeros &= ~first_bits_mask(offset); - const std::size_t count = std::popcount(zeros); - if (rank <= count) { - return word_begin + select_64(zeros, rank - 1); - } - rank -= count; - ++word; - offset = 0; - } - return npos; + const std::size_t position = bp_index_->select0(rank); + return position < bp_bit_count_ ? position : npos; } std::span values_; Compare compare_; std::vector bp_bits_; std::size_t bp_bit_count_ = 0; - std::vector close_select_samples_; std::vector top_sparse_candidates_; std::size_t top_block_size_ = kMinTopSparseBlockSize; std::size_t top_block_count_ = 0; diff --git a/include/pixie/rmq/rmq_base.h b/include/pixie/rmq/rmq_base.h index d0007ab..0008b4b 100644 --- a/include/pixie/rmq/rmq_base.h +++ b/include/pixie/rmq/rmq_base.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -66,6 +67,24 @@ class RmqBase { return impl().value_at_impl(position); } + /** + * @brief Return owned auxiliary memory usage in bytes when implemented. + * + * @details Implementations opt in by exposing + * `memory_usage_bytes_impl() const`. The reported value should include the + * index object itself and heap buffers owned by the index, but not the + * external value array indexed by the RMQ. + */ + std::size_t memory_usage_bytes() const + requires requires(const Impl& concrete) { + { + concrete.memory_usage_bytes_impl() + } -> std::convertible_to; + } + { + return impl().memory_usage_bytes_impl(); + } + private: /** * @brief Return this object as its concrete CRTP implementation. diff --git a/include/pixie/rmq/sdsl_sct_rmq.h b/include/pixie/rmq/sdsl_sct_rmq.h index af9521e..51466f6 100644 --- a/include/pixie/rmq/sdsl_sct_rmq.h +++ b/include/pixie/rmq/sdsl_sct_rmq.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -95,6 +96,17 @@ class SdslSctRmq : public RmqBase, T> { return static_cast(rmq_(left, right - 1)); } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts the wrapper object plus SDSL's serialized SCT byte count. + * The external input values are not owned and are excluded. + */ + std::size_t memory_usage_bytes_impl() const { + sdsl::nullstream out; + return sizeof(*this) + static_cast(rmq_.serialize(out)); + } + private: std::span values_; sdsl::rmq_succinct_sct rmq_; diff --git a/include/pixie/rmq/segment_btree_xl.h b/include/pixie/rmq/segment_btree_xl.h index a673ca6..55b1462 100644 --- a/include/pixie/rmq/segment_btree_xl.h +++ b/include/pixie/rmq/segment_btree_xl.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -215,6 +216,30 @@ class SegmentBTreeXL return query_node(level, node_index, left, right).position; } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this RMQ object and all selector/metadata buffers. The + * external input values are not owned and are excluded. + */ + std::size_t memory_usage_bytes_impl() const { + std::size_t bytes = sizeof(*this); + bytes += pixie::vector_capacity_bytes(leaf_selectors_); + bytes += pixie::vector_capacity_bytes(medium_selectors_); + bytes += pixie::vector_capacity_bytes(medium_min_values_); + bytes += pixie::vector_capacity_bytes(high_selectors_); + bytes += pixie::vector_capacity_bytes(high_min_positions_); + bytes += pixie::vector_capacity_bytes(high_min_values_); + bytes += pixie::vector_capacity_bytes(high_child_metadata_); + bytes += pixie::vector_capacity_bytes(high_sparse_min_slots_); + bytes += pixie::vector_capacity_bytes(medium_level_offsets_); + bytes += pixie::vector_capacity_bytes(high_level_offsets_); + bytes += pixie::vector_capacity_bytes(level_sizes_); + bytes += pixie::vector_capacity_bytes(level_value_spans_); + bytes += pixie::vector_capacity_bytes(level_fanouts_); + return bytes; + } + private: static constexpr std::size_t kSelectorEntries = 256; static constexpr std::size_t kSelectorBits = 2 * kSelectorEntries; diff --git a/include/pixie/rmq/segment_tree.h b/include/pixie/rmq/segment_tree.h index 24ce332..86d4125 100644 --- a/include/pixie/rmq/segment_tree.h +++ b/include/pixie/rmq/segment_tree.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -99,6 +100,16 @@ class SegmentTree : public RmqBase, T> { return answer; } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this segment-tree object and its flat index buffer. The + * external input values are not owned and are excluded. + */ + std::size_t memory_usage_bytes_impl() const { + return sizeof(*this) + pixie::vector_capacity_bytes(tree_); + } + private: /** * @brief Choose the better of two candidate positions. diff --git a/include/pixie/rmq/sparse_table.h b/include/pixie/rmq/sparse_table.h index 90b0699..fdb10d4 100644 --- a/include/pixie/rmq/sparse_table.h +++ b/include/pixie/rmq/sparse_table.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -98,6 +99,21 @@ class SparseTable return better(first, second); } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this sparse-table object and all stored index levels. + * The external input values are not owned and are excluded. + */ + std::size_t memory_usage_bytes_impl() const { + std::size_t bytes = sizeof(*this); + bytes += pixie::vector_capacity_bytes(table_); + for (const TableLevel& level : table_) { + bytes += pixie::vector_capacity_bytes(level); + } + return bytes; + } + private: /** * @brief Choose the better of two candidate positions. From 82db9cff4d3c7d7ebb25c57343c5c82f44dcf0a1 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Thu, 11 Jun 2026 15:44:35 +0300 Subject: [PATCH 15/27] Finalized 2n bit rmq version --- include/pixie/rmq.h | 109 +++++++---- .../cartesian_tree_segment_btree_xl_rmq.h | 180 ++++++++++++++---- src/benchmarks/bench_rmq.cpp | 116 ++++++++--- 3 files changed, 301 insertions(+), 104 deletions(-) diff --git a/include/pixie/rmq.h b/include/pixie/rmq.h index 2c7bdc0..7b3ac5c 100644 --- a/include/pixie/rmq.h +++ b/include/pixie/rmq.h @@ -1,5 +1,6 @@ #pragma once +// clang-format off /* * RMQ benchmark snapshot, 2026-06-11. * @@ -16,55 +17,80 @@ * Query CPU time. "max width" is the benchmark's maximum sampled query width. * Sparse-table rows are intentionally not registered above 2^22. "cartesian xl" * is the experimental `rmq_cartesian_tree_segment_btree_xl` row. The - * cartesian xl column was refreshed after removing leaf summaries, unused top - * sparse-table close positions, and the close-select sample cache. + * cartesian xl column was refreshed after reverting the short direct value scan + * so queries do not scan the original value array, embedding middle-node min + * summaries into the selector tail bits, and using a single root-level BP + * high-sparse layout inside the Cartesian XL fallback. This snapshot uses the + * no-LeafMinimum Cartesian XL ablation for all sizes. Rejected local value + * overlays, close-position caches, and alternative BP leaf/fanout layouts are + * not included. + * + * Rejected 2^28 Cartesian XL LeafSummary ablation, same binary and host, + * measured with `PIXIE_BENCH_VALUE_VARIANTS=1` to keep peak memory bounded. + * Query times are CPU ns/query; build times are CPU ms. LeafSummary stored one + * cached full-leaf minimum offset/excess record per BP-depth leaf, but it was + * removed because the speedup was mixed for the extra memory. + * + * | N | max width | no LeafSummary (ns) | LeafSummary (ns) | delta | + * |-----:|----------:|--------------------:|-----------------:|------:| + * | 2^28 | 64 | 422.6 | 401.9 | -4.9% | + * | 2^28 | 4096 | 607.2 | 624.2 | +2.8% | + * | 2^28 | 2^18 | 443.6 | 360.2 | -18.8% | + * | 2^28 | 2^22 | 71.2 | 76.8 | +7.9% | + * | 2^28 | 2^26 | 32.4 | 23.7 | -27.1% | + * | 2^28 | 2^28 | 19.4 | 21.2 | +9.6% | + * + * | N | variant | build CPU (ms) | aux memory (MiB) | aux bits/value | + * |-----:|----------------|---------------:|-----------------:|---------------:| + * | 2^28 | no LeafSummary | 1986.003 | 68.536 | 2.142 | + * | 2^28 | LeafSummary | 1931.149 | 72.536 | 2.267 | * * | N | max width | sparse table (ns) | segment tree (ns) | segment btree xl (ns) | cartesian tree (ns) | cartesian xl (ns) | sdsl sct (ns) | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^10 | 64 | 10.1 | 37.6 | 35.0 | 71.4 | 90.5 | 120.6 | - * | 2^10 | 2^10 | 13.3 | 62.0 | 42.0 | 93.2 | 144.6 | 255.8 | + * | 2^10 | 64 | 10.9 | 38.8 | 35.1 | 73.1 | 88.5 | 119.4 | + * | 2^10 | 2^10 | 13.5 | 62.3 | 42.3 | 99.4 | 159.3 | 250.8 | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^14 | 64 | 17.7 | 53.6 | 35.4 | 93.6 | 103.8 | 130.5 | - * | 2^14 | 4096 | 16.1 | 83.8 | 39.6 | 111.7 | 95.1 | 433.0 | - * | 2^14 | 2^14 | 18.9 | 121.1 | 34.6 | 107.8 | 60.7 | 534.9 | + * | 2^14 | 64 | 16.2 | 50.8 | 34.7 | 90.0 | 101.4 | 129.2 | + * | 2^14 | 4096 | 16.1 | 93.5 | 40.4 | 114.9 | 99.7 | 438.5 | + * | 2^14 | 2^14 | 16.3 | 106.9 | 37.3 | 112.1 | 61.5 | 548.6 | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^18 | 64 | 71.8 | 119.1 | 56.4 | 91.2 | 99.0 | 144.4 | - * | 2^18 | 4096 | 53.2 | 239.0 | 63.8 | 120.5 | 98.2 | 535.8 | - * | 2^18 | 2^18 | 47.6 | 336.2 | 45.1 | 126.3 | 26.8 | 737.6 | + * | 2^18 | 64 | 72.9 | 123.0 | 61.0 | 94.4 | 124.7 | 144.8 | + * | 2^18 | 4096 | 58.4 | 224.3 | 62.6 | 130.5 | 210.5 | 556.1 | + * | 2^18 | 2^18 | 53.1 | 348.7 | 44.3 | 129.5 | 37.1 | 758.6 | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^22 | 64 | 103.0 | 192.4 | 140.3 | 128.7 | 143.9 | 231.9 | - * | 2^22 | 4096 | 83.6 | 341.8 | 166.6 | 346.8 | 174.6 | 741.0 | - * | 2^22 | 2^18 | 73.9 | 477.8 | 67.3 | 372.1 | 41.6 | 951.0 | - * | 2^22 | 2^22 | 64.3 | 506.0 | 43.1 | 254.3 | 21.0 | 939.4 | + * | 2^22 | 64 | 118.2 | 191.9 | 133.0 | 134.7 | 136.4 | 218.1 | + * | 2^22 | 4096 | 86.2 | 337.8 | 113.8 | 364.7 | 279.9 | 704.6 | + * | 2^22 | 2^18 | 71.5 | 463.3 | 69.0 | 380.6 | 62.5 | 1018.8 | + * | 2^22 | 2^22 | 66.1 | 540.1 | 45.1 | 345.9 | 24.1 | 1091.4 | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^24 | 64 | - | 257.3 | 179.7 | 237.2 | 241.9 | 455.3 | - * | 2^24 | 4096 | - | 462.3 | 321.1 | 606.0 | 517.7 | 1254.6 | - * | 2^24 | 2^18 | - | 538.2 | 138.9 | 530.7 | 91.4 | 1855.2 | - * | 2^24 | 2^22 | - | 638.3 | 58.2 | 462.0 | 24.4 | 1512.0 | - * | 2^24 | 2^24 | - | 663.9 | 48.7 | 416.0 | 21.0 | 1562.5 | + * | 2^24 | 64 | - | 219.7 | 181.7 | 244.7 | 279.7 | 330.7 | + * | 2^24 | 4096 | - | 399.7 | 267.2 | 639.8 | 465.1 | 1049.7 | + * | 2^24 | 2^18 | - | 528.0 | 151.6 | 498.7 | 99.6 | 1408.6 | + * | 2^24 | 2^22 | - | 586.8 | 51.7 | 421.4 | 24.0 | 1546.3 | + * | 2^24 | 2^24 | - | 616.8 | 40.3 | 554.3 | 20.9 | 1294.7 | * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^26 | 64 | - | 299.1 | 274.2 | 342.3 | 362.2 | 643.3 | - * | 2^26 | 4096 | - | 519.7 | 319.1 | 608.6 | 614.7 | 1591.8 | - * | 2^26 | 2^18 | - | 742.8 | 218.2 | 569.2 | 154.6 | 2465.1 | - * | 2^26 | 2^22 | - | 1066.8 | 64.7 | 596.5 | 41.0 | 2187.8 | - * | 2^26 | 2^26 | - | 1010.0 | 62.1 | 665.6 | 21.5 | 2080.6 | + * | 2^26 | 64 | - | 288.7 | 251.7 | 302.2 | 377.2 | 616.2 | + * | 2^26 | 4096 | - | 533.1 | 307.8 | 621.7 | 680.9 | 2030.8 | + * | 2^26 | 2^18 | - | 838.9 | 270.9 | 583.3 | 201.6 | 2426.9 | + * | 2^26 | 2^22 | - | 842.4 | 63.3 | 659.4 | 33.6 | 2190.1 | + * | 2^26 | 2^26 | - | 937.0 | 57.1 | 587.5 | 21.5 | 1973.5 | * * Build CPU time. Sparse-table build rows are intentionally not registered * above 2^22. * * | N | sparse table (ms) | segment tree (ms) | segment btree xl (ms) | cartesian tree (ms) | cartesian xl (ms) | sdsl sct (ms) | * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^10 | 0.007 | 0.001 | 0.001 | 0.006 | 0.007 | 0.007 | + * | 2^10 | 0.007 | 0.001 | 0.001 | 0.006 | 0.008 | 0.007 | * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^14 | 1.118 | 0.015 | 0.017 | 0.040 | 0.051 | 0.164 | + * | 2^14 | 1.123 | 0.015 | 0.017 | 0.041 | 0.059 | 0.213 | * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^18 | 33.542 | 0.378 | 0.303 | 1.741 | 1.924 | 2.585 | + * | 2^18 | 29.597 | 0.358 | 0.281 | 1.678 | 1.845 | 2.539 | * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^22 | 934.032 | 65.087 | 5.548 | 29.030 | 30.147 | 40.664 | + * | 2^22 | 964.147 | 63.087 | 5.662 | 29.543 | 30.573 | 40.821 | * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^24 | - | 289.874 | 20.924 | 172.132 | 120.261 | 162.269 | + * | 2^24 | - | 273.543 | 21.747 | 167.399 | 118.968 | 164.629 | * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^26 | - | 1181.042 | 86.430 | 727.667 | 458.843 | 670.310 | + * | 2^26 | - | 1176.694 | 85.392 | 728.030 | 476.286 | 650.639 | * * Owned auxiliary memory. Benchmarks use 64-bit signed integer values and * 64-bit indexes. The external input values are not owned by RMQ indexes and @@ -74,32 +100,33 @@ * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| * | 2^10 | 0.071 | 0.016 | 0.009 | 0.004 | 0.014 | 0.001 | * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| - * | 2^14 | 1.626 | 0.250 | 0.011 | 0.025 | 0.017 | 0.005 | + * | 2^14 | 1.626 | 0.250 | 0.011 | 0.025 | 0.018 | 0.005 | * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| - * | 2^18 | 34.001 | 4.000 | 0.065 | 0.474 | 0.132 | 0.079 | + * | 2^18 | 34.001 | 4.000 | 0.065 | 0.474 | 0.082 | 0.079 | * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| - * | 2^22 | 672.001 | 64.000 | 0.801 | 9.539 | 1.791 | 1.262 | + * | 2^22 | 672.001 | 64.000 | 0.801 | 9.539 | 1.141 | 1.262 | * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| - * | 2^24 | - | 256.000 | 3.155 | 42.148 | 4.611 | 5.051 | + * | 2^24 | - | 256.000 | 3.155 | 42.148 | 4.585 | 5.051 | * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| - * | 2^26 | - | 1024.000 | 8.340 | 184.582 | 18.631 | 20.224 | + * | 2^26 | - | 1024.000 | 8.340 | 184.582 | 18.551 | 20.224 | * * Owned auxiliary memory normalized by indexed value count (bits/value). * * | N | sparse table | segment tree | segment btree xl | cartesian tree | cartesian xl | sdsl sct | * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^10 | 579.188 | 128.438 | 71.438 | 30.750 | 112.312 | 7.312 | + * | 2^10 | 579.188 | 128.438 | 71.438 | 30.750 | 112.938 | 7.312 | * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^14 | 832.262 | 128.027 | 5.434 | 12.562 | 8.938 | 2.770 | + * | 2^14 | 832.262 | 128.027 | 5.434 | 12.562 | 8.977 | 2.770 | * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^18 | 1088.020 | 128.002 | 2.089 | 15.175 | 4.228 | 2.534 | + * | 2^18 | 1088.020 | 128.002 | 2.089 | 15.175 | 2.629 | 2.534 | * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^22 | 1344.002 | 128.000 | 1.603 | 19.079 | 3.582 | 2.524 | + * | 2^22 | 1344.002 | 128.000 | 1.603 | 19.079 | 2.281 | 2.524 | * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^24 | - | 128.000 | 1.577 | 21.074 | 2.305 | 2.526 | + * | 2^24 | - | 128.000 | 1.577 | 21.074 | 2.293 | 2.526 | * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^26 | - | 128.000 | 1.042 | 23.073 | 2.329 | 2.528 | + * | 2^26 | - | 128.000 | 1.042 | 23.073 | 2.319 | 2.528 | */ +// clang-format on #include #include diff --git a/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h b/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h index 1bf3d87..ef643c1 100644 --- a/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h +++ b/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h @@ -33,10 +33,10 @@ namespace pixie::rmq::experimental { * This backend mirrors the high-level query shape of `SegmentBTreeXL`: depth * positions are split into leaves, leaves are grouped into a B-tree, and every * internal node stores a local Cartesian/BP selector over the minima of its - * immediate children. The top one or two levels additionally keep sparse - * tables over child-minimum slots. Unlike value `SegmentBTreeXL`, leaves do not - * store another local Cartesian selector or cached minima. Leaf minima are - * recomputed from the original BP delta bits with the 128-bit excess + * immediate children. A configurable number of top levels additionally keep + * sparse tables over child-minimum slots. Unlike value `SegmentBTreeXL`, leaves + * do not store another local Cartesian selector or cached minima. Leaf minima + * are recomputed from the original BP delta bits with the 128-bit excess * primitives from `bits.h`, using rank support to recover the absolute base * depth at the leaf start. * @@ -44,11 +44,17 @@ namespace pixie::rmq::experimental { * @tparam LeafSize Number of depth positions per low-level leaf. * @tparam Fanout Fanout used by the top high levels. * @tparam MiddleFanout Fanout used below the high levels. + * @tparam UseHighSparseLayout Whether top levels use sparse tables instead of + * local BP selectors. + * @tparam HighSparseLayoutLevels Number of top tree levels using that sparse + * layout when enabled. */ template + std::size_t MiddleFanout = 192, + bool UseHighSparseLayout = true, + std::size_t HighSparseLayoutLevels = 2> class SegmentBTreeXLPlusMinusOneRmq { public: static_assert(std::is_unsigned_v, @@ -62,12 +68,17 @@ class SegmentBTreeXLPlusMinusOneRmq { static_assert(MiddleFanout > 1 && MiddleFanout <= 256, "SegmentBTreeXLPlusMinusOneRmq middle fanout must be in " "[2, 256]"); + static_assert(!UseHighSparseLayout || HighSparseLayoutLevels > 0, + "SegmentBTreeXLPlusMinusOneRmq high sparse layout must cover " + "at least one level when enabled"); static constexpr std::size_t npos = std::numeric_limits::max(); static constexpr Index invalid_index = std::numeric_limits::max(); static constexpr std::size_t kLeafSize = LeafSize; static constexpr std::size_t kFanout = Fanout; static constexpr std::size_t kMiddleFanout = MiddleFanout; + static constexpr bool kUseHighSparseLayout = UseHighSparseLayout; + static constexpr std::size_t kHighSparseLayoutLevels = HighSparseLayoutLevels; /** * @brief Construct an empty ±1 RMQ index. @@ -148,6 +159,7 @@ class SegmentBTreeXLPlusMinusOneRmq { bytes += pixie::vector_capacity_bytes(high_child_metadata_); bytes += pixie::vector_capacity_bytes(high_sparse_min_slots_); bytes += pixie::vector_capacity_bytes(internal_level_offsets_); + bytes += pixie::vector_capacity_bytes(min_summary_level_offsets_); bytes += pixie::vector_capacity_bytes(high_level_offsets_); bytes += pixie::vector_capacity_bytes(level_sizes_); bytes += pixie::vector_capacity_bytes(level_position_spans_); @@ -207,12 +219,22 @@ class SegmentBTreeXLPlusMinusOneRmq { static constexpr std::size_t kSelectorEntries = 256; static constexpr std::size_t kSelectorBits = 2 * kSelectorEntries; static constexpr std::size_t kSelectorWords = kSelectorBits / 64; - static constexpr std::size_t kHighSparseLevels = + static constexpr std::size_t kEmbeddedSummaryWords = 2; + static constexpr std::size_t kEmbeddedSummaryBits = + 64 * kEmbeddedSummaryWords; + static constexpr std::size_t kEmbeddedSummaryMaxEntries = + (kSelectorBits - kEmbeddedSummaryBits) / 2; + static constexpr std::size_t kEmbeddedSummaryPositionWord = + kSelectorWords - 2; + static constexpr std::size_t kEmbeddedSummaryDepthWord = kSelectorWords - 1; + static constexpr std::size_t kHighSparseTableLevels = static_cast(std::bit_width(Fanout)); static constexpr std::size_t kHighSparseSlotsPerNode = - kHighSparseLevels * Fanout; + kHighSparseTableLevels * Fanout; static constexpr bool kInvalidIndexEqualsNpos = static_cast(invalid_index) == npos; + static_assert(kEmbeddedSummaryMaxEntries == 192); + static_assert(sizeof(std::size_t) <= sizeof(std::uint64_t)); struct DepthCandidate { std::size_t position = npos; @@ -305,6 +327,34 @@ class SegmentBTreeXLPlusMinusOneRmq { return entry < entry_count ? entry : npos; } + /** + * @brief Store a node's subtree minimum in the unused selector tail. + * + * @details This is valid for nodes with at most 192 entries. Their BP + * sequence uses at most the first 384 bits, and all local selector queries + * are bounded by `2 * entry_count`, so the final two words are available + * for node metadata. + */ + void set_embedded_min_summary(std::size_t position, std::int64_t depth) { + bp_bits_[kEmbeddedSummaryPositionWord] = + static_cast(position); + bp_bits_[kEmbeddedSummaryDepthWord] = std::bit_cast(depth); + } + + /** + * @brief Return the embedded subtree-minimum position. + */ + std::size_t embedded_min_position() const { + return static_cast(bp_bits_[kEmbeddedSummaryPositionWord]); + } + + /** + * @brief Return the embedded subtree-minimum depth. + */ + std::int64_t embedded_min_depth() const { + return std::bit_cast(bp_bits_[kEmbeddedSummaryDepthWord]); + } + private: /** * @brief Prepend one BP bit while building the sequence right-to-left. @@ -451,6 +501,7 @@ class SegmentBTreeXLPlusMinusOneRmq { high_child_metadata_.clear(); high_sparse_min_slots_.clear(); internal_level_offsets_.clear(); + min_summary_level_offsets_.clear(); high_level_offsets_.clear(); level_sizes_.clear(); level_position_spans_.clear(); @@ -515,32 +566,47 @@ class SegmentBTreeXLPlusMinusOneRmq { level_position_spans_.push_back(current_span); } internal_level_offsets_.assign(level_count(), 0); + min_summary_level_offsets_.assign(level_count(), npos); high_level_offsets_.assign(level_count(), 0); if (level_count() <= 1) { return; } - const std::size_t root_level = level_count() - 1; - high_level_begin_ = root_level > 1 ? root_level - 1 : 1; - std::size_t internal_count = 0; for (std::size_t level = 1; level < level_count(); ++level) { internal_level_offsets_[level] = internal_count; internal_count += level_sizes_[level]; } internal_selectors_.resize(internal_count); - internal_min_positions_.resize(internal_count, invalid_index); - internal_min_depths_.resize(internal_count, - std::numeric_limits::max()); - std::size_t high_node_count = 0; - for (std::size_t level = high_level_begin_; level < level_count(); - ++level) { - high_level_offsets_[level] = high_node_count; - high_node_count += level_sizes_[level]; + if constexpr (UseHighSparseLayout) { + const std::size_t root_level = level_count() - 1; + const std::size_t high_layout_levels = + std::min(HighSparseLayoutLevels, root_level); + high_level_begin_ = root_level + 1 - high_layout_levels; + + std::size_t high_node_count = 0; + for (std::size_t level = high_level_begin_; level < level_count(); + ++level) { + high_level_offsets_[level] = high_node_count; + high_node_count += level_sizes_[level]; + } + high_child_metadata_.resize(high_node_count * Fanout); + high_sparse_min_slots_.resize(high_node_count * kHighSparseSlotsPerNode); + } else { + high_level_begin_ = level_count(); } - high_child_metadata_.resize(high_node_count * Fanout); - high_sparse_min_slots_.resize(high_node_count * kHighSparseSlotsPerNode); + + std::size_t side_summary_count = 0; + for (std::size_t level = 1; level < level_count(); ++level) { + if (!level_embeds_min_summary(level)) { + min_summary_level_offsets_[level] = side_summary_count; + side_summary_count += level_sizes_[level]; + } + } + internal_min_positions_.resize(side_summary_count, invalid_index); + internal_min_depths_.resize(side_summary_count, + std::numeric_limits::max()); } /** @@ -554,8 +620,7 @@ class SegmentBTreeXLPlusMinusOneRmq { std::array child_minima{}; for (std::size_t slot = 0; slot < count; ++slot) { - child_minima[slot] = - subtree_min_candidate(level - 1, first_child + slot); + child_minima[slot] = subtree_min_candidate(level - 1, first_child + slot); } if (high_level) { @@ -574,15 +639,18 @@ class SegmentBTreeXLPlusMinusOneRmq { Bp512Selector& selector = mutable_selector_at(level, node); selector.build(count, [&](std::size_t left, std::size_t right) { - return strictly_better_candidate(child_minima[left], - child_minima[right]); + return strictly_better_candidate(child_minima[left], child_minima[right]); }); const std::size_t slot = selector.arg_min(0, count, count); const DepthCandidate minimum = child_minima[slot]; - const std::size_t flat = internal_flat_index(level, node); - internal_min_positions_[flat] = static_cast(minimum.position); - internal_min_depths_[flat] = minimum.depth; + if (level_embeds_min_summary(level)) { + selector.set_embedded_min_summary(minimum.position, minimum.depth); + } else { + const std::size_t flat = min_summary_flat_index(level, node); + internal_min_positions_[flat] = static_cast(minimum.position); + internal_min_depths_[flat] = minimum.depth; + } } /** @@ -924,7 +992,11 @@ class SegmentBTreeXLPlusMinusOneRmq { return leaf_range_min(node, node_position_begin(0, node), node_position_end(0, node)); } - const std::size_t flat = internal_flat_index(level, node); + if (level_embeds_min_summary(level)) { + const Bp512Selector& selector = selector_at(level, node); + return {selector.embedded_min_position(), selector.embedded_min_depth()}; + } + const std::size_t flat = min_summary_flat_index(level, node); return {static_cast(internal_min_positions_[flat]), internal_min_depths_[flat]}; } @@ -976,8 +1048,10 @@ class SegmentBTreeXLPlusMinusOneRmq { std::size_t slot_left, std::size_t slot_right, std::size_t count) const { - if (is_high_level(level)) { - return high_sparse_arg_min(level, node, slot_left, slot_right, count); + if constexpr (UseHighSparseLayout) { + if (is_high_level(level)) { + return high_sparse_arg_min(level, node, slot_left, slot_right, count); + } } return selector_at(level, node).arg_min(slot_left, slot_right, count); } @@ -986,6 +1060,10 @@ class SegmentBTreeXLPlusMinusOneRmq { * @brief Return whether a level uses the high-node layout. */ bool is_high_level(std::size_t level) const { + if constexpr (!UseHighSparseLayout) { + (void)level; + return false; + } return level > 0 && level >= high_level_begin_ && level < level_count(); } @@ -996,6 +1074,22 @@ class SegmentBTreeXLPlusMinusOneRmq { return internal_level_offsets_[level] + node; } + /** + * @brief Return whether a level embeds subtree minima in selector tail bits. + */ + bool level_embeds_min_summary(std::size_t level) const { + return level > 0 && !is_high_level(level) && + fanout_at_level(level) <= kEmbeddedSummaryMaxEntries; + } + + /** + * @brief Map a non-embedded internal node to side summary storage. + */ + std::size_t min_summary_flat_index(std::size_t level, + std::size_t node) const { + return min_summary_level_offsets_[level] + node; + } + /** * @brief Return the fanout used to group children at a level. */ @@ -1075,7 +1169,7 @@ class SegmentBTreeXLPlusMinusOneRmq { table[slot] = static_cast(slot); } - for (std::size_t table_level = 1; table_level < kHighSparseLevels; + for (std::size_t table_level = 1; table_level < kHighSparseTableLevels; ++table_level) { const std::size_t span = std::size_t{1} << table_level; if (span > count) { @@ -1126,6 +1220,7 @@ class SegmentBTreeXLPlusMinusOneRmq { std::vector high_child_metadata_; std::vector high_sparse_min_slots_; std::vector internal_level_offsets_; + std::vector min_summary_level_offsets_; std::vector high_level_offsets_; std::vector level_sizes_; std::vector level_position_spans_; @@ -1138,14 +1233,13 @@ class SegmentBTreeXLPlusMinusOneRmq { * * @details This class keeps the same public value-RMQ contract as * `CartesianTreeRmq`. It builds the same stable Ferrada-Navarro BP - * Cartesian-tree encoding, uses `BitVector` for close-parenthesis rank, keeps a - * dense close-select sample table for value endpoint mapping, and delegates the - * BP-depth minimum query to `SegmentBTreeXLPlusMinusOneRmq`. A single coarse - * value-level sparse table is checked first; it uses at least 4096-value blocks - * and grows the block width when needed so the top layer has at most 2^14 - * blocks. Wide queries whose padded block-cover minimum lies inside the - * requested range return from this top table without touching the global BP - * rank/select path. + * Cartesian-tree encoding, uses `BitVector` for close-parenthesis rank/select, + * and delegates the BP-depth minimum query to + * `SegmentBTreeXLPlusMinusOneRmq`. A single coarse value-level sparse table is + * checked first; it uses at least 4096-value blocks and grows the block width + * when needed so the top layer has at most 2^14 blocks. Wide queries whose + * padded block-cover minimum lies inside the requested range return from this + * top table without touching the global BP rank/select path. * * The implementation is intentionally kept in the experimental namespace and * is not included from `pixie/rmq.h`. It is meant to compare the usual @@ -1362,8 +1456,12 @@ class CartesianTreeSegmentBTreeXLRmq } private: - using BpDepthRmq = - SegmentBTreeXLPlusMinusOneRmq; + using BpDepthRmq = SegmentBTreeXLPlusMinusOneRmq; struct TopCandidate { Index position = invalid_index; diff --git a/src/benchmarks/bench_rmq.cpp b/src/benchmarks/bench_rmq.cpp index 5d22886..3e5fea0 100644 --- a/src/benchmarks/bench_rmq.cpp +++ b/src/benchmarks/bench_rmq.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -54,6 +55,11 @@ using Index = std::size_t; static_assert(kQueryPoolBytes % sizeof(std::pair) == 0); +template +concept HasRmqMemoryUsage = requires(const Rmq& rmq) { + { rmq.memory_usage_bytes() } -> std::convertible_to; +}; + struct Dataset { std::size_t size = 0; std::size_t max_width = 0; @@ -815,6 +821,15 @@ std::vector make_block_summaries( return summaries; } +void set_aux_memory_counters(benchmark::State& state, + std::size_t size, + double aux_bytes); + +template +void set_query_aux_memory_counters(benchmark::State& state, + std::size_t size, + const std::vector& rmqs); + template void run_queries(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); @@ -855,6 +870,7 @@ void run_queries(benchmark::State& state) { state.counters["max_width"] = static_cast(max_width); state.counters["index_bytes"] = static_cast(sizeof(Index)); state.counters["value_arrays"] = static_cast(datasets.size()); + set_query_aux_memory_counters(state, size, rmqs); } void set_build_counters(benchmark::State& state, @@ -867,6 +883,29 @@ void set_build_counters(benchmark::State& state, static_cast(size)); } +void set_aux_memory_counters(benchmark::State& state, + std::size_t size, + double aux_bytes) { + state.counters["aux_bytes"] = aux_bytes; + state.counters["aux_mib"] = aux_bytes / (1024.0 * 1024.0); + state.counters["aux_bits_per_value"] = + size == 0 ? 0.0 : (8.0 * aux_bytes) / static_cast(size); +} + +template +void set_query_aux_memory_counters(benchmark::State& state, + std::size_t size, + const std::vector& rmqs) { + if constexpr (HasRmqMemoryUsage) { + double total = 0.0; + for (const Rmq& rmq : rmqs) { + total += static_cast(rmq.memory_usage_bytes()); + } + const double average = rmqs.empty() ? 0.0 : total / rmqs.size(); + set_aux_memory_counters(state, size, average); + } +} + template void run_value_rmq_build(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); @@ -889,6 +928,11 @@ void run_value_rmq_build(benchmark::State& state) { set_build_counters(state, size, datasets.front().values.size() * sizeof(std::int64_t)); state.counters["value_arrays"] = static_cast(datasets.size()); + if constexpr (HasRmqMemoryUsage) { + const Rmq rmq(std::span(datasets.front().values)); + const std::size_t aux_bytes = rmq.memory_usage_bytes(); + set_aux_memory_counters(state, size, static_cast(aux_bytes)); + } } template @@ -904,6 +948,11 @@ void run_bp_rmq_build(benchmark::State& state) { } set_build_counters(state, size, dataset.bits.size() * sizeof(std::uint64_t)); + if constexpr (HasRmqMemoryUsage) { + const Rmq rmq(std::span(dataset.bits), dataset.size); + const std::size_t aux_bytes = rmq.memory_usage_bytes(); + set_aux_memory_counters(state, size, static_cast(aux_bytes)); + } } void run_sparse_table_footprint(benchmark::State& state) { @@ -1109,18 +1158,42 @@ void run_depth_queries(benchmark::State& state) { } set_depth_counters(state, dataset, true, BlockSize); + if constexpr (HasRmqMemoryUsage) { + set_aux_memory_counters(state, size, + static_cast(rmq.memory_usage_bytes())); + } } void register_benchmarks() { + using CartesianXl = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq< + std::int64_t, std::less, Index>; + const std::vector sizes = {1ull << 10, 1ull << 14, 1ull << 18, 1ull << 22, 1ull << 24, 1ull << 26}; const std::vector build_sizes = { - 1ull << 10, 1ull << 14, 1ull << 18, 1ull << 22, 1ull << 24}; + 1ull << 10, 1ull << 14, 1ull << 18, 1ull << 22, 1ull << 24, 1ull << 26}; const std::vector widths = {64, 4096, 1ull << 18, 1ull << 22, 1ull << 26}; // Pure sparse-table rows materialize O(n log n) indexes; above 2^22 they // dominate memory/runtime and make large benchmark passes noisy. constexpr std::size_t kSparseTableBenchmarkMaxSize = 1ull << 22; + constexpr std::size_t kFocusedCartesianXlLargeSize = 1ull << 28; + + auto effective_widths_for = [&](std::size_t size) { + std::vector effective_widths; + for (const std::size_t width : widths) { + if (width > size) { + continue; + } + effective_widths.push_back(width); + } + effective_widths.push_back(size); + std::sort(effective_widths.begin(), effective_widths.end()); + effective_widths.erase( + std::unique(effective_widths.begin(), effective_widths.end()), + effective_widths.end()); + return effective_widths; + }; for (const std::size_t size : build_sizes) { if (size <= kSparseTableBenchmarkMaxSize) { @@ -1149,11 +1222,8 @@ void register_benchmarks() { std::int64_t, std::less, Index>>) ->Arg(static_cast(size)) ->Unit(benchmark::kMillisecond); - benchmark::RegisterBenchmark( - "rmq_build_cartesian_tree_segment_btree_xl", - &run_value_rmq_build< - pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq< - std::int64_t, std::less, Index>>) + benchmark::RegisterBenchmark("rmq_build_cartesian_tree_segment_btree_xl", + &run_value_rmq_build) ->Arg(static_cast(size)) ->Unit(benchmark::kMillisecond); #ifdef PIXIE_THIRD_PARTY_BENCHMARKS @@ -1207,19 +1277,14 @@ void register_benchmarks() { ->Unit(benchmark::kMillisecond); } + benchmark::RegisterBenchmark("rmq_build_cartesian_tree_segment_btree_xl", + &run_value_rmq_build) + ->Arg(static_cast(kFocusedCartesianXlLargeSize)) + ->Unit(benchmark::kMillisecond); + for (const std::size_t size : sizes) { - std::vector effective_widths; - for (const std::size_t width : widths) { - if (width > size) { - continue; - } - effective_widths.push_back(width); - } - effective_widths.push_back(size); - std::sort(effective_widths.begin(), effective_widths.end()); - effective_widths.erase( - std::unique(effective_widths.begin(), effective_widths.end()), - effective_widths.end()); + const std::vector effective_widths = + effective_widths_for(size); for (const std::size_t width : effective_widths) { if (size <= kSparseTableBenchmarkMaxSize) { @@ -1267,10 +1332,8 @@ void register_benchmarks() { ->Args({static_cast(size), static_cast(width)}) ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_cartesian_tree_segment_btree_xl", - &run_queries, Index>>) + benchmark::RegisterBenchmark("rmq_cartesian_tree_segment_btree_xl", + &run_queries) ->Args({static_cast(size), static_cast(width)}) ->Unit(benchmark::kNanosecond); @@ -1445,6 +1508,15 @@ void register_benchmarks() { ->Unit(benchmark::kNanosecond); } } + + for (const std::size_t width : + effective_widths_for(kFocusedCartesianXlLargeSize)) { + benchmark::RegisterBenchmark("rmq_cartesian_tree_segment_btree_xl", + &run_queries) + ->Args({static_cast(kFocusedCartesianXlLargeSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); + } } } // namespace From 359898092683d62ef0ebfb7ad9733f59e62a323e Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Thu, 11 Jun 2026 17:44:27 +0300 Subject: [PATCH 16/27] Parameters cleanup --- .../cartesian_tree_segment_btree_xl_rmq.h | 122 ++++++++---------- 1 file changed, 57 insertions(+), 65 deletions(-) diff --git a/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h b/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h index ef643c1..fe2174e 100644 --- a/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h +++ b/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h @@ -31,19 +31,20 @@ namespace pixie::rmq::experimental { * over depth positions, and ties return the smaller depth position. * * This backend mirrors the high-level query shape of `SegmentBTreeXL`: depth - * positions are split into leaves, leaves are grouped into a B-tree, and every - * internal node stores a local Cartesian/BP selector over the minima of its - * immediate children. A configurable number of top levels additionally keep - * sparse tables over child-minimum slots. Unlike value `SegmentBTreeXL`, leaves - * do not store another local Cartesian selector or cached minima. Leaf minima - * are recomputed from the original BP delta bits with the 128-bit excess - * primitives from `bits.h`, using rank support to recover the absolute base - * depth at the leaf start. + * positions are split into configurable-size leaves, leaves are grouped into a + * B-tree, and every internal node stores a local Cartesian/BP selector over the + * minima of its immediate children. Middle levels are fixed at 192 child slots: + * their selector uses 384 BP bits and reserves the remaining 128 bits for the + * embedded subtree minimum position/depth. A configurable number of top levels + * additionally keep sparse tables over child-minimum slots. Unlike value + * `SegmentBTreeXL`, leaves do not store another local Cartesian selector or + * cached minima. Leaf minima are recomputed from the original BP delta bits + * with the 128-bit excess primitives from `bits.h`, using rank support to + * recover the absolute base depth at the leaf start. * * @tparam Index Unsigned integer type used for stored positions. - * @tparam LeafSize Number of depth positions per low-level leaf. - * @tparam Fanout Fanout used by the top high levels. - * @tparam MiddleFanout Fanout used below the high levels. + * @tparam LeafSize Number of depth positions per low-level leaf; must be a + * multiple of 512. * @tparam UseHighSparseLayout Whether top levels use sparse tables instead of * local BP selectors. * @tparam HighSparseLayoutLevels Number of top tree levels using that sparse @@ -51,23 +52,15 @@ namespace pixie::rmq::experimental { */ template class SegmentBTreeXLPlusMinusOneRmq { public: static_assert(std::is_unsigned_v, "SegmentBTreeXLPlusMinusOneRmq index type must be unsigned"); - static_assert(LeafSize != 0 && LeafSize % 128 == 0, + static_assert(LeafSize != 0 && LeafSize % 512 == 0, "SegmentBTreeXLPlusMinusOneRmq leaf size must be a positive " - "multiple of 128"); - static_assert(Fanout > 1 && Fanout <= 256, - "SegmentBTreeXLPlusMinusOneRmq high fanout must be in " - "[2, 256]"); - static_assert(MiddleFanout > 1 && MiddleFanout <= 256, - "SegmentBTreeXLPlusMinusOneRmq middle fanout must be in " - "[2, 256]"); + "multiple of 512"); static_assert(!UseHighSparseLayout || HighSparseLayoutLevels > 0, "SegmentBTreeXLPlusMinusOneRmq high sparse layout must cover " "at least one level when enabled"); @@ -75,8 +68,8 @@ class SegmentBTreeXLPlusMinusOneRmq { static constexpr std::size_t npos = std::numeric_limits::max(); static constexpr Index invalid_index = std::numeric_limits::max(); static constexpr std::size_t kLeafSize = LeafSize; - static constexpr std::size_t kFanout = Fanout; - static constexpr std::size_t kMiddleFanout = MiddleFanout; + static constexpr std::size_t kHighLevelFanout = 256; + static constexpr std::size_t kMiddleFanout = 192; static constexpr bool kUseHighSparseLayout = UseHighSparseLayout; static constexpr std::size_t kHighSparseLayoutLevels = HighSparseLayoutLevels; @@ -228,12 +221,13 @@ class SegmentBTreeXLPlusMinusOneRmq { kSelectorWords - 2; static constexpr std::size_t kEmbeddedSummaryDepthWord = kSelectorWords - 1; static constexpr std::size_t kHighSparseTableLevels = - static_cast(std::bit_width(Fanout)); + static_cast(std::bit_width(kHighLevelFanout)); static constexpr std::size_t kHighSparseSlotsPerNode = - kHighSparseTableLevels * Fanout; + kHighSparseTableLevels * kHighLevelFanout; static constexpr bool kInvalidIndexEqualsNpos = static_cast(invalid_index) == npos; static_assert(kEmbeddedSummaryMaxEntries == 192); + static_assert(kMiddleFanout == kEmbeddedSummaryMaxEntries); static_assert(sizeof(std::size_t) <= sizeof(std::uint64_t)); struct DepthCandidate { @@ -551,17 +545,17 @@ class SegmentBTreeXLPlusMinusOneRmq { std::size_t current_count = leaf_count; std::size_t current_span = LeafSize; - while (current_count > Fanout * Fanout) { - level_fanouts_.push_back(MiddleFanout); - current_count = ceil_div(current_count, MiddleFanout); - current_span = saturating_product(current_span, MiddleFanout); + while (current_count > kHighLevelFanout * kHighLevelFanout) { + level_fanouts_.push_back(kMiddleFanout); + current_count = ceil_div(current_count, kMiddleFanout); + current_span = saturating_product(current_span, kMiddleFanout); level_sizes_.push_back(current_count); level_position_spans_.push_back(current_span); } while (current_count > 1) { - level_fanouts_.push_back(Fanout); - current_count = ceil_div(current_count, Fanout); - current_span = saturating_product(current_span, Fanout); + level_fanouts_.push_back(kHighLevelFanout); + current_count = ceil_div(current_count, kHighLevelFanout); + current_span = saturating_product(current_span, kHighLevelFanout); level_sizes_.push_back(current_count); level_position_spans_.push_back(current_span); } @@ -581,9 +575,16 @@ class SegmentBTreeXLPlusMinusOneRmq { if constexpr (UseHighSparseLayout) { const std::size_t root_level = level_count() - 1; - const std::size_t high_layout_levels = - std::min(HighSparseLayoutLevels, root_level); - high_level_begin_ = root_level + 1 - high_layout_levels; + std::size_t high_layout_levels = 0; + for (std::size_t level = root_level; + level > 0 && high_layout_levels < HighSparseLayoutLevels && + fanout_at_level(level) == kHighLevelFanout; + --level) { + ++high_layout_levels; + } + high_level_begin_ = high_layout_levels == 0 + ? level_count() + : root_level + 1 - high_layout_levels; std::size_t high_node_count = 0; for (std::size_t level = high_level_begin_; level < level_count(); @@ -591,7 +592,7 @@ class SegmentBTreeXLPlusMinusOneRmq { high_level_offsets_[level] = high_node_count; high_node_count += level_sizes_[level]; } - high_child_metadata_.resize(high_node_count * Fanout); + high_child_metadata_.resize(high_node_count * kHighLevelFanout); high_sparse_min_slots_.resize(high_node_count * kHighSparseSlotsPerNode); } else { high_level_begin_ = level_count(); @@ -1109,7 +1110,8 @@ class SegmentBTreeXLPlusMinusOneRmq { */ const HighChildMetadata* high_child_metadata_begin(std::size_t level, std::size_t node) const { - return high_child_metadata_.data() + high_flat_index(level, node) * Fanout; + return high_child_metadata_.data() + + high_flat_index(level, node) * kHighLevelFanout; } /** @@ -1117,7 +1119,7 @@ class SegmentBTreeXLPlusMinusOneRmq { */ const HighChildMetadata& high_child_metadata_at(std::size_t high_flat, std::size_t slot) const { - return high_child_metadata_[high_flat * Fanout + slot]; + return high_child_metadata_[high_flat * kHighLevelFanout + slot]; } /** @@ -1125,7 +1127,7 @@ class SegmentBTreeXLPlusMinusOneRmq { */ HighChildMetadata& mutable_high_child_metadata_at(std::size_t high_flat, std::size_t slot) { - return high_child_metadata_[high_flat * Fanout + slot]; + return high_child_metadata_[high_flat * kHighLevelFanout + slot]; } /** @@ -1176,8 +1178,9 @@ class SegmentBTreeXLPlusMinusOneRmq { break; } const std::size_t half_span = span >> 1; - const std::uint8_t* previous = table + (table_level - 1) * Fanout; - std::uint8_t* current = table + table_level * Fanout; + const std::uint8_t* previous = + table + (table_level - 1) * kHighLevelFanout; + std::uint8_t* current = table + table_level * kHighLevelFanout; for (std::size_t slot = 0; slot + span <= count; ++slot) { current[slot] = static_cast(better_high_child_slot( level, node, previous[slot], previous[slot + half_span])); @@ -1205,7 +1208,7 @@ class SegmentBTreeXLPlusMinusOneRmq { const std::size_t table_level = std::bit_width(length) - 1; const std::size_t span = std::size_t{1} << table_level; const std::uint8_t* table = - high_sparse_min_slots_begin(high_flat) + table_level * Fanout; + high_sparse_min_slots_begin(high_flat) + table_level * kHighLevelFanout; return better_high_child_slot(level, node, table[slot_left], table[slot_right - span]); } @@ -1235,7 +1238,9 @@ class SegmentBTreeXLPlusMinusOneRmq { * `CartesianTreeRmq`. It builds the same stable Ferrada-Navarro BP * Cartesian-tree encoding, uses `BitVector` for close-parenthesis rank/select, * and delegates the BP-depth minimum query to - * `SegmentBTreeXLPlusMinusOneRmq`. A single coarse value-level sparse table is + * `SegmentBTreeXLPlusMinusOneRmq`. The BP-depth backend keeps a configurable + * low-level leaf size, fixed 192-entry middle nodes with embedded minima, and + * fixed 256-entry high nodes. A single coarse value-level sparse table is * checked first; it uses at least 4096-value blocks and grows the block width * when needed so the top layer has at most 2^14 blocks. Wide queries whose * padded block-cover minimum lies inside the requested range return from this @@ -1248,27 +1253,19 @@ class SegmentBTreeXLPlusMinusOneRmq { template , class Index = std::size_t, - std::size_t LeafSize = 512, - std::size_t Fanout = 256, - std::size_t MiddleFanout = 192> + std::size_t LeafSize = 512> class CartesianTreeSegmentBTreeXLRmq - : public RmqBase, - T> { + : public RmqBase< + CartesianTreeSegmentBTreeXLRmq, + T> { public: static_assert(std::is_unsigned_v, "CartesianTreeSegmentBTreeXLRmq index type must be unsigned"); + static_assert(LeafSize != 0 && LeafSize % 512 == 0, + "CartesianTreeSegmentBTreeXLRmq leaf size must be a positive " + "multiple of 512"); - using Self = CartesianTreeSegmentBTreeXLRmq; + using Self = CartesianTreeSegmentBTreeXLRmq; static constexpr std::size_t npos = RmqBase::npos; static constexpr Index invalid_index = std::numeric_limits::max(); @@ -1456,12 +1453,7 @@ class CartesianTreeSegmentBTreeXLRmq } private: - using BpDepthRmq = SegmentBTreeXLPlusMinusOneRmq; + using BpDepthRmq = SegmentBTreeXLPlusMinusOneRmq; struct TopCandidate { Index position = invalid_index; From d778b26e69727fb6c85357ec2fb6d2973a2607d9 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Fri, 12 Jun 2026 19:31:16 +0300 Subject: [PATCH 17/27] test correction --- src/tests/rmq_tests.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index 210011e..7ed60b5 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -1288,8 +1288,8 @@ TYPED_TEST(DepthRmqContractTest, DifferentialRandomWalks) { TEST(RmqSegmentBTreeXLPlusMinusOne, DefaultBoundaryRanges) { using Rmq = pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq<>; constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kFanout = Rmq::kFanout; - constexpr std::size_t kHighBoundary = kLeaf * kFanout; + constexpr std::size_t kHighFanout = Rmq::kHighLevelFanout; + constexpr std::size_t kHighBoundary = kLeaf * kHighFanout; const std::vector sizes = { kLeaf - 1, kLeaf, kLeaf + 1, kHighBoundary - 1, kHighBoundary, kHighBoundary + 1, @@ -1332,8 +1332,8 @@ TEST(RmqSegmentBTreeXLPlusMinusOne, DefaultBoundaryRanges) { TEST(RmqSegmentBTreeXLPlusMinusOne, Select0MatchesNaiveAcrossHighNodes) { using Rmq = pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq<>; constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kFanout = Rmq::kFanout; - std::vector depths(kLeaf * kFanout + 777); + constexpr std::size_t kHighFanout = Rmq::kHighLevelFanout; + std::vector depths(kLeaf * kHighFanout + 777); for (std::size_t i = 1; i < depths.size(); ++i) { const bool up = (i % 5 == 0) || (i % 17 == 3) || (i % 257 == 11); depths[i] = depths[i - 1] + (up ? 1 : -1); @@ -1363,13 +1363,13 @@ TEST(RmqSegmentBTreeXLPlusMinusOne, Select0MatchesNaiveAcrossHighNodes) { EXPECT_EQ(rmq.select0(zero_count + 1), Rmq::npos); } -TEST(RmqSegmentBTreeXLPlusMinusOne, SmallFanoutExercisesMiddleAndHighLevels) { +TEST(RmqSegmentBTreeXLPlusMinusOne, FixedFanoutsExerciseHighBoundary) { using Rmq = - pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq; - constexpr std::size_t kBoundary = - Rmq::kLeafSize * Rmq::kFanout * Rmq::kFanout; - std::vector depths(kBoundary + 3 * Rmq::kLeafSize + 17); + pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq; + static_assert(Rmq::kMiddleFanout == 192); + static_assert(Rmq::kHighLevelFanout == 256); + constexpr std::size_t kHighBoundary = Rmq::kLeafSize * Rmq::kHighLevelFanout; + std::vector depths(kHighBoundary + 3 * Rmq::kLeafSize + 17); for (std::size_t i = 1; i < depths.size(); ++i) { const bool up = (i % 13 == 0) || (i % 17 == 1) || (i % 29 == 7); depths[i] = depths[i - 1] + (up ? 1 : -1); @@ -1379,10 +1379,10 @@ TEST(RmqSegmentBTreeXLPlusMinusOne, SmallFanoutExercisesMiddleAndHighLevels) { const Rmq rmq(bits, depths.size()); const std::vector> ranges = { {0, depths.size()}, - {kBoundary - 2, kBoundary + 2}, - {kBoundary - Rmq::kLeafSize - 3, kBoundary + Rmq::kLeafSize + 5}, - {Rmq::kLeafSize, kBoundary + 17}, - {kBoundary + 1, depths.size()}, + {kHighBoundary - 2, kHighBoundary + 2}, + {kHighBoundary - Rmq::kLeafSize - 3, kHighBoundary + Rmq::kLeafSize + 5}, + {Rmq::kLeafSize, kHighBoundary + 17}, + {kHighBoundary + 1, depths.size()}, {depths.size() - Rmq::kLeafSize, depths.size()}, }; check_depth_ranges( From 0f7e6b9b023fc9d373e5cd50333d876363a71c2c Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Mon, 15 Jun 2026 14:40:48 +0300 Subject: [PATCH 18/27] Build optimizations --- CMakePresets.json | 3 + README.md | 23 +- .../optimization-experiment/EXAMPLES.md | 146 +- include/pixie/bitvector.h | 336 +++- include/pixie/cache_line.h | 37 +- include/pixie/experimental/rmm_btree.h | 394 +++- include/pixie/rmq.h | 230 +-- include/pixie/rmq/bp_plus_minus_one_rmq.h | 633 ------- ...tree_xl_rmq.h => cartesian_hybrid_btree.h} | 190 +- ...n_tree_rmm_btree_rmq.h => cartesian_rmm.h} | 229 ++- include/pixie/rmq/cartesian_tree_rmq.h | 316 ---- .../rmq/experimental/node_euler_btree_rmq.h | 1344 -------------- .../{segment_btree_xl.h => hybrid_btree.h} | 560 +++--- include/pixie/rmq/node_euler_btree_rmq.h | 1016 ----------- include/pixie/rmq/rmq_one_interval_btree.h | 833 --------- .../pixie/rmq/{sdsl_sct_rmq.h => sdsl_sct.h} | 10 +- .../pixie/rmq/utils/succinct_monotone_stack.h | 213 +++ src/benchmarks/bench_rmq.cpp | 1551 ++++------------ src/tests/rmq_tests.cpp | 1600 ++++------------- src/tests/test_rmm.cpp | 16 + src/tests/unittests.cpp | 67 + 21 files changed, 2281 insertions(+), 7466 deletions(-) delete mode 100644 include/pixie/rmq/bp_plus_minus_one_rmq.h rename include/pixie/rmq/{experimental/cartesian_tree_segment_btree_xl_rmq.h => cartesian_hybrid_btree.h} (91%) rename include/pixie/rmq/{experimental/cartesian_tree_rmm_btree_rmq.h => cartesian_rmm.h} (55%) delete mode 100644 include/pixie/rmq/cartesian_tree_rmq.h delete mode 100644 include/pixie/rmq/experimental/node_euler_btree_rmq.h rename include/pixie/rmq/{segment_btree_xl.h => hybrid_btree.h} (76%) delete mode 100644 include/pixie/rmq/node_euler_btree_rmq.h delete mode 100644 include/pixie/rmq/rmq_one_interval_btree.h rename include/pixie/rmq/{sdsl_sct_rmq.h => sdsl_sct.h} (92%) create mode 100644 include/pixie/rmq/utils/succinct_monotone_stack.h diff --git a/CMakePresets.json b/CMakePresets.json index 5da7763..becff7a 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -138,6 +138,7 @@ "targets": [ "benchmarks", "bench_rmm", + "bench_rmq", "louds_tree_benchmarks", "dfuds_tree_benchmarks", "alignment_comparison", @@ -152,6 +153,7 @@ "benchmarks", "bench_rmm", "bench_rmm_sdsl", + "bench_rmq", "louds_tree_benchmarks", "dfuds_tree_benchmarks", "alignment_comparison", @@ -165,6 +167,7 @@ "targets": [ "benchmarks", "bench_rmm", + "bench_rmq", "louds_tree_benchmarks", "dfuds_tree_benchmarks", "alignment_comparison", diff --git a/README.md b/README.md index 48ae472..56ad227 100644 --- a/README.md +++ b/README.md @@ -214,22 +214,39 @@ The RMQ benchmark harness rotates through several value arrays so results are less dependent on the global-minimum position. The `width` argument is the maximum query width, not an exact width. -To compare the new backend with `SegmentBTreeXL`, run both benchmark families +To compare the new backend with `HybridBTree`, run both benchmark families with a Google Benchmark filter. For example, after registering the new backend as `rmq_linear` and `rmq_build_linear`: ```sh ./build/release/bench_rmq \ - --benchmark_filter='^(rmq_linear|rmq_segment_btree_xl)/(4194304|16777216)/(64|4096|262144|4194304|16777216)$' + --benchmark_filter='^(rmq_linear|rmq_hybrid_btree)/(4194304|16777216)/(64|4096|262144|4194304|16777216)$' ./build/release/bench_rmq \ - --benchmark_filter='^(rmq_build_linear|rmq_build_segment_btree_xl)/(262144|4194304|16777216)$' + --benchmark_filter='^(rmq_build_linear|rmq_build_hybrid_btree)/(262144|4194304|16777216)$' ``` The first command compares query time for `2^22` and `2^24` input sizes across the common RMQ widths. The second command compares construction time for the same implementations. +For hardware counters, use the diagnostic preset, which builds Google Benchmark +with libpfm support: + +```sh +cmake --preset benchmarks-diagnostic +cmake --build --preset benchmarks-diagnostic -j + +./build/release-with-deb/bench_rmq \ + --benchmark_filter='rmq_cartesian_hybrid_btree/67108864/4096' \ + --benchmark_perf_counters=CYCLES,INSTRUCTIONS,CACHE-MISSES \ + --benchmark_counters_tabular=true +``` + +Counter names are platform/libpfm dependent. Google Benchmark pauses timing and +perf counters during `state.PauseTiming()`, so RMQ dataset-variant rebuilds are +excluded from query counter rows. + ### RmM Tree ```sh diff --git a/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md b/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md index 4c2e3bb..6f4854c 100644 --- a/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md +++ b/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md @@ -68,98 +68,98 @@ RMQ benchmark snapshot, 2026-06-11. Query command: taskset -c 0 ./build/release/bench_rmq - --benchmark_filter='^(rmq_sparse_table|rmq_segment_tree|rmq_segment_btree_xl|rmq_cartesian_tree|rmq_cartesian_tree_segment_btree_xl|rmq_sdsl_sct)/' + --benchmark_filter='^(rmq_sparse_table|rmq_segment_tree|rmq_hybrid_btree|rmq_cartesian_rmm|rmq_cartesian_hybrid_btree|rmq_sdsl_sct)/' --benchmark_min_time=0.25s Build/memory command: taskset -c 0 ./build/release/bench_rmq - --benchmark_filter='^(rmq_build_sparse_table|rmq_build_segment_tree|rmq_build_segment_btree_xl|rmq_build_cartesian_tree|rmq_build_cartesian_tree_segment_btree_xl|rmq_build_sdsl_sct)/' + --benchmark_filter='^(rmq_build_sparse_table|rmq_build_segment_tree|rmq_build_hybrid_btree|rmq_build_cartesian_rmm|rmq_build_cartesian_hybrid_btree|rmq_build_sdsl_sct)/' --benchmark_min_time=0.20s Query CPU time. "max width" is the benchmark's maximum sampled query width. -Sparse-table rows are intentionally not registered above 2^22. "cartesian xl" -is the experimental `rmq_cartesian_tree_segment_btree_xl` row. - -| N | max width | sparse table (ns) | segment tree (ns) | segment btree xl (ns) | cartesian tree (ns) | cartesian xl (ns) | sdsl sct (ns) | -|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^10 | 64 | 10.1 | 35.9 | 34.6 | 72.7 | 66.4 | 111.2 | -| 2^10 | 2^10 | 13.1 | 58.8 | 41.0 | 90.0 | 118.6 | 231.3 | -|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^14 | 64 | 14.7 | 43.6 | 33.5 | 86.3 | 67.3 | 125.4 | -| 2^14 | 4096 | 15.9 | 76.7 | 38.1 | 108.0 | 66.3 | 430.8 | -| 2^14 | 2^14 | 15.8 | 93.5 | 33.7 | 103.6 | 49.2 | 532.1 | -|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^18 | 64 | 63.0 | 103.7 | 50.9 | 87.0 | 74.7 | 141.3 | -| 2^18 | 4096 | 52.2 | 183.0 | 52.5 | 114.2 | 72.6 | 586.7 | -| 2^18 | 2^18 | 40.9 | 299.1 | 46.0 | 148.5 | 27.6 | 791.0 | -|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^22 | 64 | 97.2 | 169.1 | 137.4 | 164.7 | 256.9 | 263.2 | -| 2^22 | 4096 | 93.4 | 298.7 | 101.9 | 236.1 | 251.9 | 726.7 | -| 2^22 | 2^18 | 68.0 | 437.3 | 82.2 | 344.1 | 86.4 | 1053.0 | -| 2^22 | 2^22 | 60.7 | 485.3 | 45.5 | 225.6 | 22.3 | 1129.7 | -|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^24 | 64 | - | 208.0 | 166.6 | 326.0 | 480.4 | 549.6 | -| 2^24 | 4096 | - | 357.2 | 222.2 | 755.7 | 535.4 | 864.8 | -| 2^24 | 2^18 | - | 472.3 | 143.0 | 734.9 | 83.5 | 2824.2 | -| 2^24 | 2^22 | - | 510.7 | 47.1 | 403.5 | 25.3 | 1169.8 | -| 2^24 | 2^24 | - | 546.5 | 43.4 | 479.6 | 20.1 | 1239.6 | -|-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^26 | 64 | - | 273.5 | 342.3 | 539.2 | 567.5 | 710.4 | -| 2^26 | 4096 | - | 938.4 | 308.9 | 566.5 | 688.5 | 1701.6 | -| 2^26 | 2^18 | - | 802.5 | 175.6 | 664.8 | 195.1 | 2609.4 | -| 2^26 | 2^22 | - | 1690.7 | 64.0 | 625.2 | 45.2 | 2387.0 | -| 2^26 | 2^26 | - | 1003.7 | 51.0 | 588.9 | 24.1 | 2152.0 | +Sparse-table rows are intentionally not registered above 2^22. "cartesian hybrid" +is the `rmq_cartesian_hybrid_btree` row. + +| N | max width | sparse table (ns) | segment tree (ns) | hybrid btree (ns) | cartesian hybrid (ns) | sdsl sct (ns) | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^10 | 64 | 10.1 | 35.9 | 34.6 | 66.4 | 111.2 | +| 2^10 | 2^10 | 13.1 | 58.8 | 41.0 | 118.6 | 231.3 | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^14 | 64 | 14.7 | 43.6 | 33.5 | 67.3 | 125.4 | +| 2^14 | 4096 | 15.9 | 76.7 | 38.1 | 66.3 | 430.8 | +| 2^14 | 2^14 | 15.8 | 93.5 | 33.7 | 49.2 | 532.1 | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^18 | 64 | 63.0 | 103.7 | 50.9 | 74.7 | 141.3 | +| 2^18 | 4096 | 52.2 | 183.0 | 52.5 | 72.6 | 586.7 | +| 2^18 | 2^18 | 40.9 | 299.1 | 46.0 | 27.6 | 791.0 | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^22 | 64 | 97.2 | 169.1 | 137.4 | 256.9 | 263.2 | +| 2^22 | 4096 | 93.4 | 298.7 | 101.9 | 251.9 | 726.7 | +| 2^22 | 2^18 | 68.0 | 437.3 | 82.2 | 86.4 | 1053.0 | +| 2^22 | 2^22 | 60.7 | 485.3 | 45.5 | 22.3 | 1129.7 | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^24 | 64 | - | 208.0 | 166.6 | 480.4 | 549.6 | +| 2^24 | 4096 | - | 357.2 | 222.2 | 535.4 | 864.8 | +| 2^24 | 2^18 | - | 472.3 | 143.0 | 83.5 | 2824.2 | +| 2^24 | 2^22 | - | 510.7 | 47.1 | 25.3 | 1169.8 | +| 2^24 | 2^24 | - | 546.5 | 43.4 | 20.1 | 1239.6 | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^26 | 64 | - | 273.5 | 342.3 | 567.5 | 710.4 | +| 2^26 | 4096 | - | 938.4 | 308.9 | 688.5 | 1701.6 | +| 2^26 | 2^18 | - | 802.5 | 175.6 | 195.1 | 2609.4 | +| 2^26 | 2^22 | - | 1690.7 | 64.0 | 45.2 | 2387.0 | +| 2^26 | 2^26 | - | 1003.7 | 51.0 | 24.1 | 2152.0 | Build CPU time. Sparse-table build rows are intentionally not registered above 2^22. -| N | sparse table (ms) | segment tree (ms) | segment btree xl (ms) | cartesian tree (ms) | cartesian xl (ms) | sdsl sct (ms) | -|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^10 | 0.005 | 0.001 | 0.001 | 0.008 | 0.009 | 0.007 | -|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^14 | 1.096 | 0.015 | 0.018 | 0.045 | 0.058 | 0.162 | -|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^18 | 28.405 | 0.292 | 0.301 | 1.686 | 1.985 | 2.484 | -|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^22 | 860.619 | 92.812 | 6.077 | 29.128 | 31.552 | 40.262 | -|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^24 | - | 426.733 | 21.802 | 166.533 | 124.868 | 158.091 | -|-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| -| 2^26 | - | 1432.849 | 86.641 | 852.740 | 518.439 | 647.360 | +| N | sparse table (ms) | segment tree (ms) | hybrid btree (ms) | cartesian hybrid (ms) | sdsl sct (ms) | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^10 | 0.005 | 0.001 | 0.001 | 0.009 | 0.007 | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^14 | 1.096 | 0.015 | 0.018 | 0.058 | 0.162 | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^18 | 28.405 | 0.292 | 0.301 | 1.985 | 2.484 | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^22 | 860.619 | 92.812 | 6.077 | 31.552 | 40.262 | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^24 | - | 426.733 | 21.802 | 124.868 | 158.091 | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^26 | - | 1432.849 | 86.641 | 518.439 | 647.360 | Owned auxiliary memory. Benchmarks use 64-bit signed integer values and 64-bit indexes. The external input values are not owned by RMQ indexes and are excluded. -| N | sparse table (MiB) | segment tree (MiB) | segment btree xl (MiB) | cartesian tree (MiB) | cartesian xl (MiB) | sdsl sct (MiB) | -|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| -| 2^10 | 0.071 | 0.016 | 0.009 | 0.004 | 0.016 | 0.001 | -|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| -| 2^14 | 1.626 | 0.250 | 0.011 | 0.025 | 0.022 | 0.005 | -|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| -| 2^18 | 34.001 | 4.000 | 0.065 | 0.474 | 0.186 | 0.079 | -|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| -| 2^22 | 672.001 | 64.000 | 0.801 | 9.539 | 2.632 | 1.262 | -|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| -| 2^24 | - | 256.000 | 3.155 | 42.148 | 7.525 | 5.051 | -|-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| -| 2^26 | - | 1024.000 | 8.340 | 184.582 | 30.530 | 20.224 | +| N | sparse table (MiB) | segment tree (MiB) | hybrid btree (MiB) | cartesian hybrid (MiB) | sdsl sct (MiB) | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^10 | 0.071 | 0.016 | 0.009 | 0.016 | 0.001 | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^14 | 1.626 | 0.250 | 0.011 | 0.022 | 0.005 | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^18 | 34.001 | 4.000 | 0.065 | 0.186 | 0.079 | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^22 | 672.001 | 64.000 | 0.801 | 2.632 | 1.262 | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^24 | - | 256.000 | 3.155 | 7.525 | 5.051 | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^26 | - | 1024.000 | 8.340 | 30.530 | 20.224 | Owned auxiliary memory normalized by indexed value count (bits/value). -| N | sparse table | segment tree | segment btree xl | cartesian tree | cartesian xl | sdsl sct | -|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| -| 2^10 | 579.188 | 128.438 | 71.438 | 30.750 | 127.500 | 7.312 | -|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| -| 2^14 | 832.262 | 128.027 | 5.434 | 12.562 | 11.102 | 2.770 | -|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| -| 2^18 | 1088.020 | 128.002 | 2.089 | 15.175 | 5.957 | 2.534 | -|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| -| 2^22 | 1344.002 | 128.000 | 1.603 | 19.079 | 5.264 | 2.524 | -|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| -| 2^24 | - | 128.000 | 1.577 | 21.074 | 3.763 | 2.526 | -|-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| -| 2^26 | - | 128.000 | 1.042 | 23.073 | 3.816 | 2.528 | +| N | sparse table | segment tree | hybrid btree | cartesian hybrid | sdsl sct | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^10 | 579.188 | 128.438 | 71.438 | 127.500 | 7.312 | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^14 | 832.262 | 128.027 | 5.434 | 11.102 | 2.770 | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^18 | 1088.020 | 128.002 | 2.089 | 5.957 | 2.534 | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^22 | 1344.002 | 128.000 | 1.603 | 5.264 | 2.524 | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^24 | - | 128.000 | 1.577 | 3.763 | 2.526 | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^26 | - | 128.000 | 1.042 | 3.816 | 2.528 | ``` ## Documentation Pattern diff --git a/include/pixie/bitvector.h b/include/pixie/bitvector.h index 28840a4..ab6427f 100644 --- a/include/pixie/bitvector.h +++ b/include/pixie/bitvector.h @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include @@ -45,7 +47,8 @@ namespace pixie { * overhead). * - Basic blocks of 512 bits with 16-bit ranks (~3.125% * overhead). - * - Select samples every 16384 bits (~0.39% overhead). + * - Optional select samples every 16384 bits for 1-bits, 0-bits, or both + * (~0.39% overhead per enabled direction). * * * Rank: 2 table lookups plus SIMD popcount in the 512-bit block. @@ -58,9 +61,21 @@ namespace pixie { * - SIMD linear scan to find the basic block. * * This variant does - * not interleave data and index, favoring simpler scans. + * not interleave data and index, favoring simpler scans. Rank metadata and + * enabled select samples are built in one scan over the source words. */ class BitVector { + public: + /** + * @brief Select directions to index during construction. + */ + enum class SelectSupport : uint8_t { + kNone = 0, + kSelect1 = 1, + kSelect0 = 2, + kBoth = 3, + }; + private: constexpr static size_t kWordSize = 64; constexpr static size_t kSuperBlockRankIntSize = 64; @@ -76,14 +91,29 @@ class BitVector { AlignedStorage super_block_rank_; // 64-bit global prefix sums AlignedStorage basic_block_rank_; // 16-bit local prefix sums - AlignedStorage select1_samples_; // 64-bit global positions - AlignedStorage select0_samples_; // 64-bit global positions + AlignedStorage select_samples_; // 64-bit global positions size_t num_bits_{}; size_t padded_size_{}; size_t max_rank_{}; + size_t select1_sample_begin_{}; + size_t select1_sample_count_{}; + size_t select0_sample_begin_{}; + size_t select0_sample_count_{}; + SelectSupport select_support_ = SelectSupport::kNone; + bool select0_samples_reversed_ = false; std::span bits_; + static bool builds_select1(SelectSupport support) { + return (static_cast(support) & + static_cast(SelectSupport::kSelect1)) != 0; + } + + static bool builds_select0(SelectSupport support) { + return (static_cast(support) & + static_cast(SelectSupport::kSelect0)) != 0; + } + size_t logical_word_count() const { return (num_bits_ + kWordSize - 1) / kWordSize; } @@ -157,10 +187,160 @@ class BitVector { return num_bits_; } + static size_t select_sample_count_for_rank(size_t rank_count) { + return 1 + rank_count / kSelectSampleFrequency; + } + + static size_t select_sample_upper_bound(size_t bit_count) { + return select_sample_count_for_rank(bit_count); + } + + struct SelectSampleWriter { + std::span words; + size_t next = 0; + size_t count = 0; + size_t capacity = 0; + bool enabled = false; + bool reversed = false; + + SelectSampleWriter() = default; + + SelectSampleWriter(std::span words, + size_t begin, + size_t capacity, + bool enabled, + bool reversed) + : words(words), + next(begin), + capacity(capacity), + enabled(enabled), + reversed(reversed) {} + + void append(uint64_t sample) { + if (!enabled) { + return; + } + if (count >= capacity) [[unlikely]] { + throw std::invalid_argument( + "BitVector one_count hint is inconsistent with input bits"); + } + words[next] = sample; + ++count; + if (reversed) { + if (next != 0) { + --next; + } + } else { + ++next; + } + } + }; + + struct SelectSampleWriters { + SelectSampleWriter ones; + SelectSampleWriter zeros; + bool shrink_after_build = false; + }; + + SelectSampleWriters initialize_select_sample_writers( + bool need_select1, + bool need_select0, + std::optional one_count) { + select1_sample_begin_ = 0; + select1_sample_count_ = 0; + select0_sample_begin_ = 0; + select0_sample_count_ = 0; + select0_samples_reversed_ = false; + select_samples_.resize(0); + + SelectSampleWriters writers; + if (!need_select1 && !need_select0) { + return writers; + } + + const std::optional zero_count = + one_count ? std::optional(num_bits_ - *one_count) + : std::nullopt; + if (need_select1 && need_select0) { + const size_t one_sample_capacity = + one_count ? select_sample_count_for_rank(*one_count) + : 2 + num_bits_ / kSelectSampleFrequency; + const size_t zero_sample_capacity = + zero_count ? select_sample_count_for_rank(*zero_count) + : 2 + num_bits_ / kSelectSampleFrequency; + const size_t total_samples = + one_count ? one_sample_capacity + zero_sample_capacity + : 2 + num_bits_ / kSelectSampleFrequency; + select_samples_.resize(total_samples * kWordSize); + auto samples = select_samples_.As64BitInts(); + select1_sample_begin_ = 0; + select0_samples_reversed_ = true; + writers.ones = + SelectSampleWriter(samples, 0, one_sample_capacity, true, false); + writers.zeros = SelectSampleWriter(samples, total_samples - 1, + zero_sample_capacity, true, true); + writers.ones.append(0); + writers.zeros.append(0); + return writers; + } + + const size_t sample_capacity = + need_select1 ? (one_count ? select_sample_count_for_rank(*one_count) + : select_sample_upper_bound(num_bits_)) + : (zero_count ? select_sample_count_for_rank(*zero_count) + : select_sample_upper_bound(num_bits_)); + select_samples_.resize(sample_capacity * kWordSize); + auto samples = select_samples_.As64BitInts(); + writers.shrink_after_build = !one_count; + if (need_select1) { + select1_sample_begin_ = 0; + writers.ones = + SelectSampleWriter(samples, 0, sample_capacity, true, false); + writers.ones.append(0); + } else { + select0_sample_begin_ = 0; + writers.zeros = + SelectSampleWriter(samples, 0, sample_capacity, true, false); + writers.zeros.append(0); + } + return writers; + } + + void finalize_select_sample_writers(SelectSampleWriters writers) { + select1_sample_count_ = writers.ones.count; + select0_sample_count_ = writers.zeros.count; + if (writers.zeros.reversed) { + select0_sample_begin_ = writers.zeros.next + 1; + } + if (writers.shrink_after_build) { + const size_t sample_count = select1_sample_count_ != 0 + ? select1_sample_count_ + : select0_sample_count_; + select_samples_.resize(sample_count * kWordSize); + select_samples_.shrink_to_fit(); + } + } + + uint64_t select1_sample(size_t sample_index) const { + auto samples = select_samples_.AsConst64BitInts(); + return samples[select1_sample_begin_ + sample_index]; + } + + uint64_t select0_sample(size_t sample_index) const { + auto samples = select_samples_.AsConst64BitInts(); + if (select0_samples_reversed_) { + return samples[select0_sample_begin_ + select0_sample_count_ - 1 - + sample_index]; + } + return samples[select0_sample_begin_ + sample_index]; + } + /** - * @brief Precompute rank for fast queries. + * @brief Precompute rank and requested select samples in one word scan. */ - void build_rank() { + void build_rank_select(SelectSupport support, + std::optional one_count) { + select_support_ = support; size_t num_superblocks = 8 + (padded_size_ == 0 ? 0 : (padded_size_ - 1) / kSuperBlockSize); // Add more blocks to ease SIMD processing @@ -174,8 +354,21 @@ class BitVector { auto super_block_rank = super_block_rank_.As64BitInts(); auto basic_block_rank = basic_block_rank_.As16BitInts(); + const bool need_select1 = builds_select1(support); + const bool need_select0 = builds_select0(support); + if (one_count && *one_count > num_bits_) { + throw std::invalid_argument( + "BitVector one_count hint cannot exceed num_bits"); + } + auto select_writers = + initialize_select_sample_writers(need_select1, need_select0, one_count); + uint64_t super_block_sum = 0; uint64_t basic_block_sum = 0; + uint64_t milestone = kSelectSampleFrequency; + uint64_t milestone0 = kSelectSampleFrequency; + uint64_t rank = 0; + uint64_t rank0 = 0; for (size_t i = 0; i / kBasicBlockSize < basic_block_rank.size(); i += kWordSize) { @@ -189,58 +382,32 @@ class BitVector { static_cast(basic_block_sum); } if (i / kWordSize < logical_word_count()) { - basic_block_sum += std::popcount(logical_word(i / kWordSize)); + const size_t word_index = i / kWordSize; + const uint64_t word = logical_word(word_index); + const size_t word_bits = logical_word_bits(word_index); + const uint64_t ones = std::popcount(word); + const uint64_t zeros = word_bits - ones; + if (need_select1 && rank + ones >= milestone) { + const auto pos = select_64(word, milestone - rank - 1); + // TODO: try including global rank into select samples to save + // a cache miss on global rank scan + select_writers.ones.append((64 * word_index + pos) / kSuperBlockSize); + milestone += kSelectSampleFrequency; + } + if (need_select0 && rank0 + zeros >= milestone0) { + const uint64_t zero_word = ~word & first_bits_mask(word_bits); + const auto pos = select_64(zero_word, milestone0 - rank0 - 1); + select_writers.zeros.append((64 * word_index + pos) / + kSuperBlockSize); + milestone0 += kSelectSampleFrequency; + } + basic_block_sum += ones; + rank += ones; + rank0 += zeros; } } max_rank_ = super_block_sum + basic_block_sum; - } - - /** - * @brief Calculate select samples. - */ - void build_select() { - uint64_t milestone = kSelectSampleFrequency; - uint64_t milestone0 = kSelectSampleFrequency; - uint64_t rank = 0; - uint64_t rank0 = 0; - - size_t num_one_samples = - 1 + (max_rank_ + kSelectSampleFrequency - 1) / kSelectSampleFrequency; - size_t num_zero_samples = - 1 + (num_bits_ - max_rank_ + kSelectSampleFrequency - 1) / - kSelectSampleFrequency; - - select1_samples_.resize(num_one_samples * 64); - select0_samples_.resize(num_zero_samples * 64); - auto select1_samples = select1_samples_.As64BitInts(); - auto select0_samples = select0_samples_.As64BitInts(); - - select1_samples[0] = 0; - select0_samples[0] = 0; - - size_t num_zeros = 1, num_ones = 1; - - for (size_t i = 0; i < logical_word_count(); ++i) { - const uint64_t word = logical_word(i); - const auto ones = std::popcount(word); - const auto zeros = logical_word_bits(i) - ones; - if (rank + ones >= milestone) { - auto pos = select_64(word, milestone - rank - 1); - // TODO: try including global rank into select samples to save - // a cache miss on global rank scan - select1_samples[num_ones++] = (64 * i + pos) / kSuperBlockSize; - milestone += kSelectSampleFrequency; - } - if (rank0 + zeros >= milestone0) { - const uint64_t zero_word = - ~word & first_bits_mask(logical_word_bits(i)); - auto pos = select_64(zero_word, milestone0 - rank0 - 1); - select0_samples[num_zeros++] = (64 * i + pos) / kSuperBlockSize; - milestone0 += kSelectSampleFrequency; - } - rank += ones; - rank0 += zeros; - } + finalize_select_sample_writers(select_writers); for (size_t i = 0; i < 8; ++i) { delta_super[i] = i * kSuperBlockSize; @@ -256,10 +423,9 @@ class BitVector { * rank of the 1-bit to locate. */ uint64_t find_superblock(uint64_t rank) const { - auto select1_samples = select1_samples_.AsConst64BitInts(); auto super_block_rank = super_block_rank_.AsConst64BitInts(); - uint64_t left = select1_samples[rank / kSelectSampleFrequency]; + uint64_t left = select1_sample(rank / kSelectSampleFrequency); while (left + 7 < super_block_rank.size()) { auto len = lower_bound_8x64(&super_block_rank[left], rank); @@ -287,10 +453,9 @@ class BitVector { * rank of the 0-bit to locate. */ uint64_t find_superblock_zeros(uint64_t rank0) const { - auto select0_samples = select0_samples_.AsConst64BitInts(); auto super_block_rank = super_block_rank_.AsConst64BitInts(); - uint64_t left = select0_samples[rank0 / kSelectSampleFrequency]; + uint64_t left = select0_sample(rank0 / kSelectSampleFrequency); while (left + 7 < super_block_rank.size()) { auto len = lower_bound_delta_8x64(&super_block_rank[left], rank0, @@ -487,11 +652,11 @@ class BitVector { result.source_bitvector_bytes = (num_bits_ + 7) / 8; result.super_block_rank_bytes = super_block_rank_.AsConstBytes().size(); result.basic_block_rank_bytes = basic_block_rank_.AsConstBytes().size(); - result.select1_samples_bytes = select1_samples_.AsConstBytes().size(); - result.select0_samples_bytes = select0_samples_.AsConstBytes().size(); - result.total_bytes = - result.super_block_rank_bytes + result.basic_block_rank_bytes + - result.select1_samples_bytes + result.select0_samples_bytes; + result.select1_samples_bytes = select1_sample_count_ * sizeof(uint64_t); + result.select0_samples_bytes = select0_sample_count_ * sizeof(uint64_t); + result.total_bytes = result.super_block_rank_bytes + + result.basic_block_rank_bytes + + select_samples_.AsConstBytes().size(); return result; } @@ -523,13 +688,19 @@ class BitVector { * bit_vector Backing data, not owned. * @param num_bits Number of valid * bits in the vector. + * @param select_support Which select sample tables to build. Rank and rank0 + * remain available in all modes. + * @param one_count Optional exact number of 1-bits. When supplied, + * construction can allocate select sample storage exactly. */ - explicit BitVector(std::span bit_vector, size_t num_bits) + explicit BitVector(std::span bit_vector, + size_t num_bits, + SelectSupport select_support = SelectSupport::kBoth, + std::optional one_count = std::nullopt) : num_bits_(std::min(num_bits, bit_vector.size() * kWordSize)), padded_size_(((num_bits_ + kWordSize - 1) / kWordSize) * kWordSize), bits_(bit_vector) { - build_rank(); - build_select(); + build_rank_select(select_support, one_count); } /** @@ -537,6 +708,16 @@ class BitVector { */ size_t size() const { return num_bits_; } + /** + * @brief Whether this index stores samples for select1 queries. + */ + bool supports_select1() const { return builds_select1(select_support_); } + + /** + * @brief Whether this index stores samples for select0 queries. + */ + bool supports_select0() const { return builds_select0(select_support_); } + /** * @brief Return owned auxiliary memory usage in bytes. * @@ -546,8 +727,7 @@ class BitVector { size_t memory_usage_bytes() const { return sizeof(*this) + super_block_rank_.allocated_bytes() + basic_block_rank_.allocated_bytes() + - select1_samples_.allocated_bytes() + - select0_samples_.allocated_bytes(); + select_samples_.allocated_bytes(); } /** @@ -606,12 +786,15 @@ class BitVector { * size() if rank is out of range. */ uint64_t select(size_t rank) const { - if (rank > max_rank_) [[unlikely]] { - return num_bits_; - } if (rank == 0) [[unlikely]] { return 0; } + if (!supports_select1()) [[unlikely]] { + return num_bits_; + } + if (rank > max_rank_) [[unlikely]] { + return num_bits_; + } auto super_block_rank = super_block_rank_.AsConst64BitInts(); auto basic_block_rank = basic_block_rank_.AsConst16BitInts(); @@ -630,12 +813,15 @@ class BitVector { * or size() if rank0 is out of range. */ uint64_t select0(size_t rank0) const { - if (rank0 > num_bits_ - max_rank_) [[unlikely]] { - return num_bits_; - } if (rank0 == 0) [[unlikely]] { return 0; } + if (!supports_select0()) [[unlikely]] { + return num_bits_; + } + if (rank0 > num_bits_ - max_rank_) [[unlikely]] { + return num_bits_; + } auto super_block_rank = super_block_rank_.AsConst64BitInts(); auto basic_block_rank = basic_block_rank_.AsConst16BitInts(); diff --git a/include/pixie/cache_line.h b/include/pixie/cache_line.h index dcae1f7..f027df4 100644 --- a/include/pixie/cache_line.h +++ b/include/pixie/cache_line.h @@ -55,24 +55,39 @@ class AlignedStorage { */ void resize(std::size_t bits) { data_.resize(LinesForBits(bits)); } - /** @brief Padded storage capacity in bits. */ + /** + * @brief Padded storage capacity in bits. + */ std::size_t capacity_bits() const { return data_.size() * kAlignedStorageLineBits; } - /** @brief Padded storage capacity in bytes. */ + /** + * @brief Padded storage capacity in bytes. + */ std::size_t capacity_bytes() const { return data_.size() * kAlignedStorageLineBytes; } - /** @brief Bytes reserved by the underlying aligned cache-line buffer. */ + /** + * @brief Bytes reserved by the underlying aligned cache-line buffer. + */ std::size_t allocated_bytes() const { return data_.capacity() * kAlignedStorageLineBytes; } - /** @brief Mutable view as cache lines. */ + /** + * @brief Request that reserved storage be reduced to the current size. + */ + void shrink_to_fit() { data_.shrink_to_fit(); } + + /** + * @brief Mutable view as cache lines. + */ std::span AsLines() { return data_; } - /** @brief Const view as cache lines. */ + /** + * @brief Const view as cache lines. + */ std::span AsConstLines() const { return data_; } /** @@ -84,7 +99,9 @@ class AlignedStorage { data_.size() * kAlignedStorageLineWords64); } - /** @brief Const view as 64-bit words. */ + /** + * @brief Const view as 64-bit words. + */ std::span AsConst64BitInts() const { return std::span( reinterpret_cast(data_.data()), @@ -96,7 +113,9 @@ class AlignedStorage { */ std::span AsBytes() { return std::as_writable_bytes(AsLines()); } - /** @brief Const view as bytes. */ + /** + * @brief Const view as bytes. + */ std::span AsConstBytes() const { return std::as_bytes(AsConstLines()); } @@ -110,7 +129,9 @@ class AlignedStorage { data_.size() * kAlignedStorageLineWords16); } - /** @brief Const view as 16-bit words. */ + /** + * @brief Const view as 16-bit words. + */ std::span AsConst16BitInts() const { return std::span( reinterpret_cast(data_.data()), diff --git a/include/pixie/experimental/rmm_btree.h b/include/pixie/experimental/rmm_btree.h index 1d83a70..e44a7e4 100644 --- a/include/pixie/experimental/rmm_btree.h +++ b/include/pixie/experimental/rmm_btree.h @@ -1,5 +1,36 @@ #pragma once +/** + * RmMBTree construction-summary experiment, 2026-06-13. + * + * Change under test: full aligned 512-bit block summaries use existing 128-bit + * excess kernels from bits.h instead of scanning every bit. + * + * Command shape: + * taskset -c 0 ./build/release/bench_rmq + * --benchmark_filter='^rmq_build_(cartesian_rmm|cartesian_hybrid_btree|sdsl_sct)/(4194304|67108864)$' + * --benchmark_repetitions=5 + * + * CPU mean, milliseconds. + * + * | N | row | CPU ms | + * | ---: | ----------------------- | ------: | + * | 2^22 | CartesianRmM before | 54.407 | + * | 2^22 | CartesianRmM after | 44.731 | + * | 2^22 | CartesianHybrid control | 47.943 | + * | 2^22 | SdslSct control | 39.911 | + * | ---- | ----------------------- | ------- | + * | 2^26 | CartesianRmM before | 915.712 | + * | 2^26 | CartesianRmM after | 750.316 | + * | 2^26 | CartesianHybrid control | 772.157 | + * | 2^26 | SdslSct control | 642.540 | + * + * Perf profile note: before this change, RmMBTree::summarize_bits()/bit() + * accounted for about 14% of CartesianRmM build samples. After this change, + * those source lines disappear from the visible hotspot list; the replacement + * excess_min_128() work is about 3%. + */ + #include #include #include @@ -104,6 +135,14 @@ class RmMBTree : public RmMBase> { RmMBTree& operator=(const RmMBTree&) = default; RmMBTree& operator=(RmMBTree&&) noexcept = default; + /** + * @brief Minimum position and value returned by one range-min traversal. + */ + struct RangeMinQueryResult { + std::size_t position = npos; + int value = 0; + }; + /** * @brief Construct an RmM btree over an external bit-vector span. * @details The tree stores a non-owning view of @p words and builds its @@ -120,6 +159,22 @@ class RmMBTree : public RmMBase> { build(words, bit_count); } + /** + * @brief Construct with explicit rank/select support options. + * + * @details This keeps the normal RmMBTree API full-featured by default while + * allowing adapters that only need rank/select0 to skip select1 samples. + * @param one_count Optional exact number of 1-bits for exact select-sample + * allocation. + */ + RmMBTree(std::span words, + std::size_t bit_count, + BitVector::SelectSupport select_support, + std::optional one_count = std::nullopt, + std::size_t = kBlockBits) { + build(words, bit_count, select_support, one_count); + } + std::size_t size_impl() const { return bit_count_; } std::size_t rank1_impl(std::size_t end_position) const { @@ -276,6 +331,21 @@ class RmMBTree : public RmMBase> { return range_extreme_query_val(range_begin, range_end, true); } + /** + * @brief Return the first minimum position and value in one traversal. + * @details This is a concrete-type helper for adapters that need to compare + * a boundary prefix against the interior range minimum. Invalid ranges return + * `{npos, 0}`. + */ + RangeMinQueryResult range_min_query_result(std::size_t range_begin, + std::size_t range_end) const { + if (range_begin > range_end || range_end >= bit_count_) { + return {}; + } + const auto result = range_extreme_query(range_begin, range_end, true); + return {result.position, static_cast(result.value)}; + } + std::size_t range_max_query_pos_impl(std::size_t range_begin, std::size_t range_end) const { if (range_begin > range_end || range_end >= bit_count_) { @@ -460,6 +530,16 @@ class RmMBTree : public RmMBase> { * @p bit_count. */ void build(std::span words, std::size_t bit_count) { + build(words, bit_count, BitVector::SelectSupport::kBoth, std::nullopt); + } + + /** + * @brief Build with explicit rank/select support options. + */ + void build(std::span words, + std::size_t bit_count, + BitVector::SelectSupport select_support, + std::optional one_count = std::nullopt) { const std::size_t required_words = (bit_count + 63) / 64; if (words.size() < required_words) { throw std::invalid_argument( @@ -468,7 +548,7 @@ class RmMBTree : public RmMBase> { bits_ = words; bit_count_ = bit_count; - rank_index_.emplace(words, bit_count); + rank_index_.emplace(words, bit_count, select_support, one_count); block_count_ = (bit_count_ + kBlockBits - 1) / kBlockBits; std::vector block_summaries(block_count_); @@ -586,6 +666,13 @@ class RmMBTree : public RmMBase> { * @return Relative summary of `[begin, begin + length)`. */ Summary summarize_bits(std::size_t begin, std::size_t length) const { + if (length == kBlockBits && (begin % kBlockBits) == 0) { + const std::size_t block_index = begin / kBlockBits; + if (full_block_has_words(block_index)) { + return summarize_full_block(block_index); + } + } + Summary summary; summary.size_bits = length; if (length == 0) { @@ -614,6 +701,52 @@ class RmMBTree : public RmMBase> { return summary; } + /** + * @brief Summarize one complete 512-bit block with 128-bit excess kernels. + * @details Full Cartesian-BP construction blocks are aligned and contain all + * eight backing words. This fast path reuses the existing SIMD/minimum + * primitives from `bits.h` instead of reading every bit individually. + */ + Summary summarize_full_block(std::size_t block_index) const { + const std::uint64_t* block = bits_.data() + block_index * kBlockWords; + Summary summary; + for (std::size_t chunk = 0; chunk < kSearchChunkCount; ++chunk) { + summary = append(summary, + summarize_128_chunk(block + chunk * kSearchChunkWords)); + } + return summary; + } + + /** + * @brief Summarize one full 128-bit chunk relative to its first bit. + * @details Minima are found directly; maxima are minima of the bit-inverted + * chunk with the sign flipped. Minimum multiplicity is computed by the + * existing SIMD target-position kernel. + */ + static Summary summarize_128_chunk(const std::uint64_t* chunk) { + Summary summary; + summary.size_bits = kSearchChunkBits; + summary.ones = std::popcount(chunk[0]) + std::popcount(chunk[1]); + summary.block_excess = 2 * static_cast(summary.ones) - + static_cast(kSearchChunkBits); + + const ExcessResult minimum = excess_min_128(chunk, 1, kSearchChunkBits); + summary.min_excess = minimum.min_excess; + + const std::array inverted = {~chunk[0], + ~chunk[1]}; + const ExcessResult inverted_min = + excess_min_128(inverted.data(), 1, kSearchChunkBits); + summary.max_excess = -static_cast(inverted_min.min_excess); + + std::uint64_t minimum_positions[kSearchChunkWords]; + excess_positions_128(chunk, static_cast(summary.min_excess), + minimum_positions); + summary.min_count = std::popcount(minimum_positions[0]) + + std::popcount(minimum_positions[1]); + return summary; + } + /** * @brief Concatenate two relative summaries. * @details Produces the summary for `left || right`, translating right-side @@ -1413,8 +1546,8 @@ class RmMBTree : public RmMBase> { } struct NodeRef { - std::size_t level = 0; - std::size_t index = 0; + std::size_t level; + std::size_t index; }; static constexpr std::size_t kMaxCoverNodes = 512; @@ -1428,6 +1561,12 @@ class RmMBTree : public RmMBase> { std::size_t max_position = npos; }; + struct MinScanResult { + std::int64_t block_excess = 0; + std::int64_t min_value = std::numeric_limits::max(); + std::size_t min_position = npos; + }; + struct RangeExtremeResult { std::size_t position = npos; std::int64_t value = 0; @@ -1438,23 +1577,6 @@ class RmMBTree : public RmMBase> { std::uint64_t count = 0; }; - struct Cover { - std::array nodes{}; - std::size_t size = 0; - - /** - * @brief Append a cover node if the fixed-capacity buffer has room. - * @details Cover construction has a conservative fixed upper bound. Extra - * pushes are ignored instead of growing storage. - * @param node Node reference to append. - */ - void push(NodeRef node) { - if (size < nodes.size()) { - nodes[size++] = node; - } - } - }; - /** * @brief Return the position of the first range minimum or maximum. * @details Wrapper around `range_extreme_query` that extracts only the @@ -1483,7 +1605,78 @@ class RmMBTree : public RmMBase> { std::size_t range_end, bool find_min) const { return static_cast( - range_extreme_query(range_begin, range_end, find_min).value); + range_extreme_query_value(range_begin, range_end, find_min)); + } + + /** + * @brief Return only the minimum or maximum relative excess in a range. + * @details Uses the same decomposition as `range_extreme_query`, but skips + * the final descent needed to recover a bit position. + * @param range_begin Inclusive range start. + * @param range_end Inclusive range end. + * @param find_min True for minimum, false for maximum. + * @return Extreme relative excess value. + */ + std::int64_t range_extreme_query_value(std::size_t range_begin, + std::size_t range_end, + bool find_min) const { + std::int64_t value = 0; + std::int64_t best = find_min ? std::numeric_limits::max() + : std::numeric_limits::min(); + + auto consider_value = [&](std::int64_t candidate) { + if ((find_min && candidate < best) || (!find_min && candidate > best)) { + best = candidate; + } + }; + + const std::size_t range_end_exclusive = range_end + 1; + const std::size_t first_full_block = + (range_begin + kBlockBits - 1) / kBlockBits; + const std::size_t full_begin = + std::min(range_end_exclusive, first_full_block * kBlockBits); + if (range_begin < full_begin) { + if (find_min) { + const MinScanResult scan = scan_min_range(range_begin, full_begin); + consider_value(scan.min_value); + value += scan.block_excess; + } else { + const ScanResult scan = scan_range(range_begin, full_begin); + consider_value(scan.max_value); + value += scan.block_excess; + } + } + + const std::size_t last_full_block_exclusive = + range_end_exclusive / kBlockBits; + const std::size_t middle_begin = full_begin; + const std::size_t middle_end = + std::max(middle_begin, last_full_block_exclusive * kBlockBits); + if (middle_begin < middle_end) { + for_each_cover_node(middle_begin, middle_end, [&](NodeRef node) { + const Summary summary = summary_at(node.level, node.index); + consider_value(value + + (find_min ? summary.min_excess : summary.max_excess)); + value += summary.block_excess; + return true; + }); + } + + if (middle_end < range_end_exclusive) { + if (find_min) { + const MinScanResult scan = + scan_min_range(middle_end, range_end_exclusive); + consider_value(value + scan.min_value); + } else { + const ScanResult scan = scan_range(middle_end, range_end_exclusive); + consider_value(value + scan.max_value); + } + } + if (best == std::numeric_limits::max() || + best == std::numeric_limits::min()) { + return 0; + } + return best; } /** @@ -1521,10 +1714,15 @@ class RmMBTree : public RmMBase> { const std::size_t full_begin = std::min(range_end_exclusive, first_full_block * kBlockBits); if (range_begin < full_begin) { - const ScanResult scan = scan_range(range_begin, full_begin); - consider_point(find_min ? scan.min_value : scan.max_value, - find_min ? scan.min_position : scan.max_position); - value += scan.block_excess; + if (find_min) { + const MinScanResult scan = scan_min_range(range_begin, full_begin); + consider_point(scan.min_value, scan.min_position); + value += scan.block_excess; + } else { + const ScanResult scan = scan_range(range_begin, full_begin); + consider_point(scan.max_value, scan.max_position); + value += scan.block_excess; + } } const std::size_t last_full_block_exclusive = @@ -1533,11 +1731,8 @@ class RmMBTree : public RmMBase> { const std::size_t middle_end = std::max(middle_begin, last_full_block_exclusive * kBlockBits); if (middle_begin < middle_end) { - Cover cover; - collect_cover(middle_begin, middle_end, cover); - for (std::size_t i = 0; i < cover.size; ++i) { - const NodeRef& node = cover.nodes[i]; - Summary summary = summary_at(node.level, node.index); + for_each_cover_node(middle_begin, middle_end, [&](NodeRef node) { + const Summary summary = summary_at(node.level, node.index); const std::int64_t candidate = value + (find_min ? summary.min_excess : summary.max_excess); if ((find_min && candidate < best) || (!find_min && candidate > best)) { @@ -1547,15 +1742,19 @@ class RmMBTree : public RmMBase> { best_is_node = true; } value += summary.block_excess; - } + return true; + }); } if (middle_end < range_end_exclusive) { - const ScanResult scan = scan_range(middle_end, range_end_exclusive); - const std::int64_t candidate = - value + (find_min ? scan.min_value : scan.max_value); - consider_point(candidate, - find_min ? scan.min_position : scan.max_position); + if (find_min) { + const MinScanResult scan = + scan_min_range(middle_end, range_end_exclusive); + consider_point(value + scan.min_value, scan.min_position); + } else { + const ScanResult scan = scan_range(middle_end, range_end_exclusive); + consider_point(value + scan.max_value, scan.max_position); + } } if (best_is_node) { @@ -1610,14 +1809,12 @@ class RmMBTree : public RmMBase> { const std::size_t middle_end = std::max(middle_begin, last_full_block_exclusive * kBlockBits); if (middle_begin < middle_end) { - Cover cover; - collect_cover(middle_begin, middle_end, cover); - for (std::size_t i = 0; i < cover.size; ++i) { - const NodeRef& node = cover.nodes[i]; - Summary summary = summary_at(node.level, node.index); + for_each_cover_node(middle_begin, middle_end, [&](NodeRef node) { + const Summary summary = summary_at(node.level, node.index); consider(value + summary.min_excess, summary.min_count); value += summary.block_excess; - } + return true; + }); } if (middle_end < range_end_exclusive) { @@ -1665,20 +1862,25 @@ class RmMBTree : public RmMBase> { const std::size_t middle_end = std::max(middle_begin, last_full_block_exclusive * kBlockBits); if (middle_begin < middle_end) { - Cover cover; - collect_cover(middle_begin, middle_end, cover); - for (std::size_t i = 0; i < cover.size; ++i) { - const NodeRef& node = cover.nodes[i]; - Summary summary = summary_at(node.level, node.index); + bool found_node = false; + std::size_t selected = npos; + for_each_cover_node(middle_begin, middle_end, [&](NodeRef node) { + const Summary summary = summary_at(node.level, node.index); const std::int64_t candidate = value + summary.min_excess; if (candidate == target) { if (rank <= summary.min_count) { - return descend_qth_min(node.level, node.index, target - value, - rank); + selected = + descend_qth_min(node.level, node.index, target - value, rank); + found_node = true; + return false; } rank -= summary.min_count; } value += summary.block_excess; + return true; + }); + if (found_node) { + return selected; } } @@ -1693,49 +1895,74 @@ class RmMBTree : public RmMBase> { } /** - * @brief Decompose an aligned block interval into summary nodes. - * @details Produces a left-to-right cover of `[begin, end)` using the largest - * summary nodes available. Both boundaries must be aligned to block size. - * @param begin Inclusive bit start, aligned to `kBlockBits`. - * @param end Exclusive bit end, aligned to `kBlockBits`. - * @param out Destination fixed-size cover buffer. + * @brief Visit an aligned block interval as summary nodes. + * @details Produces the same left-to-right cover as the previous materialized + * cover helper, but streams nodes into @p callback and only keeps the right + * boundary stack needed to preserve bit order. The callback should return + * false to stop early. */ - void collect_cover(std::size_t begin, std::size_t end, Cover& out) const { + template + bool for_each_cover_node(std::size_t begin, + std::size_t end, + Callback&& callback) const { if (begin >= end || total_levels() == 0 || (begin % kBlockBits) != 0 || (end % kBlockBits) != 0) { - return; + return true; } - Cover right_cover; + std::array right_nodes; + std::size_t right_size = 0; + std::size_t emitted = 0; std::size_t level = 0; std::size_t left = begin / kBlockBits; std::size_t right = end / kBlockBits; + auto emit = [&](NodeRef node) { + if (emitted >= kMaxCoverNodes) { + return true; + } + ++emitted; + return callback(node); + }; + + auto push_right = [&](NodeRef node) { + if (right_size < right_nodes.size()) { + right_nodes[right_size++] = node; + } + }; + while (left < right) { if (!has_parent_level(level)) { for (std::size_t index = left; index < right; ++index) { - out.push({level, index}); + if (!emit({level, index})) { + return false; + } } break; } const std::size_t fanout = fanout_to_parent(level); while (left < right && (left % fanout) != 0) { - out.push({level, left}); + if (!emit({level, left})) { + return false; + } ++left; } while (left < right && (right % fanout) != 0) { --right; - right_cover.push({level, right}); + push_right({level, right}); } left /= fanout; right /= fanout; ++level; } - while (right_cover.size > 0) { - out.push(right_cover.nodes[--right_cover.size]); + while (right_size > 0) { + if (!emit(right_nodes[--right_size])) { + return false; + } } + return true; } /** @@ -1927,6 +2154,35 @@ class RmMBTree : public RmMBase> { return result; } + /** + * @brief Byte-accelerated minimum-only scan of an arbitrary bit interval. + * @details Used by range-min position/value queries that do not need maximum + * fields or minimum multiplicity. + */ + MinScanResult scan_min_range(std::size_t begin, std::size_t end) const { + MinScanResult result; + const auto& lut = byte_lut(); + while (begin < end && (begin & 7) != 0) { + append_scanned_min_bit(result, begin); + ++begin; + } + while (begin + 8 <= end) { + const ByteAgg& byte = lut[get_byte(begin)]; + const std::int64_t min_candidate = result.block_excess + byte.min_excess; + if (min_candidate < result.min_value) { + result.min_value = min_candidate; + result.min_position = begin + byte.pos_first_min; + } + result.block_excess += byte.block_excess; + begin += 8; + } + while (begin < end) { + append_scanned_min_bit(result, begin); + ++begin; + } + return result; + } + /** * @brief Append one bit to an incremental range scan. * @details Updates total excess, min/max values, first min/max positions, and @@ -1949,6 +2205,18 @@ class RmMBTree : public RmMBase> { } } + /** + * @brief Append one bit to an incremental minimum-only range scan. + */ + void append_scanned_min_bit(MinScanResult& result, + std::size_t position) const { + result.block_excess += bit(position) ? 1 : -1; + if (result.block_excess < result.min_value) { + result.min_value = result.block_excess; + result.min_position = position; + } + } + /** * @brief Return the byte starting at a byte-aligned bit position. * @details Reads eight bits from the backing span by shifting the containing diff --git a/include/pixie/rmq.h b/include/pixie/rmq.h index 7b3ac5c..4e16f14 100644 --- a/include/pixie/rmq.h +++ b/include/pixie/rmq.h @@ -2,137 +2,147 @@ // clang-format off /* - * RMQ benchmark snapshot, 2026-06-11. + * RMQ benchmark snapshot, 2026-06-15. * - * Query command: - * taskset -c 0 ./build/release/bench_rmq - * --benchmark_filter='^(rmq_sparse_table|rmq_segment_tree|rmq_segment_btree_xl|rmq_cartesian_tree|rmq_cartesian_tree_segment_btree_xl|rmq_sdsl_sct)/' - * --benchmark_min_time=0.25s - * - * Build/memory command: - * taskset -c 0 ./build/release/bench_rmq - * --benchmark_filter='^(rmq_build_sparse_table|rmq_build_segment_tree|rmq_build_segment_btree_xl|rmq_build_cartesian_tree|rmq_build_cartesian_tree_segment_btree_xl|rmq_build_sdsl_sct)/' + * Baseline command: + * ./build/release/bench_rmq + * --benchmark_filter='^(rmq_sparse_table|rmq_segment_tree|rmq_hybrid_btree|rmq_cartesian_rmm|rmq_cartesian_hybrid_btree|rmq_sdsl_sct|rmq_build_sparse_table|rmq_build_segment_tree|rmq_build_hybrid_btree|rmq_build_cartesian_rmm|rmq_build_cartesian_hybrid_btree|rmq_build_sdsl_sct)/' * --benchmark_min_time=0.20s + * --benchmark_repetitions=10 + * --benchmark_report_aggregates_only=true * - * Query CPU time. "max width" is the benchmark's maximum sampled query width. - * Sparse-table rows are intentionally not registered above 2^22. "cartesian xl" - * is the experimental `rmq_cartesian_tree_segment_btree_xl` row. The - * cartesian xl column was refreshed after reverting the short direct value scan - * so queries do not scan the original value array, embedding middle-node min - * summaries into the selector tail bits, and using a single root-level BP - * high-sparse layout inside the Cartesian XL fallback. This snapshot uses the - * no-LeafMinimum Cartesian XL ablation for all sizes. Rejected local value - * overlays, close-position caches, and alternative BP leaf/fanout layouts are - * not included. + * Focused CartesianHybridBTree refresh for N=2^28 and N=2^30: + * ./build/release/bench_rmq + * --benchmark_filter='^(rmq_build_cartesian_hybrid_btree/(268435456|1073741824)|rmq_cartesian_hybrid_btree/(268435456|1073741824)/)' + * --benchmark_repetitions=10 + * --benchmark_report_aggregates_only=true * - * Rejected 2^28 Cartesian XL LeafSummary ablation, same binary and host, - * measured with `PIXIE_BENCH_VALUE_VARIANTS=1` to keep peak memory bounded. - * Query times are CPU ns/query; build times are CPU ms. LeafSummary stored one - * cached full-leaf minimum offset/excess record per BP-depth leaf, but it was - * removed because the speedup was mixed for the extra memory. + * The focused refresh used current benchmark registrations: 0.2s warmup and + * 2.0s minimum time. * - * | N | max width | no LeafSummary (ns) | LeafSummary (ns) | delta | - * |-----:|----------:|--------------------:|-----------------:|------:| - * | 2^28 | 64 | 422.6 | 401.9 | -4.9% | - * | 2^28 | 4096 | 607.2 | 624.2 | +2.8% | - * | 2^28 | 2^18 | 443.6 | 360.2 | -18.8% | - * | 2^28 | 2^22 | 71.2 | 76.8 | +7.9% | - * | 2^28 | 2^26 | 32.4 | 23.7 | -27.1% | - * | 2^28 | 2^28 | 19.4 | 21.2 | +9.6% | + * Tables report CPU mean across 10 repetitions. "max width" is the benchmark's + * maximum sampled query width. Sparse-table rows are intentionally not + * registered above 2^22. Rows above 2^26 are focused large-RMQ measurements; + * unavailable columns are marked with "-". * - * | N | variant | build CPU (ms) | aux memory (MiB) | aux bits/value | - * |-----:|----------------|---------------:|-----------------:|---------------:| - * | 2^28 | no LeafSummary | 1986.003 | 68.536 | 2.142 | - * | 2^28 | LeafSummary | 1931.149 | 72.536 | 2.267 | + * Query CPU time. * - * | N | max width | sparse table (ns) | segment tree (ns) | segment btree xl (ns) | cartesian tree (ns) | cartesian xl (ns) | sdsl sct (ns) | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^10 | 64 | 10.9 | 38.8 | 35.1 | 73.1 | 88.5 | 119.4 | - * | 2^10 | 2^10 | 13.5 | 62.3 | 42.3 | 99.4 | 159.3 | 250.8 | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^14 | 64 | 16.2 | 50.8 | 34.7 | 90.0 | 101.4 | 129.2 | - * | 2^14 | 4096 | 16.1 | 93.5 | 40.4 | 114.9 | 99.7 | 438.5 | - * | 2^14 | 2^14 | 16.3 | 106.9 | 37.3 | 112.1 | 61.5 | 548.6 | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^18 | 64 | 72.9 | 123.0 | 61.0 | 94.4 | 124.7 | 144.8 | - * | 2^18 | 4096 | 58.4 | 224.3 | 62.6 | 130.5 | 210.5 | 556.1 | - * | 2^18 | 2^18 | 53.1 | 348.7 | 44.3 | 129.5 | 37.1 | 758.6 | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^22 | 64 | 118.2 | 191.9 | 133.0 | 134.7 | 136.4 | 218.1 | - * | 2^22 | 4096 | 86.2 | 337.8 | 113.8 | 364.7 | 279.9 | 704.6 | - * | 2^22 | 2^18 | 71.5 | 463.3 | 69.0 | 380.6 | 62.5 | 1018.8 | - * | 2^22 | 2^22 | 66.1 | 540.1 | 45.1 | 345.9 | 24.1 | 1091.4 | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^24 | 64 | - | 219.7 | 181.7 | 244.7 | 279.7 | 330.7 | - * | 2^24 | 4096 | - | 399.7 | 267.2 | 639.8 | 465.1 | 1049.7 | - * | 2^24 | 2^18 | - | 528.0 | 151.6 | 498.7 | 99.6 | 1408.6 | - * | 2^24 | 2^22 | - | 586.8 | 51.7 | 421.4 | 24.0 | 1546.3 | - * | 2^24 | 2^24 | - | 616.8 | 40.3 | 554.3 | 20.9 | 1294.7 | - * |-----:|----------:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^26 | 64 | - | 288.7 | 251.7 | 302.2 | 377.2 | 616.2 | - * | 2^26 | 4096 | - | 533.1 | 307.8 | 621.7 | 680.9 | 2030.8 | - * | 2^26 | 2^18 | - | 838.9 | 270.9 | 583.3 | 201.6 | 2426.9 | - * | 2^26 | 2^22 | - | 842.4 | 63.3 | 659.4 | 33.6 | 2190.1 | - * | 2^26 | 2^26 | - | 937.0 | 57.1 | 587.5 | 21.5 | 1973.5 | + * | N | max width | sparse table (ns) | segment tree (ns) | hybrid btree (ns) | cartesian rmm (ns) | cartesian hybrid (ns) | sdsl sct (ns) | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^10 | 64 | 17.2 | 51.3 | 65.8 | 174.2 | 96.1 | 136.9 | + * | 2^10 | 2^10 | 24.0 | 76.4 | 54.0 | 576.2 | 145.0 | 245.9 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^14 | 64 | 19.0 | 54.1 | 44.4 | 207.0 | 108.4 | 166.1 | + * | 2^14 | 4096 | 20.1 | 103.8 | 63.6 | 777.2 | 105.7 | 512.3 | + * | 2^14 | 2^14 | 18.1 | 114.9 | 46.1 | 976.5 | 66.1 | 612.8 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^18 | 64 | 36.3 | 78.1 | 51.8 | 201.2 | 116.1 | 187.0 | + * | 2^18 | 4096 | 35.7 | 148.7 | 79.7 | 874.1 | 220.7 | 655.6 | + * | 2^18 | 2^18 | 27.8 | 196.8 | 23.3 | 1255.2 | 37.9 | 838.7 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^22 | 64 | 89.3 | 170.5 | 128.9 | 220.6 | 125.9 | 204.4 | + * | 2^22 | 4096 | 73.2 | 310.2 | 131.8 | 882.9 | 239.3 | 750.4 | + * | 2^22 | 2^18 | 61.9 | 461.6 | 33.2 | 1361.6 | 53.0 | 1107.3 | + * | 2^22 | 2^22 | 57.3 | 507.1 | 17.9 | 1460.8 | 20.5 | 962.6 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^24 | 64 | - | 193.9 | 169.9 | 336.3 | 184.1 | 274.9 | + * | 2^24 | 4096 | - | 365.6 | 155.9 | 1023.5 | 291.8 | 931.9 | + * | 2^24 | 2^18 | - | 497.2 | 43.1 | 1493.6 | 74.1 | 1276.3 | + * | 2^24 | 2^22 | - | 603.4 | 20.8 | 1659.0 | 22.2 | 1287.3 | + * | 2^24 | 2^24 | - | 574.9 | 17.5 | 1637.9 | 20.0 | 1251.4 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^26 | 64 | - | 250.4 | 251.0 | 359.3 | 289.3 | 807.8 | + * | 2^26 | 4096 | - | 450.0 | 243.5 | 1243.7 | 465.4 | 1389.8 | + * | 2^26 | 2^18 | - | 678.1 | 77.0 | 1685.7 | 121.2 | 2113.1 | + * | 2^26 | 2^22 | - | 697.8 | 23.6 | 2016.6 | 31.2 | 1990.0 | + * | 2^26 | 2^26 | - | 698.8 | 17.5 | 1844.8 | 19.0 | 1832.5 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^28 | 64 | - | - | 300.0 | 471.1 | 451.8 | - | + * | 2^28 | 4096 | - | - | 372.6 | 1421.0 | 803.9 | - | + * | 2^28 | 2^18 | - | - | 224.9 | 2084.5 | 416.6 | - | + * | 2^28 | 2^22 | - | - | 56.9 | 2095.4 | 77.2 | - | + * | 2^28 | 2^26 | - | - | 21.2 | 2301.6 | 31.7 | - | + * | 2^28 | 2^28 | - | - | 21.3 | 2515.1 | 19.0 | - | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^30 | 64 | - | - | 334.2 | 715.4 | 656.2 | - | + * | 2^30 | 4096 | - | - | 408.5 | 2061.2 | 1006.4 | - | + * | 2^30 | 2^18 | - | - | 557.7 | 2575.8 | 927.3 | - | + * | 2^30 | 2^22 | - | - | 145.5 | 2703.3 | 258.8 | - | + * | 2^30 | 2^26 | - | - | 45.5 | 2826.7 | 82.3 | - | + * | 2^30 | 2^30 | - | - | 30.4 | 3110.2 | 28.5 | - | * * Build CPU time. Sparse-table build rows are intentionally not registered * above 2^22. * - * | N | sparse table (ms) | segment tree (ms) | segment btree xl (ms) | cartesian tree (ms) | cartesian xl (ms) | sdsl sct (ms) | - * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^10 | 0.007 | 0.001 | 0.001 | 0.006 | 0.008 | 0.007 | - * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^14 | 1.123 | 0.015 | 0.017 | 0.041 | 0.059 | 0.213 | - * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^18 | 29.597 | 0.358 | 0.281 | 1.678 | 1.845 | 2.539 | - * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^22 | 964.147 | 63.087 | 5.662 | 29.543 | 30.573 | 40.821 | - * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^24 | - | 273.543 | 21.747 | 167.399 | 118.968 | 164.629 | - * |-----:|------------------:|------------------:|----------------------:|--------------------:|------------------:|--------------:| - * | 2^26 | - | 1176.694 | 85.392 | 728.030 | 476.286 | 650.639 | + * | N | sparse table (ms) | segment tree (ms) | hybrid btree (ms) | cartesian rmm (ms) | cartesian hybrid (ms) | sdsl sct (ms) | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^10 | 0.007 | 0.001 | 0.002 | 0.014 | 0.013 | 0.007 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^14 | 1.232 | 0.014 | 0.028 | 0.137 | 0.115 | 0.218 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^18 | 30.883 | 0.279 | 0.453 | 3.372 | 2.980 | 2.556 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^22 | 715.861 | 65.124 | 8.960 | 55.726 | 49.400 | 41.505 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^24 | - | 276.132 | 35.264 | 225.624 | 200.655 | 170.930 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^26 | - | 1281.908 | 138.719 | 891.819 | 779.080 | 663.442 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^28 | - | - | 586.827 | 3730.628 | 3347.897 | - | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^30 | - | - | 2324.537 | 15002.269 | 16070.000 | - | * * Owned auxiliary memory. Benchmarks use 64-bit signed integer values and * 64-bit indexes. The external input values are not owned by RMQ indexes and - * are excluded. + * are excluded. CartesianRmM does not currently expose an owned-memory counter. * - * | N | sparse table (MiB) | segment tree (MiB) | segment btree xl (MiB) | cartesian tree (MiB) | cartesian xl (MiB) | sdsl sct (MiB) | - * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| - * | 2^10 | 0.071 | 0.016 | 0.009 | 0.004 | 0.014 | 0.001 | - * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| - * | 2^14 | 1.626 | 0.250 | 0.011 | 0.025 | 0.018 | 0.005 | - * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| - * | 2^18 | 34.001 | 4.000 | 0.065 | 0.474 | 0.082 | 0.079 | - * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| - * | 2^22 | 672.001 | 64.000 | 0.801 | 9.539 | 1.141 | 1.262 | - * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| - * | 2^24 | - | 256.000 | 3.155 | 42.148 | 4.585 | 5.051 | - * |-----:|-------------------:|-------------------:|-----------------------:|---------------------:|-------------------:|---------------:| - * | 2^26 | - | 1024.000 | 8.340 | 184.582 | 18.551 | 20.224 | + * | N | sparse table (MiB) | segment tree (MiB) | hybrid btree (MiB) | cartesian rmm (MiB) | cartesian hybrid (MiB) | sdsl sct (MiB) | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^10 | 0.071 | 0.016 | 0.001 | - | 0.014 | 0.001 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^14 | 1.626 | 0.250 | 0.003 | - | 0.018 | 0.005 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^18 | 34.001 | 4.000 | 0.036 | - | 0.082 | 0.079 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^22 | 672.001 | 64.000 | 0.606 | - | 1.141 | 1.262 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^24 | - | 256.000 | 2.483 | - | 4.585 | 5.051 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^26 | - | 1024.000 | 10.182 | - | 18.551 | 20.224 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^28 | - | - | 35.102 | - | 68.411 | - | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^30 | - | - | 134.783 | - | 267.979 | - | * * Owned auxiliary memory normalized by indexed value count (bits/value). * - * | N | sparse table | segment tree | segment btree xl | cartesian tree | cartesian xl | sdsl sct | - * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^10 | 579.188 | 128.438 | 71.438 | 30.750 | 112.938 | 7.312 | - * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^14 | 832.262 | 128.027 | 5.434 | 12.562 | 8.977 | 2.770 | - * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^18 | 1088.020 | 128.002 | 2.089 | 15.175 | 2.629 | 2.534 | - * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^22 | 1344.002 | 128.000 | 1.603 | 19.079 | 2.281 | 2.524 | - * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^24 | - | 128.000 | 1.577 | 21.074 | 2.293 | 2.526 | - * |-----:|-------------:|-------------:|-----------------:|---------------:|-------------:|---------:| - * | 2^26 | - | 128.000 | 1.042 | 23.073 | 2.319 | 2.528 | + * | N | sparse table | segment tree | hybrid btree | cartesian rmm | cartesian hybrid | sdsl sct | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^10 | 579.188 | 128.438 | 4.500 | - | 112.938 | 7.312 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^14 | 832.262 | 128.027 | 1.293 | - | 8.977 | 2.770 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^18 | 1088.020 | 128.002 | 1.162 | - | 2.629 | 2.534 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^22 | 1344.002 | 128.000 | 1.211 | - | 2.281 | 2.524 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^24 | - | 128.000 | 1.242 | - | 2.293 | 2.526 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^26 | - | 128.000 | 1.273 | - | 2.319 | 2.528 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^28 | - | - | 1.097 | - | 2.138 | - | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^30 | - | - | 1.053 | - | 2.094 | - | */ // clang-format on -#include -#include -#include +#include +#include +#include #include -#include -#include #include #include + +#ifdef SDSL_SUPPORT +#include +#endif diff --git a/include/pixie/rmq/bp_plus_minus_one_rmq.h b/include/pixie/rmq/bp_plus_minus_one_rmq.h deleted file mode 100644 index 2aa0df9..0000000 --- a/include/pixie/rmq/bp_plus_minus_one_rmq.h +++ /dev/null @@ -1,633 +0,0 @@ -#pragma once - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace pixie::rmq { - -/** - * @brief FCB-style RMQ backend for arrays with adjacent differences ±1. - * - * @details The indexed depth sequence is represented by BP deltas: bit 1 means - * the next depth is current + 1, and bit 0 means current - 1. A sequence with - * @p depth_count depth positions has @p depth_count - 1 delta bits. Blocks - * match the 64-bit or 128-bit excess primitives in `bits.h`; each block stores - * a compact 16-byte summary with its base depth, absolute minimum value, and - * first local offset attaining that minimum. - * - * @tparam Index Unsigned integer type used for stored positions. - * @tparam BlockSize Number of depth positions per microblock. - */ -template -class BpPlusMinusOneRmq { - public: - static_assert(BlockSize == 64 || BlockSize == 128); - - static constexpr std::size_t npos = std::numeric_limits::max(); - static constexpr Index invalid_index = std::numeric_limits::max(); - - /** - * @brief Construct an empty ±1 RMQ index. - */ - BpPlusMinusOneRmq() = default; - - /** - * @brief Build a ±1 RMQ index over a BP-encoded depth-delta sequence. - * - * @details The bit span is not copied and must outlive this object. Bit `1` - * encodes a +1 step between adjacent depths, and bit `0` encodes a -1 step. - * A sequence with @p depth_count positions requires @p depth_count - 1 bits. - * - * @param bits Packed delta bits in little-endian bit order within each word. - * @param depth_count Number of depth positions indexed by the RMQ. - * @throws std::length_error if @p Index cannot represent all positions or a - * packed 48-bit block summary overflows. - * @throws std::invalid_argument if @p bits does not contain enough words. - */ - BpPlusMinusOneRmq(std::span bits, - std::size_t depth_count) - : input_bits_(bits), depth_count_(depth_count) { - build(); - } - - /** - * @brief Copy a ±1 RMQ index and rebuild internal non-owning views. - * - * @param other Source index. - */ - BpPlusMinusOneRmq(const BpPlusMinusOneRmq& other) - : input_bits_(other.input_bits_), - depth_count_(other.depth_count_), - block_summaries_(other.block_summaries_) { - reset_macro_rmq(); - } - - /** - * @brief Copy-assign a ±1 RMQ index and rebuild internal non-owning views. - * - * @param other Source index. - * @return Reference to this object. - */ - BpPlusMinusOneRmq& operator=(const BpPlusMinusOneRmq& other) { - if (this == &other) { - return *this; - } - input_bits_ = other.input_bits_; - depth_count_ = other.depth_count_; - block_summaries_ = other.block_summaries_; - reset_macro_rmq(); - return *this; - } - - /** - * @brief Move a ±1 RMQ index and rebuild internal non-owning views. - * - * @param other Source index. - */ - BpPlusMinusOneRmq(BpPlusMinusOneRmq&& other) noexcept - : input_bits_(other.input_bits_), - depth_count_(other.depth_count_), - block_summaries_(std::move(other.block_summaries_)) { - reset_macro_rmq(); - } - - /** - * @brief Move-assign a ±1 RMQ index and rebuild internal non-owning views. - * - * @param other Source index. - * @return Reference to this object. - */ - BpPlusMinusOneRmq& operator=(BpPlusMinusOneRmq&& other) noexcept { - if (this == &other) { - return *this; - } - input_bits_ = other.input_bits_; - depth_count_ = other.depth_count_; - block_summaries_ = std::move(other.block_summaries_); - reset_macro_rmq(); - return *this; - } - - /** - * @brief Return the number of indexed depth positions. - * - * @return The @p depth_count passed to construction. - */ - std::size_t size() const { return depth_count_; } - - /** - * @brief Whether the indexed depth sequence is empty. - * - * @return `true` when `size() == 0`. - */ - bool empty() const { return depth_count_ == 0; } - - /** - * @brief Return owned auxiliary memory usage in bytes. - * - * @details Counts this ±1 RMQ object, block summaries, and the macro sparse - * table over those summaries. The external packed delta bits are not owned - * and are excluded. - */ - std::size_t memory_usage_bytes() const { - return sizeof(*this) + pixie::vector_capacity_bytes(block_summaries_) + - pixie::nested_owned_memory_bytes(macro_rmq_); - } - - /** - * @brief Return the first minimum depth position in [@p left, @p right). - * - * @details The query range is half-open over depth positions, not delta-bit - * positions. Ties return the smallest position attaining the minimum depth. - * Empty or invalid ranges return `npos`. - * - * @param left First depth position in the query range. - * @param right One past the last depth position in the query range. - * @return Zero-based position of the first range minimum, or `npos`. - */ - std::size_t arg_min(std::size_t left, std::size_t right) const { - if (left >= right || right > depth_count_) { - return npos; - } - - const std::size_t last = right - 1; - const std::size_t left_block = left / BlockSize; - const std::size_t right_block = last / BlockSize; - if (left_block == right_block) { - return scan_block_range_position(left_block, left % BlockSize, - last % BlockSize); - } - - const std::size_t left_offset = left % BlockSize; - const std::size_t right_offset = last % BlockSize; - Candidate answer; - if (left_offset > right_offset && left_offset < BlockSize - 1 && - right_offset > 0) { - const auto left_bits = block_bits(left_block); - const auto right_bits = block_bits(right_block); - const ExcessBoundaryPairResult result = block_disjoint_boundary_min( - left_bits.data(), left_offset, right_bits.data(), right_offset); - answer = local_excess_result_candidate(left_block, result.suffix); - answer = better( - answer, local_excess_result_candidate(right_block, result.prefix)); - } else { - answer = - scan_block_range(left_block, left_offset, block_size(left_block) - 1); - answer = better(answer, scan_block_range(right_block, 0, right_offset)); - } - - if (left_block + 1 < right_block) { - const std::size_t block_position = - macro_rmq_.arg_min(left_block + 1, right_block); - if (block_position != MacroRmq::npos) { - answer = better(answer, scan_full_block(block_position)); - } - } - - return answer.position; - } - - private: - struct Candidate { - std::size_t position = npos; - std::int64_t value = std::numeric_limits::max(); - }; - - /** - * @brief Packed summary for one depth block. - * - * @details Stores signed 48-bit base depth, signed 48-bit absolute block - * minimum, and 16-bit first local offset of that minimum in two 64-bit words. - * The hot macro-RMQ comparison keeps the sign-biased minimum in the low - * 48 bits of `word0`, so signed ordering is available as one masked unsigned - * comparison. The high 16 bits of `word0` store the local minimum offset, and - * `word1` stores the base depth. - */ - struct alignas(16) BlockSummary { - static constexpr std::uint64_t kSigned48Mask = (std::uint64_t{1} << 48) - 1; - static constexpr std::uint64_t kSigned48SignBit = std::uint64_t{1} << 47; - static constexpr std::int64_t kSigned48Min = -(std::int64_t{1} << 47); - static constexpr std::int64_t kSigned48Max = (std::int64_t{1} << 47) - 1; - - std::uint64_t word0 = 0; - std::uint64_t word1 = 0; - - /** - * @brief Pack block metadata into a 16-byte summary. - * - * @param base_depth Absolute depth at local offset 0. - * @param min_value Absolute minimum depth in the block. - * @param min_offset First local offset attaining @p min_value. - * @return Packed block summary. - * @throws std::length_error if a signed 48-bit field or 16-bit offset - * overflows. - */ - static BlockSummary make(std::int64_t base_depth, - std::int64_t min_value, - std::size_t min_offset) { - if (!fits_signed48(base_depth) || !fits_signed48(min_value)) { - throw std::length_error("RMQ +/-1 block summary depth overflow"); - } - if (min_offset > std::numeric_limits::max()) { - throw std::length_error("RMQ +/-1 block summary offset overflow"); - } - - const std::uint64_t packed_base = pack_signed48(base_depth); - const std::uint64_t ordered_min = pack_ordered48(min_value); - return {ordered_min | (static_cast(min_offset) << 48), - packed_base}; - } - - /** - * @brief Decode the absolute depth at local offset 0. - * - * @return Signed base depth for this block. - */ - std::int64_t base_depth() const { - return unpack_signed48(word1 & kSigned48Mask); - } - - /** - * @brief Decode the absolute minimum depth in this block. - * - * @return Signed block minimum value. - */ - std::int64_t min_value() const { - return unpack_ordered48(ordered_min_value()); - } - - /** - * @brief Return the sign-biased minimum payload for fast comparisons. - * - * @details The returned low-48-bit value preserves signed order under - * unsigned comparison: smaller signed depths have smaller payloads. - * - * @return Sign-biased 48-bit minimum depth payload. - */ - std::uint64_t ordered_min_value() const { return word0 & kSigned48Mask; } - - /** - * @brief Decode the first local minimum offset in this block. - * - * @return Local depth-position offset in [0, BlockSize). - */ - std::size_t min_offset() const { - return static_cast(word0 >> 48); - } - - private: - /** - * @brief Test whether a value fits in the signed 48-bit packed field. - * - * @param value Value to test. - * @return `true` if @p value is representable. - */ - static bool fits_signed48(std::int64_t value) { - return value >= kSigned48Min && value <= kSigned48Max; - } - - /** - * @brief Truncate a signed value to its 48-bit two's-complement payload. - * - * @param value Signed 48-bit value. - * @return Low 48 payload bits. - */ - static std::uint64_t pack_signed48(std::int64_t value) { - return static_cast(value) & kSigned48Mask; - } - - /** - * @brief Encode a signed 48-bit value for unsigned ordered comparison. - * - * @param value Signed 48-bit value. - * @return Low 48 payload bits with the sign bit flipped. - */ - static std::uint64_t pack_ordered48(std::int64_t value) { - return pack_signed48(value) ^ kSigned48SignBit; - } - - /** - * @brief Sign-extend a 48-bit two's-complement payload. - * - * @param value Low 48 payload bits. - * @return Decoded signed value. - */ - static std::int64_t unpack_signed48(std::uint64_t value) { - if ((value & kSigned48SignBit) != 0) { - value |= ~kSigned48Mask; - } - return static_cast(value); - } - - /** - * @brief Decode a sign-biased 48-bit ordered payload. - * - * @param value Low 48 payload bits with the sign bit flipped. - * @return Decoded signed value. - */ - static std::int64_t unpack_ordered48(std::uint64_t value) { - return unpack_signed48(value ^ kSigned48SignBit); - } - }; - - static_assert(sizeof(BlockSummary) == 16); - static_assert(alignof(BlockSummary) == 16); - - struct BlockSummaryMinLess { - /** - * @brief Compare two block summaries by absolute minimum value. - * - * @param left First block summary. - * @param right Second block summary. - * @return `true` if @p left has a strictly smaller block minimum. - */ - bool operator()(const BlockSummary& left, const BlockSummary& right) const { - return left.ordered_min_value() < right.ordered_min_value(); - } - }; - - using MacroRmq = SparseTable; - - /** - * @brief Return the number of depth positions in a block. - * - * @param block Zero-based block index. - * @return `BlockSize` for full blocks, or the tail size for the last block. - */ - std::size_t block_size(std::size_t block) const { - const std::size_t begin = block * BlockSize; - return std::min(BlockSize, depth_count_ - begin); - } - - /** - * @brief Choose the better of two candidate minima. - * - * @details Missing candidates use `npos`. Smaller values win; equal values - * return the smaller global position. - * - * @param left First candidate. - * @param right Second candidate. - * @return Selected candidate. - */ - Candidate better(Candidate left, Candidate right) const { - if (left.position == npos) { - return right; - } - if (right.position == npos) { - return left; - } - if (right.value < left.value) { - return right; - } - if (left.value < right.value) { - return left; - } - return right.position < left.position ? right : left; - } - - /** - * @brief Build block summaries and the macro sparse table. - * - * @details Computes the absolute base depth, absolute block minimum, and - * first local minimum offset for each block. - * - * @throws std::length_error if @p Index or a packed block summary overflows. - * @throws std::invalid_argument if the input bit span is too small. - */ - void build() { - block_summaries_.clear(); - macro_rmq_ = MacroRmq(); - - if (depth_count_ == 0) { - return; - } - if (depth_count_ > static_cast(invalid_index)) { - throw std::length_error("RMQ ±1 index type is too small"); - } - if (input_bits_.size() < (depth_count_ - 1 + 63) / 64) { - throw std::invalid_argument("RMQ ±1 bit span is too small"); - } - - const std::size_t block_count = (depth_count_ + BlockSize - 1) / BlockSize; - block_summaries_.reserve(block_count); - - std::int64_t base_depth = 0; - for (std::size_t block = 0; block < block_count; ++block) { - const std::size_t begin = block * BlockSize; - const std::size_t size = std::min(BlockSize, depth_count_ - begin); - const auto bits = block_bits(block); - const ExcessResult block_min = block_excess_min(bits.data(), 0, size - 1); - - block_summaries_.push_back(BlockSummary::make( - base_depth, base_depth + block_min.min_excess, block_min.offset)); - if (block + 1 < block_count) { - base_depth += block_excess(bits, next_block_delta_count(begin)); - } - } - - reset_macro_rmq(); - } - - /** - * @brief Read an input word or return zero past the available span. - * - * @param word Zero-based 64-bit word index. - * @return Input word value, or zero when @p word is out of range. - */ - std::uint64_t word_or_zero(std::size_t word) const { - return word < input_bits_.size() ? input_bits_[word] : 0; - } - - /** - * @brief Load the words backing one block. - * - * @param block Zero-based block index. - * @return Pair of words; 64-position blocks use only the first word. - */ - std::array block_bits(std::size_t block) const { - const std::size_t first_word = block * (BlockSize / 64); - if constexpr (BlockSize == 64) { - return {word_or_zero(first_word), 0}; - } - return {word_or_zero(first_word), word_or_zero(first_word + 1)}; - } - - /** - * @brief Count delta bits from a block start to the next block boundary. - * - * @param begin Global depth-position offset of the block start. - * @return Number of delta bits contributing to the transition to the next - * block base depth. - */ - std::size_t next_block_delta_count(std::size_t begin) const { - const std::size_t next_begin = begin + BlockSize; - return std::min(next_begin, depth_count_ - 1) - begin; - } - - /** - * @brief Compute total excess over a contiguous delta-bit range. - * - * @param bits Two words loaded from a block boundary. - * @param delta_count Number of delta bits to scan. - * @return Sum of +1/-1 deltas in the range. - */ - std::int64_t block_excess(const std::array& bits, - std::size_t delta_count) const { - if constexpr (BlockSize == 64) { - return prefix_excess_64(bits.data(), delta_count); - } - return prefix_excess_128(bits.data(), delta_count); - } - - /** - * @brief Find the local minimum in a block-width bit range. - * - * @param bits Words loaded from a block boundary. - * @param left First local prefix offset, inclusive. - * @param right Last local prefix offset, inclusive. - * @return Local minimum excess result. - */ - ExcessResult block_excess_min(const std::uint64_t* bits, - std::size_t left, - std::size_t right) const { - if constexpr (BlockSize == 64) { - return excess_min_64(bits, left, right); - } - return excess_min_128(bits, left, right); - } - - /** - * @brief Find local minima for disjoint left-suffix and right-prefix ranges. - * - * @param suffix_bits Words loaded from the left boundary block. - * @param suffix_left First local prefix offset in the suffix range. - * @param prefix_bits Words loaded from the right boundary block. - * @param prefix_right Last local prefix offset in the prefix range. - * @return Pair of local minimum results. - */ - ExcessBoundaryPairResult block_disjoint_boundary_min( - const std::uint64_t* suffix_bits, - std::size_t suffix_left, - const std::uint64_t* prefix_bits, - std::size_t prefix_right) const { - if constexpr (BlockSize == 64) { - return excess_min_64_disjoint_suffix_prefix(suffix_bits, suffix_left, - prefix_bits, prefix_right); - } - return excess_min_128_disjoint_suffix_prefix(suffix_bits, suffix_left, - prefix_bits, prefix_right); - } - - /** - * @brief Return only the minimum position inside one block range. - * - * @details Used for same-block queries where the absolute minimum value is - * not needed. The range is inclusive in local block offsets. - * - * @param block Zero-based block index. - * @param left_offset First local depth-position offset. - * @param right_offset Last local depth-position offset. - * @return Global position of the first local minimum, or `npos`. - */ - std::size_t scan_block_range_position(std::size_t block, - std::size_t left_offset, - std::size_t right_offset) const { - const std::size_t begin = block * BlockSize; - const std::size_t size = block_size(block); - right_offset = std::min(right_offset, size - 1); - const auto bits = block_bits(block); - const ExcessResult result = - block_excess_min(bits.data(), left_offset, right_offset); - if (result.offset == npos || result.offset >= size) { - return npos; - } - return begin + result.offset; - } - - /** - * @brief Scan an inclusive local range inside one block. - * - * @details Uses the block-width excess-min primitive for the relative minimum - * and combines it - * with the stored block base depth to return an absolute-depth candidate. - * - * @param block Zero-based block index. - * @param left_offset First local depth-position offset. - * @param right_offset Last local depth-position offset. - * @return Candidate containing global position and absolute depth. - */ - Candidate scan_block_range(std::size_t block, - std::size_t left_offset, - std::size_t right_offset) const { - const std::size_t begin = block * BlockSize; - const std::size_t size = block_size(block); - right_offset = std::min(right_offset, size - 1); - const auto bits = block_bits(block); - const ExcessResult result = - block_excess_min(bits.data(), left_offset, right_offset); - if (result.offset == npos || result.offset >= size) { - return {}; - } - return {begin + result.offset, - block_summaries_[block].base_depth() + result.min_excess}; - } - - /** - * @brief Convert a local excess-min result into an absolute-depth candidate. - * - * @param block Zero-based block index owning @p result. - * @param result Local minimum result from a block-width excess primitive. - * @return Candidate containing global position and absolute depth. - */ - Candidate local_excess_result_candidate(std::size_t block, - ExcessResult result) const { - const std::size_t size = block_size(block); - if (result.offset == npos || result.offset >= size) { - return {}; - } - return {block * BlockSize + result.offset, - block_summaries_[block].base_depth() + result.min_excess}; - } - - /** - * @brief Return the precomputed minimum candidate for a full block. - * - * @param block Zero-based block index. - * @return Candidate containing global position and absolute depth. - */ - Candidate scan_full_block(std::size_t block) const { - const std::size_t begin = block * BlockSize; - return {begin + block_summaries_[block].min_offset(), - block_summaries_[block].min_value()}; - } - - /** - * @brief Rebuild the macro sparse table over block summaries. - * - * @details Called after build, copy, and move operations because the sparse - * table stores a non-owning span into this object's block-summary vector. - */ - void reset_macro_rmq() { - macro_rmq_ = MacroRmq(std::span(block_summaries_)); - } - - std::span input_bits_; - std::size_t depth_count_ = 0; - std::vector block_summaries_; - MacroRmq macro_rmq_; -}; - -} // namespace pixie::rmq diff --git a/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h b/include/pixie/rmq/cartesian_hybrid_btree.h similarity index 91% rename from include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h rename to include/pixie/rmq/cartesian_hybrid_btree.h index fe2174e..eeecfdf 100644 --- a/include/pixie/rmq/experimental/cartesian_tree_segment_btree_xl_rmq.h +++ b/include/pixie/rmq/cartesian_hybrid_btree.h @@ -2,8 +2,10 @@ #include #include +#include #include #include +#include #include #include @@ -12,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -20,24 +21,26 @@ #include #include -namespace pixie::rmq::experimental { +namespace pixie::rmq { + +namespace detail { /** - * @brief SegmentBTreeXL-style RMQ backend for ±1 depth sequences. + * @brief HybridBTree-style RMQ backend for ±1 depth sequences. * * @details The input words encode adjacent depth deltas: bit 1 means +1 and * bit 0 means -1. The indexed sequence has `depth_count` prefix positions, so * the bit sequence has `depth_count - 1` deltas. Public ranges are half-open * over depth positions, and ties return the smaller depth position. * - * This backend mirrors the high-level query shape of `SegmentBTreeXL`: depth + * This backend mirrors the high-level query shape of `HybridBTree`: depth * positions are split into configurable-size leaves, leaves are grouped into a * B-tree, and every internal node stores a local Cartesian/BP selector over the * minima of its immediate children. Middle levels are fixed at 192 child slots: * their selector uses 384 BP bits and reserves the remaining 128 bits for the * embedded subtree minimum position/depth. A configurable number of top levels * additionally keep sparse tables over child-minimum slots. Unlike value - * `SegmentBTreeXL`, leaves do not store another local Cartesian selector or + * `HybridBTree`, leaves do not store another local Cartesian selector or * cached minima. Leaf minima are recomputed from the original BP delta bits * with the 128-bit excess primitives from `bits.h`, using rank support to * recover the absolute base depth at the leaf start. @@ -54,15 +57,15 @@ template -class SegmentBTreeXLPlusMinusOneRmq { +class HybridBTreePlusMinusOne { public: static_assert(std::is_unsigned_v, - "SegmentBTreeXLPlusMinusOneRmq index type must be unsigned"); + "HybridBTreePlusMinusOne index type must be unsigned"); static_assert(LeafSize != 0 && LeafSize % 512 == 0, - "SegmentBTreeXLPlusMinusOneRmq leaf size must be a positive " + "HybridBTreePlusMinusOne leaf size must be a positive " "multiple of 512"); static_assert(!UseHighSparseLayout || HighSparseLayoutLevels > 0, - "SegmentBTreeXLPlusMinusOneRmq high sparse layout must cover " + "HybridBTreePlusMinusOne high sparse layout must cover " "at least one level when enabled"); static constexpr std::size_t npos = std::numeric_limits::max(); @@ -76,7 +79,7 @@ class SegmentBTreeXLPlusMinusOneRmq { /** * @brief Construct an empty ±1 RMQ index. */ - SegmentBTreeXLPlusMinusOneRmq() = default; + HybridBTreePlusMinusOne() = default; /** * @brief Build a ±1 RMQ index over external packed delta bits. @@ -86,8 +89,8 @@ class SegmentBTreeXLPlusMinusOneRmq { * @throws std::length_error if @p Index cannot represent all positions. * @throws std::invalid_argument if @p bits is shorter than required. */ - SegmentBTreeXLPlusMinusOneRmq(std::span bits, - std::size_t depth_count) { + HybridBTreePlusMinusOne(std::span bits, + std::size_t depth_count) { build(bits, depth_count); } @@ -95,11 +98,12 @@ class SegmentBTreeXLPlusMinusOneRmq { * @brief Build a ±1 RMQ index over external packed bits and rank support. * * @details The supplied rank index is non-owning and must outlive this RMQ - * object. This is used by Cartesian XL to reuse its BP rank/select index. + * object. This is used by CartesianHybridBTree to reuse its BP rank/select + * index. */ - SegmentBTreeXLPlusMinusOneRmq(std::span bits, - std::size_t depth_count, - const BitVector& rank_index) { + HybridBTreePlusMinusOne(std::span bits, + std::size_t depth_count, + const BitVector& rank_index) { build(bits, depth_count, rank_index); } @@ -260,7 +264,7 @@ class SegmentBTreeXLPlusMinusOneRmq { void build(std::size_t entry_count, EntryLess entry_less) { if (entry_count > kSelectorEntries) { throw std::length_error( - "SegmentBTreeXLPlusMinusOneRmq local selector too large"); + "HybridBTreePlusMinusOne local selector too large"); } bp_bits_.fill(0); @@ -507,24 +511,25 @@ class SegmentBTreeXLPlusMinusOneRmq { } if (depth_count_ > static_cast(invalid_index)) { throw std::length_error( - "SegmentBTreeXLPlusMinusOneRmq index type is too small"); + "HybridBTreePlusMinusOne index type is too small"); } if (depth_count_ > static_cast(std::numeric_limits::max())) { throw std::length_error( - "SegmentBTreeXLPlusMinusOneRmq depth range is too large"); + "HybridBTreePlusMinusOne depth range is too large"); } if (input_bits_.size() < (depth_count_ - 1 + 63) / 64) { throw std::invalid_argument( - "SegmentBTreeXLPlusMinusOneRmq bit span is too small"); + "HybridBTreePlusMinusOne bit span is too small"); } const std::size_t delta_count = depth_count_ - 1; if (external_rank_index_ == nullptr) { - owned_rank_index_.emplace(input_bits_, delta_count); + owned_rank_index_.emplace(input_bits_, delta_count, + BitVector::SelectSupport::kSelect0); } else if (external_rank_index_->size() < delta_count) { throw std::invalid_argument( - "SegmentBTreeXLPlusMinusOneRmq external rank index is too small"); + "HybridBTreePlusMinusOne external rank index is too small"); } initialize_layout((depth_count_ + LeafSize - 1) / LeafSize); @@ -1231,41 +1236,42 @@ class SegmentBTreeXLPlusMinusOneRmq { std::size_t high_level_begin_ = std::numeric_limits::max(); }; +} // namespace detail + /** - * @brief Experimental Cartesian-tree value RMQ using SegmentBTreeXL-style LCA. + * @brief Cartesian-tree value RMQ using HybridBTree-style LCA. * - * @details This class keeps the same public value-RMQ contract as - * `CartesianTreeRmq`. It builds the same stable Ferrada-Navarro BP - * Cartesian-tree encoding, uses `BitVector` for close-parenthesis rank/select, - * and delegates the BP-depth minimum query to - * `SegmentBTreeXLPlusMinusOneRmq`. The BP-depth backend keeps a configurable + * @details This class keeps the same public value-RMQ contract as the other + * value RMQ backends. It builds a stable Ferrada-Navarro BP Cartesian-tree + * encoding, uses `BitVector` for close-parenthesis rank/select, and delegates + * the BP-depth minimum query to + * `detail::HybridBTreePlusMinusOne`. The BP-depth backend keeps a configurable * low-level leaf size, fixed 192-entry middle nodes with embedded minima, and * fixed 256-entry high nodes. A single coarse value-level sparse table is * checked first; it uses at least 4096-value blocks and grows the block width * when needed so the top layer has at most 2^14 blocks. Wide queries whose * padded block-cover minimum lies inside the requested range return from this - * top table without touching the global BP rank/select path. + * top table without touching the global BP rank/select path. BP construction + * uses a succinct monotone bit-stack, preserving the same stable Cartesian-tree + * shape without an n-entry index stack. * - * The implementation is intentionally kept in the experimental namespace and - * is not included from `pixie/rmq.h`. It is meant to compare the usual - * Cartesian-tree reduction against a SegmentBTreeXL-shaped ±1 RMQ backend. + * This implementation is included from `pixie/rmq.h` as the compact + * Cartesian-tree reduction backed by a HybridBTree-shaped ±1 RMQ index. */ template , class Index = std::size_t, std::size_t LeafSize = 512> -class CartesianTreeSegmentBTreeXLRmq - : public RmqBase< - CartesianTreeSegmentBTreeXLRmq, - T> { +class CartesianHybridBTree + : public RmqBase, T> { public: static_assert(std::is_unsigned_v, - "CartesianTreeSegmentBTreeXLRmq index type must be unsigned"); + "CartesianHybridBTree index type must be unsigned"); static_assert(LeafSize != 0 && LeafSize % 512 == 0, - "CartesianTreeSegmentBTreeXLRmq leaf size must be a positive " + "CartesianHybridBTree leaf size must be a positive " "multiple of 512"); - using Self = CartesianTreeSegmentBTreeXLRmq; + using Self = CartesianHybridBTree; static constexpr std::size_t npos = RmqBase::npos; static constexpr Index invalid_index = std::numeric_limits::max(); @@ -1275,16 +1281,16 @@ class CartesianTreeSegmentBTreeXLRmq /** * @brief Construct an empty Cartesian-tree RMQ index. */ - CartesianTreeSegmentBTreeXLRmq() = default; + CartesianHybridBTree() = default; /** - * @brief Build an experimental Cartesian-tree RMQ index over @p values. + * @brief Build a Cartesian-tree RMQ index over @p values. * * @details The values are not copied and must outlive this object. Equal * values stay stable: the smaller index remains the first minimum. */ - explicit CartesianTreeSegmentBTreeXLRmq(std::span values, - Compare compare = Compare()) + explicit CartesianHybridBTree(std::span values, + Compare compare = Compare()) : values_(values), compare_(compare) { build(); } @@ -1292,7 +1298,7 @@ class CartesianTreeSegmentBTreeXLRmq /** * @brief Copy an RMQ index and rebuild internal non-owning views. */ - CartesianTreeSegmentBTreeXLRmq(const CartesianTreeSegmentBTreeXLRmq& other) + CartesianHybridBTree(const CartesianHybridBTree& other) : values_(other.values_), compare_(other.compare_), bp_bits_(other.bp_bits_), @@ -1307,8 +1313,7 @@ class CartesianTreeSegmentBTreeXLRmq /** * @brief Copy-assign an RMQ index and rebuild internal non-owning views. */ - CartesianTreeSegmentBTreeXLRmq& operator=( - const CartesianTreeSegmentBTreeXLRmq& other) { + CartesianHybridBTree& operator=(const CartesianHybridBTree& other) { if (this == &other) { return *this; } @@ -1327,8 +1332,7 @@ class CartesianTreeSegmentBTreeXLRmq /** * @brief Move an RMQ index and rebuild internal non-owning views. */ - CartesianTreeSegmentBTreeXLRmq( - CartesianTreeSegmentBTreeXLRmq&& other) noexcept + CartesianHybridBTree(CartesianHybridBTree&& other) noexcept : values_(other.values_), compare_(std::move(other.compare_)), bp_bits_(std::move(other.bp_bits_)), @@ -1348,8 +1352,7 @@ class CartesianTreeSegmentBTreeXLRmq /** * @brief Move-assign an RMQ index and rebuild internal non-owning views. */ - CartesianTreeSegmentBTreeXLRmq& operator=( - CartesianTreeSegmentBTreeXLRmq&& other) noexcept { + CartesianHybridBTree& operator=(CartesianHybridBTree&& other) noexcept { if (this == &other) { return *this; } @@ -1405,7 +1408,9 @@ class CartesianTreeSegmentBTreeXLRmq /** * @brief Return the packed BP words used by the RMQ encoding. */ - std::span bp_words() const { return bp_bits_; } + std::span bp_words() const { + return bp_storage_words().first(bp_word_count()); + } /** * @brief Return the top sparse-table block width chosen for a value count. @@ -1446,14 +1451,14 @@ class CartesianTreeSegmentBTreeXLRmq * input values are not owned and are excluded. */ std::size_t memory_usage_bytes_impl() const { - return sizeof(*this) + pixie::vector_capacity_bytes(bp_bits_) + + return sizeof(*this) + bp_bits_.allocated_bytes() + pixie::vector_capacity_bytes(top_sparse_candidates_) + pixie::optional_nested_owned_memory_bytes(bp_index_) + pixie::nested_owned_memory_bytes(bp_depth_rmq_); } private: - using BpDepthRmq = SegmentBTreeXLPlusMinusOneRmq; + using BpDepthRmq = detail::HybridBTreePlusMinusOne; struct TopCandidate { Index position = invalid_index; @@ -1485,7 +1490,7 @@ class CartesianTreeSegmentBTreeXLRmq * @brief Rebuild the BP Cartesian-tree representation and support indexes. */ void build() { - bp_bits_.clear(); + bp_bits_.resize(0); bp_bit_count_ = 0; top_sparse_candidates_.clear(); top_block_size_ = kMinTopSparseBlockSize; @@ -1498,11 +1503,12 @@ class CartesianTreeSegmentBTreeXLRmq } if (values_.size() > (static_cast(invalid_index) - 1) / 2) { throw std::length_error( - "Cartesian SegmentBTreeXL RMQ index type is too small"); + "CartesianHybridBTree RMQ index type is too small"); } bp_bit_count_ = 2 * values_.size(); - bp_bits_.assign((bp_bit_count_ + 63) / 64, 0); + bp_bits_.resize(padded_bp_bit_capacity()); + std::ranges::fill(bp_storage_words(), std::uint64_t{0}); build_bp_bits(); build_top_sparse_table(); reset_bp_indexes(); @@ -1512,17 +1518,16 @@ class CartesianTreeSegmentBTreeXLRmq * @brief Build the Ferrada-Navarro BP bits with a monotone stack. */ void build_bp_bits() { - const auto stack = std::make_unique_for_overwrite(values_.size()); - std::size_t stack_size = 0; + utils::SuccinctIncreasingStack stack(values_.size()); std::size_t write_position = bp_bit_count_; for (std::size_t i = values_.size(); i-- > 0;) { - while (stack_size != 0 && - !compare_(values_[stack[stack_size - 1]], values_[i])) { - --stack_size; + while (!stack.empty() && + !compare_(values_[stack_index(stack.top())], values_[i])) { + stack.pop(); prepend_bp_bit(write_position, true); } - stack[stack_size++] = static_cast(i); + stack.push(stack_key(i)); prepend_bp_bit(write_position, false); } @@ -1531,14 +1536,30 @@ class CartesianTreeSegmentBTreeXLRmq } } + /** + * @brief Convert a value position to the increasing key used by the + * construction stack. + */ + std::size_t stack_key(std::size_t value_index) const { + return values_.size() - 1 - value_index; + } + + /** + * @brief Convert a construction-stack key back to the original value + * position. + */ + std::size_t stack_index(std::size_t key) const { + return values_.size() - 1 - key; + } + /** * @brief Prepend one BP bit into the right-to-left construction buffer. */ void prepend_bp_bit(std::size_t& write_position, bool bit) { --write_position; if (bit) { - bp_bits_[write_position >> 6] |= std::uint64_t{1} - << (write_position & 63); + bp_storage_words()[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); } } @@ -1551,9 +1572,40 @@ class CartesianTreeSegmentBTreeXLRmq if (bp_bit_count_ == 0) { return; } - const std::span words(bp_bits_); - bp_index_.emplace(words, bp_bit_count_); - bp_depth_rmq_ = BpDepthRmq(words, bp_bit_count_ + 1, *bp_index_); + const std::span words = bp_words(); + const std::span padded_words = bp_storage_words(); + // TODO: try incorporating rank/select information into the tree. + bp_index_.emplace(words, bp_bit_count_, BitVector::SelectSupport::kSelect0, + values_.size()); + bp_depth_rmq_ = BpDepthRmq(padded_words, bp_bit_count_ + 1, *bp_index_); + } + + /** + * @brief Return the exact number of 64-bit words in the BP encoding. + */ + std::size_t bp_word_count() const { return ceil_div(bp_bit_count_, 64); } + + /** + * @brief Return BP storage capacity padded to full low-level depth leaves. + */ + std::size_t padded_bp_bit_capacity() const { + if (bp_bit_count_ == 0) { + return 0; + } + const std::size_t depth_count = bp_bit_count_ + 1; + return ceil_div(depth_count, LeafSize) * LeafSize; + } + + /** + * @brief Return mutable padded BP storage as 64-bit words. + */ + std::span bp_storage_words() { return bp_bits_.As64BitInts(); } + + /** + * @brief Return padded BP storage as 64-bit words. + */ + std::span bp_storage_words() const { + return bp_bits_.AsConst64BitInts(); } /** @@ -1745,7 +1797,7 @@ class CartesianTreeSegmentBTreeXLRmq std::span values_; Compare compare_; - std::vector bp_bits_; + pixie::AlignedStorage bp_bits_; std::size_t bp_bit_count_ = 0; std::vector top_sparse_candidates_; std::size_t top_block_size_ = kMinTopSparseBlockSize; @@ -1755,4 +1807,4 @@ class CartesianTreeSegmentBTreeXLRmq BpDepthRmq bp_depth_rmq_; }; -} // namespace pixie::rmq::experimental +} // namespace pixie::rmq diff --git a/include/pixie/rmq/experimental/cartesian_tree_rmm_btree_rmq.h b/include/pixie/rmq/cartesian_rmm.h similarity index 55% rename from include/pixie/rmq/experimental/cartesian_tree_rmm_btree_rmq.h rename to include/pixie/rmq/cartesian_rmm.h index 05f4771..eaee562 100644 --- a/include/pixie/rmq/experimental/cartesian_tree_rmm_btree_rmq.h +++ b/include/pixie/rmq/cartesian_rmm.h @@ -1,60 +1,90 @@ #pragma once /** - * Benchmark snapshot: experimental rmM-backed Cartesian/BP RMQ, 2026-06-10. + * rmM-backed Cartesian RMQ. * + * The benchmark harness registers this retained variant as + * `rmq_cartesian_rmm` and `rmq_build_cartesian_rmm`. + * It is included from `pixie/rmq.h` with the rest of the retained RMQ backends. + * + * Diagnostic snapshot, 2026-06-13: + * + * Focused value-RMQ rows, N=2^22 values, CPU mean across 5 repetitions. + * Command shape: + * taskset -c 0 ./build/release/bench_rmq + * --benchmark_filter='^rmq_(cartesian_rmm|sdsl_sct)/4194304/' + * + * | max width | CartesianRmM (ns) | SdslSct (ns) | + * | --------: | ----------------: | -----------: | + * | 64 | 121.0 | 211.0 | + * | 4096 | 310.0 | 757.0 | + * | 2^18 | 529.0 | 1057.0 | + * | 2^22 | 598.0 | 979.0 | + * + * Raw BP rmM rows at 2^23 bits, Q=32768, CPU mean across 5 repetitions. * Command shape: - * taskset -c 0 ./build/release/bench_rmq - * --benchmark_filter='^(rmq_cartesian_tree|rmq_cartesian_tree_rmm_btree| - * rmq_bp_plus_minus_one|rmq_bp_rmm_btree)/...|^(rmq_build_...)/...' - * --benchmark_min_time=0.30s - * --benchmark_out=/tmp/rmq_rmm_btree_20260610.json + * ./build/release/bench_rmm_{btree,sdsl} + * --ops=range_min_query_pos,range_min_query_val + * --explicit_sizes=8388608 --Q=32768 + * + * | backend | range_min_pos (ns) | range_min_val (ns) | + * | ------------------- | -----------------: | -----------------: | + * | RmMBTree | 602.0 | 540.0 | + * | SdslRmMTree | 727.0 | 742.0 | * - * Query CPU time, ns: - * | N | width | cartesian | cartesian+rmm | bp +/-1 | bp+rmm | - * |----------|----------|-----------|---------------|---------|---------| - * | 2^10 | 64 | 80.5 | 155.5 | 42.8 | 107.1 | - * |----------|----------|-----------|---------------|---------|---------| - * | 2^14 | 64 | 98.8 | 177.4 | 42.7 | 117.9 | - * | 2^14 | 4096 | 123.9 | 722.2 | 69.0 | 634.9 | - * |----------|----------|-----------|---------------|---------|---------| - * | 2^18 | 64 | 110.8 | 176.7 | 42.5 | 118.1 | - * | 2^18 | 4096 | 156.0 | 744.6 | 66.6 | 735.1 | - * | 2^18 | 2^18 | 165.8 | 1222.1 | 67.6 | 1204.5 | - * |----------|----------|-----------|---------------|---------|---------| - * | 2^22 | 64 | 199.1 | 328.6 | 46.5 | 131.7 | - * | 2^22 | 4096 | 327.0 | 1271.9 | 95.8 | 772.3 | - * | 2^22 | 2^22 | 312.8 | 1601.3 | 88.6 | 1498.6 | - * |----------|----------|-----------|---------------|---------|---------| - * | 2^24 | 64 | 272.5 | 396.2 | 77.0 | 202.9 | - * | 2^24 | 4096 | 500.1 | 1248.3 | 189.6 | 1038.2 | - * | 2^24 | 2^24 | 483.5 | 1832.2 | 126.7 | 1687.9 | + * The 2026-06-13 optimization pass streams cover nodes instead of materializing + * a zero-initialized per-query cover, adds a combined min position/value query + * for the Cartesian adapter, and uses a minimum-only boundary scanner for + * range-min position/value queries. Current perf samples for raw + * range_min_query_pos are concentrated in for_each_cover_node(), + * scan_min_range(), and summary_at(). + * + * Historical build snapshot after select0-only BP rank/select construction, + * 2026-06-13, before the RmMBTree full-block summary fast path. + * Command shape: + * taskset -c 0 ./build/release/bench_rmq + * --benchmark_filter='^rmq_build_(cartesian_rmm|cartesian_hybrid_btree|sdsl_sct)/(4194304|67108864)$' + * --benchmark_repetitions=5 * - * Build CPU time, ms: - * | N | cartesian | cartesian+rmm | bp +/-1 | bp+rmm | - * |----------|-----------|---------------|---------|--------| - * | 2^10 | 0.006 | 0.008 | 0.000 | 0.005 | - * | 2^14 | 0.053 | 0.074 | 0.002 | 0.026 | - * | 2^18 | 1.770 | 2.243 | 0.047 | 0.377 | - * | 2^22 | 43.794 | 35.318 | 1.230 | 6.164 | - * | 2^24 | 192.036 | 155.852 | 20.232 | 23.921 | + * CPU mean, milliseconds. + * + * | N | CartesianRmM | CartesianHybrid | SdslSct | + * | ---: | -----------: | --------------: | ------: | + * | 2^22 | 54.407 | 49.361 | 41.317 | + * | 2^26 | 915.712 | 795.562 | 666.048 | + * + * Historical construction perf profile snapshot, 2026-06-13, before the + * RmMBTree full-block summary fast path. + * Profiling build: RelWithDebInfo, -O3, debug info, frame pointers. Rows are + * N=2^26 build benchmarks sampled with perf at 999 Hz. + * + * CartesianHybridBTree: 867 ms CPU in the profiled run. BP construction is the + * dominant cost: the succinct monotone-stack operations plus BP writes account + * for roughly two thirds of samples. Top sparse-table construction is about 6%. + * CartesianRmM: 1013 ms CPU in the profiled run. BP construction was still the + * dominant cost, while rmM summary construction added a visible second cost: + * summarize_bits()/bit() lines accounted for about 14%. Benchmark dataset + * generation was about 5% in both rows. */ #include #include +#include #include #include #include #include -#include +#include #include #include #include #include #include -namespace pixie::rmq::experimental { +namespace pixie::rmq { + +namespace detail { /** * @brief Depth RMQ adapter over the experimental rmM btree. @@ -71,17 +101,17 @@ namespace pixie::rmq::experimental { template -class RmMBTreePlusMinusOneRmq { +class RmMPlusMinusOne { public: static_assert(std::is_unsigned_v, - "RmMBTreePlusMinusOneRmq index type must be unsigned"); + "RmMPlusMinusOne index type must be unsigned"); static constexpr std::size_t npos = std::numeric_limits::max(); static constexpr Index invalid_index = std::numeric_limits::max(); static constexpr std::size_t kBlockSize = pixie::experimental::RmMBTree::kBlockBits; - RmMBTreePlusMinusOneRmq() = default; + RmMPlusMinusOne() = default; /** * @brief Build an rmM-backed ±1 RMQ over external packed delta bits. @@ -91,15 +121,36 @@ class RmMBTreePlusMinusOneRmq { * @throws std::length_error if @p Index cannot represent all positions. * @throws std::invalid_argument if @p bits is shorter than required. */ - RmMBTreePlusMinusOneRmq(std::span bits, - std::size_t depth_count) { + RmMPlusMinusOne(std::span bits, + std::size_t depth_count) { build(bits, depth_count); } + /** + * @brief Build with an exact count of +1 delta bits. + * + * @details Cartesian BP construction knows this count exactly, which lets the + * nested BitVector allocate select samples without a second scan. + */ + RmMPlusMinusOne(std::span bits, + std::size_t depth_count, + std::size_t one_count) { + build(bits, depth_count, one_count); + } + /** * @brief Rebuild this adapter over external packed delta bits. */ void build(std::span bits, std::size_t depth_count) { + build(bits, depth_count, std::nullopt); + } + + /** + * @brief Rebuild this adapter with an optional exact count of +1 delta bits. + */ + void build(std::span bits, + std::size_t depth_count, + std::optional one_count) { words_ = bits; depth_count_ = depth_count; rmm_ = RmM(); @@ -110,7 +161,8 @@ class RmMBTreePlusMinusOneRmq { if (depth_count_ > static_cast(invalid_index)) { throw std::length_error("RMQ rmM btree index type is too small"); } - rmm_ = RmM(words_, depth_count_ - 1); + rmm_ = RmM(words_, depth_count_ - 1, BitVector::SelectSupport::kSelect0, + one_count); } /** @@ -139,14 +191,13 @@ class RmMBTreePlusMinusOneRmq { const std::size_t bit_left = left; const std::size_t bit_right = right - 2; - const int minimum_after_left = - rmm_.range_min_query_val(bit_left, bit_right); - if (minimum_after_left >= 0) { + const auto minimum_after_left = + rmm_.range_min_query_result(bit_left, bit_right); + if (minimum_after_left.value >= 0) { return left; } - const std::size_t bit_position = - rmm_.range_min_query_pos(bit_left, bit_right); + const std::size_t bit_position = minimum_after_left.position; return bit_position == RmM::npos ? npos : bit_position + 1; } @@ -170,54 +221,52 @@ class RmMBTreePlusMinusOneRmq { RmM rmm_; }; +} // namespace detail + /** - * @brief Experimental Ferrada-Navarro Cartesian-tree RMQ using rmM support. + * @brief Ferrada-Navarro Cartesian-tree RMQ using rmM support. * - * @details This class keeps the same public value-RMQ contract as - * `CartesianTreeRmq`, but replaces the `BitVector` + `BpPlusMinusOneRmq` - * support layer with `RmMBTreePlusMinusOneRmq`, which in turn uses the - * experimental range min-max btree over the BP excess sequence. + * @details This class keeps the same public value-RMQ contract as the other + * value RMQ backends, but replaces the usual balanced-parentheses rank/select + * and depth-RMQ support with `detail::RmMPlusMinusOne`, which in turn uses the + * experimental range min-max btree over the BP excess sequence. BP + * construction uses a succinct monotone bit-stack, preserving the same stable + * Cartesian-tree shape without an n-entry index stack. * - * This is intentionally not included from `pixie/rmq.h`; it exists to benchmark - * the Ferrada-Navarro BP formula with an rmM-backed `rmq_D` primitive before - * deciding whether to promote or optimize it. + * This implementation is included from `pixie/rmq.h` as the rmM-backed + * Cartesian-tree reduction. */ template , class Index = std::size_t, std::size_t HighCacheLines = 4, std::size_t LowFanout = 32> -class CartesianTreeRmMBTreeRmq - : public RmqBase, +class CartesianRmM + : public RmqBase, T> { public: static_assert(std::is_unsigned_v, - "CartesianTreeRmMBTreeRmq index type must be unsigned"); + "CartesianRmM index type must be unsigned"); - static constexpr std::size_t npos = RmqBase< - CartesianTreeRmMBTreeRmq, - T>::npos; + static constexpr std::size_t npos = + RmqBase, + T>::npos; static constexpr Index invalid_index = std::numeric_limits::max(); - CartesianTreeRmMBTreeRmq() = default; + CartesianRmM() = default; /** - * @brief Build an experimental Cartesian-tree RMQ index over @p values. + * @brief Build a Cartesian-tree RMQ index over @p values. * * @details The values are not copied and must outlive this object. Equal * values stay stable: the smaller index remains the first minimum. */ - explicit CartesianTreeRmMBTreeRmq(std::span values, - Compare compare = Compare()) + explicit CartesianRmM(std::span values, Compare compare = Compare()) : values_(values), compare_(compare) { build(); } - CartesianTreeRmMBTreeRmq(const CartesianTreeRmMBTreeRmq& other) + CartesianRmM(const CartesianRmM& other) : values_(other.values_), compare_(other.compare_), bp_bits_(other.bp_bits_), @@ -225,7 +274,7 @@ class CartesianTreeRmMBTreeRmq reset_bp_index(); } - CartesianTreeRmMBTreeRmq& operator=(const CartesianTreeRmMBTreeRmq& other) { + CartesianRmM& operator=(const CartesianRmM& other) { if (this == &other) { return *this; } @@ -237,7 +286,7 @@ class CartesianTreeRmMBTreeRmq return *this; } - CartesianTreeRmMBTreeRmq(CartesianTreeRmMBTreeRmq&& other) noexcept + CartesianRmM(CartesianRmM&& other) noexcept : values_(other.values_), compare_(std::move(other.compare_)), bp_bits_(std::move(other.bp_bits_)), @@ -247,8 +296,7 @@ class CartesianTreeRmMBTreeRmq reset_bp_index(); } - CartesianTreeRmMBTreeRmq& operator=( - CartesianTreeRmMBTreeRmq&& other) noexcept { + CartesianRmM& operator=(CartesianRmM&& other) noexcept { if (this == &other) { return *this; } @@ -306,7 +354,7 @@ class CartesianTreeRmMBTreeRmq std::span bp_words() const { return bp_bits_; } private: - using BpSupport = RmMBTreePlusMinusOneRmq; + using BpSupport = detail::RmMPlusMinusOne; void build() { bp_bits_.clear(); @@ -327,17 +375,16 @@ class CartesianTreeRmMBTreeRmq } void build_bp_bits() { - const auto stack = std::make_unique_for_overwrite(values_.size()); - std::size_t stack_size = 0; + utils::SuccinctIncreasingStack stack(values_.size()); std::size_t write_position = bp_bit_count_; for (std::size_t i = values_.size(); i-- > 0;) { - while (stack_size != 0 && - !compare_(values_[stack[stack_size - 1]], values_[i])) { - --stack_size; + while (!stack.empty() && + !compare_(values_[stack_index(stack.top())], values_[i])) { + stack.pop(); prepend_bp_bit(write_position, true); } - stack[stack_size++] = static_cast(i); + stack.push(stack_key(i)); prepend_bp_bit(write_position, false); } @@ -346,6 +393,22 @@ class CartesianTreeRmMBTreeRmq } } + /** + * @brief Convert a value position to the increasing key used by the + * construction stack. + */ + std::size_t stack_key(std::size_t value_index) const { + return values_.size() - 1 - value_index; + } + + /** + * @brief Convert a construction-stack key back to the original value + * position. + */ + std::size_t stack_index(std::size_t key) const { + return values_.size() - 1 - key; + } + void prepend_bp_bit(std::size_t& write_position, bool bit) { --write_position; if (bit) { @@ -359,8 +422,8 @@ class CartesianTreeRmMBTreeRmq if (bp_bit_count_ == 0) { return; } - bp_support_ = - BpSupport(std::span(bp_bits_), bp_bit_count_ + 1); + bp_support_ = BpSupport(std::span(bp_bits_), + bp_bit_count_ + 1, values_.size()); } std::span values_; @@ -370,4 +433,4 @@ class CartesianTreeRmMBTreeRmq BpSupport bp_support_; }; -} // namespace pixie::rmq::experimental +} // namespace pixie::rmq diff --git a/include/pixie/rmq/cartesian_tree_rmq.h b/include/pixie/rmq/cartesian_tree_rmq.h deleted file mode 100644 index f1261b2..0000000 --- a/include/pixie/rmq/cartesian_tree_rmq.h +++ /dev/null @@ -1,316 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace pixie::rmq { - -/** - * @brief General RMQ via the Ferrada-Navarro BP Cartesian-tree encoding. - * - * @details Builds the balanced-parentheses representation of the Cartesian-tree - * RMQ information directly from the indexed values. The construction scans the - * input right-to-left with a monotone stack and emits the BP sequence used by - * the Ferrada-Navarro formulation. Queries map the half-open value interval to - * positions in that BP sequence with `select0`, run a ±1 RMQ over BP prefix - * excess, and map the winning BP position back to an array index with `rank0`. - * - * The indexed values are not owned and must outlive this object. `arg_min()` - * answers from the BP indexes alone; `range_min()` still reads the external - * value span after the position is known. Equal values are handled stably: the - * smaller original position remains the first minimum. - * - * This implementation is a practical BP Cartesian-tree backend, not a fully - * compressed 2n + o(n)-bit object: it stores the 2n BP bits plus the supporting - * `BitVector` rank/select index and a `BpPlusMinusOneRmq` over BP excess. - * - * References: - * - Johannes Fischer, "Optimal Succinctness for Range Minimum Queries", - * LATIN 2010; arXiv:0812.2775. - * - Hector Ferrada and Gonzalo Navarro, "Improved Range Minimum Queries", - * Data Compression Conference 2016; Journal of Discrete Algorithms 43 - * (2017), doi:10.1016/j.jda.2016.09.002. - * - * @tparam T Value type in the indexed array. - * @tparam Compare Strict weak ordering used to choose minima. - * @tparam Index Unsigned integer type used for stored positions. - */ -template , class Index = std::size_t> -class CartesianTreeRmq - : public RmqBase, T> { - public: - static_assert(std::is_unsigned_v, - "CartesianTreeRmq index type must be unsigned"); - - static constexpr std::size_t npos = - RmqBase, T>::npos; - static constexpr Index invalid_index = std::numeric_limits::max(); - - /** - * @brief Construct an empty Cartesian-tree RMQ index. - */ - CartesianTreeRmq() = default; - - /** - * @brief Build a Cartesian-tree RMQ index over @p values. - * - * @details The values are not copied and must outlive this object. Equal - * values stay stable: the smaller index remains the first minimum. - * - * @param values Values to index. - * @param compare Ordering used to choose minima. - * @throws std::length_error if @p Index cannot represent all positions. - */ - explicit CartesianTreeRmq(std::span values, - Compare compare = Compare()) - : values_(values), compare_(compare) { - build(); - } - - /** - * @brief Copy an RMQ index and rebuild internal non-owning views. - * - * @param other Source index. - */ - CartesianTreeRmq(const CartesianTreeRmq& other) - : values_(other.values_), - compare_(other.compare_), - bp_bits_(other.bp_bits_), - bp_bit_count_(other.bp_bit_count_) { - reset_bp_indexes(); - } - - /** - * @brief Copy-assign an RMQ index and rebuild internal non-owning views. - * - * @param other Source index. - * @return Reference to this object. - */ - CartesianTreeRmq& operator=(const CartesianTreeRmq& other) { - if (this == &other) { - return *this; - } - values_ = other.values_; - compare_ = other.compare_; - bp_bits_ = other.bp_bits_; - bp_bit_count_ = other.bp_bit_count_; - reset_bp_indexes(); - return *this; - } - - /** - * @brief Move an RMQ index and rebuild internal non-owning views. - * - * @param other Source index. - */ - CartesianTreeRmq(CartesianTreeRmq&& other) noexcept - : values_(other.values_), - compare_(std::move(other.compare_)), - bp_bits_(std::move(other.bp_bits_)), - bp_bit_count_(other.bp_bit_count_) { - other.values_ = std::span(); - other.bp_bit_count_ = 0; - reset_bp_indexes(); - } - - /** - * @brief Move-assign an RMQ index and rebuild internal non-owning views. - * - * @param other Source index. - * @return Reference to this object. - */ - CartesianTreeRmq& operator=(CartesianTreeRmq&& other) noexcept { - if (this == &other) { - return *this; - } - values_ = other.values_; - compare_ = std::move(other.compare_); - bp_bits_ = std::move(other.bp_bits_); - bp_bit_count_ = other.bp_bit_count_; - other.values_ = std::span(); - other.bp_bit_count_ = 0; - reset_bp_indexes(); - return *this; - } - - /** - * @brief Return the number of indexed values. - * - * @return `values.size()` from construction. - */ - std::size_t size_impl() const { return values_.size(); } - - /** - * @brief Return the value at an indexed position. - * - * @param position Zero-based position in the indexed values. - * @return Copy of the value at @p position. - */ - T value_at_impl(std::size_t position) const { return values_[position]; } - - /** - * @brief Return the first minimum position in [@p left, @p right). - * - * @details Adapts the Ferrada-Navarro BP formula to Pixie's zero-based - * half-open ranges. Ties return the smaller original position. - * - * @param left First position in the query range. - * @param right One past the last position in the query range. - * @return Zero-based position of the first range minimum, or `npos`. - */ - std::size_t arg_min_impl(std::size_t left, std::size_t right) const { - if (left >= right || right > values_.size()) { - return npos; - } - const BitVector& bp_index = *bp_index_; - const std::size_t first_close = bp_index.select0(left + 1); - const std::size_t last_close = bp_index.select0(right); - if (first_close == bp_bit_count_ || last_close == bp_bit_count_ || - first_close > last_close) { - return npos; - } - - const std::size_t shifted_min = - bp_depth_rmq_.arg_min(first_close + 1, last_close + 2); - if (shifted_min == npos || shifted_min == 0) { - return npos; - } - const std::size_t answer = bp_index.rank0(shifted_min) - 1; - return answer < values_.size() ? answer : npos; - } - - /** - * @brief Return the number of BP bits in the Cartesian-tree RMQ encoding. - * - * @return Logical BP bit count, equal to `2 * size()`. - */ - std::size_t bp_bit_count() const { return bp_bit_count_; } - - /** - * @brief Return the packed BP words used by the RMQ encoding. - * - * @return Non-owning span of little-endian packed BP words. - */ - std::span bp_words() const { return bp_bits_; } - - /** - * @brief Return owned auxiliary memory usage in bytes. - * - * @details Counts this Cartesian-tree object, its packed BP words, and the - * nested BP rank/select and ±1 RMQ indexes. The external input values are not - * owned and are excluded. - */ - std::size_t memory_usage_bytes_impl() const { - return sizeof(*this) + pixie::vector_capacity_bytes(bp_bits_) + - pixie::optional_nested_owned_memory_bytes(bp_index_) + - pixie::nested_owned_memory_bytes(bp_depth_rmq_); - } - - private: - /** - * @brief Rebuild the direct BP Cartesian-tree RMQ representation. - * - * @details Constructs the Ferrada-Navarro BP sequence with a right-to-left - * monotone stack, then rebuilds select/rank and excess-min indexes over the - * packed BP words. - * - * @throws std::length_error if @p Index cannot represent all positions. - */ - void build() { - bp_bits_.clear(); - bp_bit_count_ = 0; - reset_bp_indexes(); - - if (values_.empty()) { - return; - } - if (values_.size() > (static_cast(invalid_index) - 1) / 2) { - throw std::length_error("Cartesian RMQ index type is too small"); - } - - bp_bit_count_ = 2 * values_.size(); - bp_bits_.assign((bp_bit_count_ + 63) / 64, 0); - build_bp_bits(); - reset_bp_indexes(); - } - - /** - * @brief Build the BP bits with the Ferrada-Navarro stack construction. - * - * @details The paper describes prepending bits while scanning right-to-left. - * This implementation fills the destination bitvector from right to left. - */ - void build_bp_bits() { - const auto stack = std::make_unique_for_overwrite(values_.size()); - std::size_t stack_size = 0; - std::size_t write_position = bp_bit_count_; - - for (std::size_t i = values_.size(); i-- > 0;) { - while (stack_size != 0 && - !compare_(values_[stack[stack_size - 1]], values_[i])) { - --stack_size; - prepend_bp_bit(write_position, true); - } - stack[stack_size++] = static_cast(i); - prepend_bp_bit(write_position, false); - } - - while (write_position != 0) { - prepend_bp_bit(write_position, true); - } - } - - /** - * @brief Prepend one BP bit into the right-to-left construction buffer. - * - * @param write_position Current first unwritten position; decremented here. - * @param bit `true` for an opening parenthesis and `false` for closing. - */ - void prepend_bp_bit(std::size_t& write_position, bool bit) { - --write_position; - if (bit) { - bp_bits_[write_position >> 6] |= std::uint64_t{1} - << (write_position & 63); - } - } - - /** - * @brief Rebuild indexes that store non-owning spans into `bp_bits_`. - * - * @details Called after build, copy, and move operations because both indexes - * keep views into this object's packed BP word storage. - */ - void reset_bp_indexes() { - bp_index_.reset(); - bp_depth_rmq_ = BpPlusMinusOneRmq(); - if (bp_bit_count_ == 0) { - return; - } - const std::span words(bp_bits_); - bp_index_.emplace(words, bp_bit_count_); - bp_depth_rmq_ = BpPlusMinusOneRmq(words, bp_bit_count_ + 1); - } - - std::span values_; - Compare compare_; - std::vector bp_bits_; - std::size_t bp_bit_count_ = 0; - std::optional bp_index_; - BpPlusMinusOneRmq bp_depth_rmq_; -}; - -} // namespace pixie::rmq diff --git a/include/pixie/rmq/experimental/node_euler_btree_rmq.h b/include/pixie/rmq/experimental/node_euler_btree_rmq.h deleted file mode 100644 index bdd77a6..0000000 --- a/include/pixie/rmq/experimental/node_euler_btree_rmq.h +++ /dev/null @@ -1,1344 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace pixie::rmq::experimental { - -/** - * @brief Low-level selector tag for BP-based 252-value leaves. - */ -struct BpLeafSelectorTag {}; - -/** - * @brief Low-level selector tag for prefix/suffix mask leaves. - */ -struct PrefixSuffixMaskLeafSelectorTag {}; - -/** - * @brief Experimental value RMQ with compact per-level BP selectors. - * - * @details The default low level uses 252-value leaves with local BP selectors. - * The prefix/suffix mask low level supports 248-value leaves with an 8-bit - * local-minimum offset packed into 32 bytes, or 496-value leaves with a 16-bit - * local-minimum offset packed into 64 bytes. Middle internal nodes use 192-way - * fanout and store 384 BP bits, a 64-bit absolute - * subtree-minimum position, and 64 bits of zero-rank prefix metadata in the - * selector, with a side cache for each middle node's minimum value. High nodes - * use 256-way fanout and are only the root and its child level, so there are at - * most 257 of them. High nodes add sparse tables over child-minimum slots and - * cache each child slot's value range and subtree minimum. - * - * This backend is intentionally not included by `pixie/rmq.h`; include this - * header directly while evaluating the experiment. - * - * @tparam T Value type in the indexed array. - * @tparam Compare Strict weak ordering used to choose minima. - * @tparam Index Unsigned integer type used for stored positions. - * @tparam LeafSize Number of original values per leaf. This experimental - * backend currently supports 252 with BP leaves and 248 or 496 with mask - * leaves. - * @tparam Fanout Number of children per high internal node. This experimental - * backend currently supports only 256. Middle internal nodes use 192. - * @tparam LowLevelSelector Compile-time low-level selector tag. - */ -template , - class Index = std::size_t, - std::size_t LeafSize = 252, - std::size_t Fanout = 256, - class LowLevelSelector = BpLeafSelectorTag> -class NodeEulerBTreeRmq : public RmqBase, - T> { - public: - static_assert(std::is_unsigned_v, - "NodeEulerBTreeRmq index type must be unsigned"); - static constexpr bool kBpLeafSelector = - std::is_same_v; - static constexpr bool kMaskLeafSelector = - std::is_same_v; - static_assert((kBpLeafSelector && LeafSize == 252) || - (kMaskLeafSelector && (LeafSize == 248 || LeafSize == 496)), - "experimental NodeEulerBTreeRmq requires 252-value BP leaves " - "or 248/496-value prefix/suffix mask leaves"); - static_assert(Fanout == 256, - "experimental NodeEulerBTreeRmq currently requires 256-way " - "internal nodes"); - static_assert(kBpLeafSelector || kMaskLeafSelector, - "unsupported experimental low-level selector tag"); - - using Self = - NodeEulerBTreeRmq; - - static constexpr std::size_t npos = RmqBase::npos; - static constexpr Index invalid_index = std::numeric_limits::max(); - static constexpr std::size_t kLeafSize = LeafSize; - static constexpr std::size_t kFanout = Fanout; - static constexpr std::size_t kMiddleFanout = 192; - - /** - * @brief Construct an empty experimental RMQ index. - */ - NodeEulerBTreeRmq() = default; - NodeEulerBTreeRmq(const NodeEulerBTreeRmq&) = default; - NodeEulerBTreeRmq(NodeEulerBTreeRmq&&) noexcept = default; - NodeEulerBTreeRmq& operator=(const NodeEulerBTreeRmq&) = default; - NodeEulerBTreeRmq& operator=(NodeEulerBTreeRmq&&) noexcept = default; - - /** - * @brief Build an experimental B-tree RMQ index over @p values. - * - * @details The values are not copied and must outlive this object. Equal - * values keep the smaller original position as the answer. - * - * @param values Values to index. - * @param compare Ordering used to choose minima. - * @throws std::length_error if @p Index cannot represent all positions. - */ - explicit NodeEulerBTreeRmq(std::span values, - Compare compare = Compare()) - : values_(values), compare_(compare) { - build(); - } - - /** - * @brief Return the number of indexed values. - */ - std::size_t size_impl() const { return values_.size(); } - - /** - * @brief Return the value at an indexed position. - */ - T value_at_impl(std::size_t position) const { return values_[position]; } - - /** - * @brief Return the first minimum position in [@p left, @p right). - * - * @details Empty or invalid ranges return `npos`. Ties are reduced by - * comparing `(value, position)`, so traversal order cannot change first-min - * semantics. - */ - std::size_t arg_min_impl(std::size_t left, std::size_t right) const { - if (left >= right || right > values_.size() || level_sizes_.empty()) { - return npos; - } - - const std::size_t root_level = level_count() - 1; - if (left == 0 && right == values_.size()) { - return subtree_min_position(root_level, 0); - } - - const std::size_t left_leaf = leaf_for_value(left); - const std::size_t right_leaf = leaf_for_value(right - 1); - if (left_leaf == right_leaf) { - return leaf_range_min(left_leaf, left, right); - } - - const auto [level, node_index] = covering_node(left_leaf, right_leaf); - return query_node(level, node_index, left, right).position; - } - - private: - static constexpr std::size_t kSelectorEntries = 256; - static constexpr std::size_t kSelectorBits = 2 * kSelectorEntries; - static constexpr std::size_t kSelectorWords = kSelectorBits / 64; - static constexpr std::size_t kEmbeddedOffsetEntries = 252; - static constexpr std::size_t kEmbeddedPositionEntries = 192; - static constexpr std::size_t kEmbeddedOffsetBit = 2 * kEmbeddedOffsetEntries; - static constexpr std::size_t kEmbeddedPositionBit = - 2 * kEmbeddedPositionEntries; - static constexpr std::size_t kMiddleZeroPrefixWord = 7; - static constexpr std::size_t kHighSparseLevels = - static_cast(std::bit_width(Fanout)); - static constexpr std::size_t kHighSparseSlotsPerNode = - kHighSparseLevels * Fanout; - static constexpr std::size_t kLeafLinearScanThreshold = 64; - static constexpr std::size_t kLeafAvx2ScanThreshold = 16; - static constexpr std::uint64_t kEmbeddedOffsetMask = - std::uint64_t{0xFF} << (kEmbeddedOffsetBit & 63); - static constexpr bool kInvalidIndexEqualsNpos = - static_cast(invalid_index) == npos; - - static_assert(kEmbeddedOffsetBit + 8 == kSelectorBits); - static_assert(kEmbeddedPositionBit + 128 == kSelectorBits); - static_assert(!kBpLeafSelector || LeafSize <= kEmbeddedOffsetEntries); - static_assert(kMiddleFanout <= kEmbeddedPositionEntries); - - struct HighChildMetadata { - std::size_t value_begin = 0; - std::size_t value_end = 0; - Index min_position = invalid_index; - }; - - struct MinCandidate { - std::size_t position = npos; - const T* value = nullptr; - }; - - class alignas(64) Bp512Selector { - public: - Bp512Selector() = default; - - template - void build(std::size_t entry_count, EntryLess entry_less) { - if (entry_count > kSelectorEntries) { - throw std::length_error("NodeEulerBTreeRmq local selector too large"); - } - - bp_bits_.fill(0); - if (entry_count == 0) { - return; - } - - std::array stack{}; - std::size_t stack_size = 0; - std::size_t write_position = 2 * entry_count; - - for (std::size_t i = entry_count; i-- > 0;) { - while (stack_size != 0 && !entry_less(stack[stack_size - 1], i)) { - --stack_size; - prepend_bp_bit(write_position, true); - } - stack[stack_size++] = static_cast(i); - prepend_bp_bit(write_position, false); - } - - while (write_position != 0) { - prepend_bp_bit(write_position, true); - } - } - - std::size_t arg_min(std::size_t slot_left, - std::size_t slot_right, - std::size_t entry_count) const { - if (slot_left >= slot_right || slot_right > entry_count || - entry_count > kSelectorEntries) { - return npos; - } - if (slot_left + 1 == slot_right) { - return slot_left; - } - - const std::size_t bit_count = 2 * entry_count; - const std::size_t first_close = close_position(slot_left); - const std::size_t last_close = close_position(slot_right - 1); - if (first_close > last_close || last_close >= bit_count) { - return npos; - } - - const std::size_t shifted_min = - depth_arg_min(first_close + 1, last_close + 2, bit_count); - if (shifted_min == npos || shifted_min == 0) { - return npos; - } - - const std::size_t zero_rank = rank0_at(shifted_min, bit_count); - if (zero_rank == 0) { - return npos; - } - const std::size_t entry = zero_rank - 1; - return entry < entry_count ? entry : npos; - } - - std::size_t arg_min_with_zero_prefix(std::size_t slot_left, - std::size_t slot_right, - std::size_t entry_count) const { - if (slot_left >= slot_right || slot_right > entry_count || - entry_count > kEmbeddedPositionEntries) { - return npos; - } - if (slot_left + 1 == slot_right) { - return slot_left; - } - - const std::size_t bit_count = 2 * entry_count; - const std::size_t first_close = - close_position_with_zero_prefix(slot_left, bit_count); - const std::size_t last_close = - close_position_with_zero_prefix(slot_right - 1, bit_count); - if (first_close > last_close || last_close >= bit_count) { - return npos; - } - - const std::size_t shifted_min = depth_arg_min_with_zero_prefix( - first_close + 1, last_close + 2, bit_count); - if (shifted_min == npos || shifted_min == 0) { - return npos; - } - - const std::size_t zero_rank = - rank0_at_with_zero_prefix(shifted_min, bit_count); - if (zero_rank == 0) { - return npos; - } - const std::size_t entry = zero_rank - 1; - return entry < entry_count ? entry : npos; - } - - std::size_t close_position(std::size_t slot) const { - return select0_512(bp_bits_.data(), slot); - } - - std::size_t rank0_at(std::size_t position, std::size_t bit_count) const { - position = std::min(position, bit_count); - return position - rank_512(bp_bits_.data(), position); - } - - void set_embedded_min_offset(std::size_t offset) { - bp_bits_[kEmbeddedOffsetBit >> 6] = - (bp_bits_[kEmbeddedOffsetBit >> 6] & ~kEmbeddedOffsetMask) | - ((static_cast(offset) & 0xFFu) - << (kEmbeddedOffsetBit & 63)); - } - - std::uint8_t embedded_min_offset() const { - return static_cast( - (bp_bits_[kEmbeddedOffsetBit >> 6] & kEmbeddedOffsetMask) >> - (kEmbeddedOffsetBit & 63)); - } - - void set_embedded_min_position(std::size_t position) { - bp_bits_[kEmbeddedPositionBit >> 6] = - static_cast(position); - } - - std::size_t embedded_min_position() const { - return static_cast(bp_bits_[kEmbeddedPositionBit >> 6]); - } - - void build_zero_prefix_metadata(std::size_t bit_count) { - const std::size_t word_count = (bit_count + 63) / 64; - std::uint64_t packed = 0; - std::size_t zeros = 0; - for (std::size_t word = 0; word <= word_count; ++word) { - packed |= (static_cast(zeros) & 0xFFu) << (8 * word); - if (word == word_count) { - break; - } - const std::size_t word_begin = word * 64; - const std::size_t word_bits = - std::min(64, bit_count - word_begin); - const std::uint64_t bits = bp_bits_[word] & first_bits_mask(word_bits); - zeros += word_bits - std::popcount(bits); - } - bp_bits_[kMiddleZeroPrefixWord] = packed; - } - - private: - std::size_t prepend_bp_bit(std::size_t& write_position, bool bit) { - --write_position; - if (bit) { - bp_bits_[write_position >> 6] |= std::uint64_t{1} - << (write_position & 63); - } - return write_position; - } - - int prefix_excess(std::size_t position) const { - position = std::min(position, kSelectorBits); - const std::size_t ones = rank_512(bp_bits_.data(), position); - return static_cast(2 * ones) - static_cast(position); - } - - std::size_t zero_prefix_at_word(std::size_t word) const { - return static_cast(bp_bits_[kMiddleZeroPrefixWord] >> - (8 * word)); - } - - std::size_t rank0_at_with_zero_prefix(std::size_t position, - std::size_t bit_count) const { - position = std::min(position, bit_count); - const std::size_t full_words = position >> 6; - std::size_t zeros = zero_prefix_at_word(full_words); - const std::size_t tail_bits = position & 63; - if (tail_bits != 0) { - const std::uint64_t tail = - bp_bits_[full_words] & first_bits_mask(tail_bits); - zeros += tail_bits - std::popcount(tail); - } - return zeros; - } - - int prefix_excess_with_zero_prefix(std::size_t position, - std::size_t bit_count) const { - position = std::min(position, bit_count); - return static_cast(position) - - 2 * static_cast( - rank0_at_with_zero_prefix(position, bit_count)); - } - - std::size_t close_position_with_zero_prefix(std::size_t slot, - std::size_t bit_count) const { - const std::size_t word_count = (bit_count + 63) / 64; - for (std::size_t word = 0; word < word_count; ++word) { - const std::size_t next_zero_prefix = zero_prefix_at_word(word + 1); - if (next_zero_prefix <= slot) { - continue; - } - const std::size_t local_rank = slot - zero_prefix_at_word(word); - const std::size_t word_begin = word * 64; - const std::size_t word_bits = - std::min(64, bit_count - word_begin); - const std::uint64_t zeros = - (~bp_bits_[word]) & first_bits_mask(word_bits); - return word_begin + select_64(zeros, local_rank); - } - return npos; - } - - std::size_t depth_arg_min(std::size_t left, - std::size_t right, - std::size_t bit_count) const { - const std::size_t depth_count = bit_count + 1; - if (left >= right || right > depth_count) { - return npos; - } - - std::size_t position = left; - int best_depth = prefix_excess(position); - std::size_t best_position = position; - - while (position < right) { - const std::size_t chunk_begin = (position / 128) * 128; - const std::size_t local_left = position - chunk_begin; - const std::size_t local_right = - std::min(right - 1, chunk_begin + 128) - chunk_begin; - - int candidate_depth; - std::size_t candidate_position; - if (chunk_begin >= bit_count) { - candidate_depth = prefix_excess(chunk_begin); - candidate_position = chunk_begin; - } else { - const std::size_t word = chunk_begin >> 6; - const ExcessResult result = - excess_min_128(bp_bits_.data() + word, local_left, local_right); - candidate_depth = prefix_excess(chunk_begin) + result.min_excess; - candidate_position = chunk_begin + result.offset; - } - - if (candidate_depth < best_depth) { - best_depth = candidate_depth; - best_position = candidate_position; - } - - position = chunk_begin + local_right + 1; - } - - return best_position; - } - - std::size_t depth_arg_min_with_zero_prefix(std::size_t left, - std::size_t right, - std::size_t bit_count) const { - const std::size_t depth_count = bit_count + 1; - if (left >= right || right > depth_count) { - return npos; - } - - std::size_t position = left; - int best_depth = prefix_excess_with_zero_prefix(position, bit_count); - std::size_t best_position = position; - - while (position < right) { - const std::size_t chunk_begin = (position / 128) * 128; - const std::size_t local_left = position - chunk_begin; - const std::size_t local_right = - std::min(right - 1, chunk_begin + 128) - chunk_begin; - - int candidate_depth; - std::size_t candidate_position; - if (chunk_begin >= bit_count) { - candidate_depth = - prefix_excess_with_zero_prefix(chunk_begin, bit_count); - candidate_position = chunk_begin; - } else { - const std::size_t word = chunk_begin >> 6; - const ExcessResult result = - excess_min_128(bp_bits_.data() + word, local_left, local_right); - candidate_depth = - prefix_excess_with_zero_prefix(chunk_begin, bit_count) + - result.min_excess; - candidate_position = chunk_begin + result.offset; - } - - if (candidate_depth < best_depth) { - best_depth = candidate_depth; - best_position = candidate_position; - } - - position = chunk_begin + local_right + 1; - } - - return best_position; - } - - std::array bp_bits_{}; - }; - - static_assert(sizeof(Bp512Selector) == 64); - - class alignas(LeafSize == 496 ? 64 : 32) PrefixSuffixMaskLeafSelector { - public: - PrefixSuffixMaskLeafSelector() = default; - - template - void build(std::size_t entry_count, EntryLess entry_less) { - if (entry_count > kMaskEntries) { - throw std::length_error( - "NodeEulerBTreeRmq prefix/suffix leaf selector too large"); - } - - words_.fill(0); - if (entry_count == 0) { - set_embedded_min_offset(0); - return; - } - - std::size_t prefix_best = 0; - set_mask_bit(0); - for (std::size_t slot = 1; slot < entry_count; ++slot) { - if (entry_less(slot, prefix_best)) { - prefix_best = slot; - set_mask_bit(slot); - } - } - - std::size_t suffix_best = entry_count - 1; - set_mask_bit(suffix_best); - for (std::size_t slot = entry_count - 1; slot > 0;) { - --slot; - if (!entry_less(suffix_best, slot)) { - suffix_best = slot; - set_mask_bit(slot); - } - } - - set_embedded_min_offset(prefix_best); - } - - std::size_t arg_min(std::size_t slot_left, - std::size_t slot_right, - std::size_t entry_count) const { - if (slot_left >= slot_right || slot_right > entry_count || - entry_count > kMaskEntries) { - return npos; - } - if (slot_left + 1 == slot_right) { - return slot_left; - } - - const std::size_t min_offset = embedded_min_offset(); - if (slot_left <= min_offset && min_offset < slot_right) { - return min_offset; - } - if (slot_left == 0 && slot_right <= min_offset) { - return previous_set_bit_before(slot_right); - } - if (slot_right == entry_count && min_offset < slot_left) { - return next_set_bit_at_or_after(slot_left); - } - return npos; - } - - void set_embedded_min_offset(std::size_t offset) { - words_[kOffsetWord] = - (words_[kOffsetWord] & kMaskWordMask) | - ((static_cast(offset) & kOffsetMask) << kOffsetShift); - } - - std::size_t embedded_min_offset() const { - return static_cast((words_[kOffsetWord] >> kOffsetShift) & - kOffsetMask); - } - - private: - static constexpr std::size_t kMaskEntries = LeafSize; - static constexpr std::size_t kOffsetBits = LeafSize == 496 ? 16 : 8; - static constexpr std::size_t kPackedBits = kMaskEntries + kOffsetBits; - static constexpr std::size_t kWordCount = (kPackedBits + 63) / 64; - static constexpr std::size_t kOffsetWord = kMaskEntries / 64; - static constexpr std::size_t kOffsetShift = kMaskEntries & 63; - - static constexpr std::uint64_t low_bits_mask(std::size_t bits) { - if (bits == 0) { - return 0; - } - if (bits >= 64) { - return std::numeric_limits::max(); - } - return (std::uint64_t{1} << bits) - 1; - } - - static constexpr std::uint64_t kOffsetMask = low_bits_mask(kOffsetBits); - static constexpr std::uint64_t kMaskWordMask = low_bits_mask(kOffsetShift); - - static_assert(!kMaskLeafSelector || kPackedBits % 64 == 0); - static_assert(!kMaskLeafSelector || kOffsetShift + kOffsetBits == 64); - - void set_mask_bit(std::size_t slot) { - words_[slot >> 6] |= std::uint64_t{1} << (slot & 63); - } - - std::uint64_t mask_word(std::size_t word) const { - return word == kOffsetWord ? words_[word] & kMaskWordMask : words_[word]; - } - - std::size_t previous_set_bit_before(std::size_t limit) const { - if (limit == 0) { - return npos; - } - - std::size_t word = (limit - 1) >> 6; - std::uint64_t bits = - mask_word(word) & first_bits_mask(((limit - 1) & 63) + 1); - while (true) { - if (bits != 0) { - return word * 64 + 63 - std::countl_zero(bits); - } - if (word == 0) { - break; - } - --word; - bits = mask_word(word); - } - return npos; - } - - std::size_t next_set_bit_at_or_after(std::size_t slot) const { - std::size_t word = slot >> 6; - std::uint64_t bits = mask_word(word) & ~first_bits_mask(slot & 63); - while (word < kWordCount) { - if (bits != 0) { - const std::size_t result = word * 64 + std::countr_zero(bits); - return result < kMaskEntries ? result : npos; - } - ++word; - bits = word < kWordCount ? mask_word(word) : 0; - } - return npos; - } - - std::array words_{}; - }; - - static_assert(!kMaskLeafSelector || sizeof(PrefixSuffixMaskLeafSelector) == - (LeafSize == 496 ? 64 : 32)); - static_assert(!kMaskLeafSelector || alignof(PrefixSuffixMaskLeafSelector) == - (LeafSize == 496 ? 64 : 32)); - - using LeafSelector = std::conditional_t; - - bool missing_position(std::size_t position) const { - if constexpr (kInvalidIndexEqualsNpos) { - return position == npos; - } else { - return position == npos || - position == static_cast(invalid_index); - } - } - - MinCandidate value_candidate(std::size_t position) const { - if (missing_position(position)) { - return {}; - } - return {position, values_.data() + position}; - } - - MinCandidate subtree_min_candidate(std::size_t level, - std::size_t node) const { - const std::size_t position = subtree_min_position(level, node); - if (missing_position(position)) { - return {}; - } - return {position, &subtree_min_value(level, node)}; - } - - MinCandidate subtree_child_min_candidate(std::size_t level, - std::size_t node, - std::size_t slot) const { - const std::size_t child_level = level - 1; - return subtree_min_candidate(child_level, - node * fanout_at_level(level) + slot); - } - - MinCandidate high_child_min_candidate(std::size_t level, - std::size_t node, - std::size_t slot) const { - const std::size_t child_level = level - 1; - const std::size_t child = node * fanout_at_level(level) + slot; - const std::size_t position = - high_child_metadata_at(high_flat_index(level, node), slot).min_position; - if (missing_position(position)) { - return {}; - } - return {position, &subtree_min_value(child_level, child)}; - } - - bool strictly_better_candidate(MinCandidate left, MinCandidate right) const { - if (missing_position(left.position)) { - return false; - } - if (missing_position(right.position)) { - return true; - } - return compare_(*left.value, *right.value); - } - - MinCandidate better_candidate(MinCandidate left, MinCandidate right) const { - if (missing_position(left.position)) { - return right; - } - if (missing_position(right.position)) { - return left; - } - if (compare_(*right.value, *left.value)) { - return right; - } - if (compare_(*left.value, *right.value)) { - return left; - } - return right.position < left.position ? right : left; - } - - bool strictly_better_subtree_child_slot(std::size_t level, - std::size_t node, - std::size_t left_slot, - std::size_t right_slot) const { - return strictly_better_candidate( - subtree_child_min_candidate(level, node, left_slot), - subtree_child_min_candidate(level, node, right_slot)); - } - - bool strictly_better_high_child_slot(std::size_t level, - std::size_t node, - std::size_t left_slot, - std::size_t right_slot) const { - return strictly_better_candidate( - high_child_min_candidate(level, node, left_slot), - high_child_min_candidate(level, node, right_slot)); - } - - void build() { - leaf_selectors_.clear(); - medium_selectors_.clear(); - medium_min_values_.clear(); - high_selectors_.clear(); - high_min_positions_.clear(); - high_min_values_.clear(); - high_child_metadata_.clear(); - high_sparse_min_slots_.clear(); - medium_level_offsets_.clear(); - high_level_offsets_.clear(); - level_sizes_.clear(); - level_value_spans_.clear(); - level_fanouts_.clear(); - high_level_begin_ = std::numeric_limits::max(); - if (values_.empty()) { - return; - } - if (values_.size() > static_cast(invalid_index)) { - throw std::length_error("NodeEulerBTreeRmq index type is too small"); - } - - initialize_layout((values_.size() + LeafSize - 1) / LeafSize); - for (std::size_t leaf = 0; leaf < level_sizes_[0]; ++leaf) { - build_leaf(leaf); - } - - for (std::size_t level = 1; level < level_count(); ++level) { - for (std::size_t node = 0; node < level_sizes_[level]; ++node) { - build_internal_node(level, node); - } - } - } - - void initialize_layout(std::size_t leaf_count) { - level_sizes_.push_back(leaf_count); - level_value_spans_.push_back(LeafSize); - level_fanouts_.push_back(0); - - std::size_t current_count = leaf_count; - std::size_t current_span = LeafSize; - while (current_count > Fanout * Fanout) { - level_fanouts_.push_back(kMiddleFanout); - current_count = ceil_div(current_count, kMiddleFanout); - current_span = saturating_product(current_span, kMiddleFanout); - level_sizes_.push_back(current_count); - level_value_spans_.push_back(current_span); - } - while (current_count > 1) { - level_fanouts_.push_back(Fanout); - current_count = ceil_div(current_count, Fanout); - current_span = saturating_product(current_span, Fanout); - level_sizes_.push_back(current_count); - level_value_spans_.push_back(current_span); - } - - leaf_selectors_.resize(level_sizes_[0]); - - medium_level_offsets_.assign(level_count(), 0); - high_level_offsets_.assign(level_count(), 0); - if (level_count() <= 1) { - high_level_begin_ = std::numeric_limits::max(); - return; - } - - const std::size_t root_level = level_count() - 1; - high_level_begin_ = root_level > 1 ? root_level - 1 : 1; - - std::size_t medium_node_count = 0; - for (std::size_t level = 1; level < high_level_begin_; ++level) { - medium_level_offsets_[level] = medium_node_count; - medium_node_count += level_sizes_[level]; - } - medium_selectors_.resize(medium_node_count); - medium_min_values_.reserve(medium_node_count); - - std::size_t high_node_count = 0; - for (std::size_t level = high_level_begin_; level < level_count(); - ++level) { - high_level_offsets_[level] = high_node_count; - high_node_count += level_sizes_[level]; - } - high_selectors_.resize(high_node_count); - high_min_positions_.resize(high_node_count, invalid_index); - high_min_values_.reserve(high_node_count); - high_child_metadata_.resize(high_node_count * Fanout); - high_sparse_min_slots_.resize(high_node_count * kHighSparseSlotsPerNode); - } - - void build_leaf(std::size_t leaf) { - LeafSelector& selector = leaf_selectors_[leaf]; - const std::size_t begin = node_value_begin(0, leaf); - const std::size_t count = entry_count(0, leaf); - selector.build(count, [&](std::size_t left, std::size_t right) { - return compare_(values_[begin + left], values_[begin + right]); - }); - - if constexpr (kBpLeafSelector) { - const std::size_t slot = selector.arg_min(0, count, count); - selector.set_embedded_min_offset(slot); - } - } - - void build_internal_node(std::size_t level, std::size_t node) { - Bp512Selector& selector = mutable_selector_at(level, node); - const std::size_t count = entry_count(level, node); - const std::size_t first_child = node * fanout_at_level(level); - const bool high_level = is_high_level(level); - const std::size_t high_flat = high_level ? high_flat_index(level, node) : 0; - if (high_level) { - for (std::size_t slot = 0; slot < count; ++slot) { - const std::size_t child = first_child + slot; - HighChildMetadata& metadata = - mutable_high_child_metadata_at(high_flat, slot); - metadata.value_begin = node_value_begin(level - 1, child); - metadata.value_end = node_value_end(level - 1, child); - metadata.min_position = - static_cast(subtree_min_position(level - 1, child)); - } - build_high_sparse_min_slots(level, node, count); - } - - selector.build(count, [&](std::size_t left, std::size_t right) { - if (high_level) { - return strictly_better_high_child_slot(level, node, left, right); - } - return strictly_better_subtree_child_slot(level, node, left, right); - }); - - const std::size_t slot = selector.arg_min(0, count, count); - const std::size_t min_position = - high_level ? high_child_metadata_at(high_flat, slot).min_position - : subtree_min_position(level - 1, first_child + slot); - if (high_level) { - high_min_positions_[high_flat] = static_cast(min_position); - high_min_values_.push_back(values_[min_position]); - } else { - selector.set_embedded_min_position(min_position); - medium_min_values_.push_back(values_[min_position]); - selector.build_zero_prefix_metadata(2 * count); - } - } - - static std::size_t saturating_product(std::size_t left, std::size_t right) { - if (left != 0 && right > std::numeric_limits::max() / left) { - return std::numeric_limits::max(); - } - return left * right; - } - - static std::size_t ceil_div(std::size_t value, std::size_t divisor) { - return (value + divisor - 1) / divisor; - } - - std::size_t leaf_range_min(std::size_t leaf, - std::size_t left, - std::size_t right) const { - if (left >= right) { - return npos; - } - - const std::size_t begin = node_value_begin(0, leaf); - const std::size_t end = node_value_end(0, leaf); - if (left <= begin && end <= right) { - return subtree_min_position(0, leaf); - } - const std::size_t slot_left = left - begin; - const std::size_t slot_right = right - begin; - - if constexpr (kMaskLeafSelector) { - const std::size_t slot = leaf_selectors_[leaf].arg_min( - slot_left, slot_right, entry_count(0, leaf)); - if (slot != npos) { - return begin + slot; - } - return linear_range_min(left, right); - } else { - if (right - left <= kLeafLinearScanThreshold) { - return linear_range_min(left, right); - } - - const std::size_t slot = leaf_selectors_[leaf].arg_min( - slot_left, slot_right, entry_count(0, leaf)); - if (slot == npos) { - return npos; - } - return begin + slot; - } - } - - std::size_t linear_range_min(std::size_t left, std::size_t right) const { - if (left >= right) { - return npos; - } -#ifdef PIXIE_AVX2_SUPPORT - if constexpr (std::is_same_v && - std::is_same_v>) { - if (right - left >= kLeafAvx2ScanThreshold) { - return linear_range_min_i64_avx2(left, right); - } - } -#endif - std::size_t best = left; - for (std::size_t position = left + 1; position < right; ++position) { - if (compare_(values_[position], values_[best])) { - best = position; - } - } - return best; - } - -#ifdef PIXIE_AVX2_SUPPORT - std::size_t linear_range_min_i64_avx2(std::size_t left, - std::size_t right) const { - const std::int64_t* data = values_.data(); - std::size_t position = left; - - __m256i best_values = - _mm256_loadu_si256(reinterpret_cast(data + position)); - __m256i best_positions = _mm256_set_epi64x( - static_cast(position + 3), - static_cast(position + 2), - static_cast(position + 1), static_cast(position)); - position += 4; - - for (; position + 4 <= right; position += 4) { - const __m256i values = - _mm256_loadu_si256(reinterpret_cast(data + position)); - const __m256i positions = - _mm256_set_epi64x(static_cast(position + 3), - static_cast(position + 2), - static_cast(position + 1), - static_cast(position)); - const __m256i take_new = _mm256_cmpgt_epi64(best_values, values); - best_values = _mm256_blendv_epi8(best_values, values, take_new); - best_positions = _mm256_blendv_epi8(best_positions, positions, take_new); - } - - alignas(32) std::int64_t value_lanes[4]; - alignas(32) std::uint64_t position_lanes[4]; - _mm256_store_si256(reinterpret_cast<__m256i*>(value_lanes), best_values); - _mm256_store_si256(reinterpret_cast<__m256i*>(position_lanes), - best_positions); - - std::int64_t best_value = value_lanes[0]; - std::size_t best_position = static_cast(position_lanes[0]); - for (std::size_t lane = 1; lane < 4; ++lane) { - const std::size_t lane_position = - static_cast(position_lanes[lane]); - if (value_lanes[lane] < best_value || - (value_lanes[lane] == best_value && lane_position < best_position)) { - best_value = value_lanes[lane]; - best_position = lane_position; - } - } - - for (; position < right; ++position) { - if (data[position] < best_value) { - best_value = data[position]; - best_position = position; - } - } - return best_position; - } -#endif - - std::pair covering_node( - std::size_t left_leaf, - std::size_t right_leaf) const { - std::size_t level = 0; - std::size_t left_node = left_leaf; - std::size_t right_node = right_leaf; - while (left_node != right_node) { - ++level; - const std::size_t fanout = fanout_at_level(level); - left_node /= fanout; - right_node /= fanout; - } - return {level, left_node}; - } - - std::size_t leaf_for_value(std::size_t position) const { - return position / LeafSize; - } - - std::size_t child_for_value(std::size_t child_level, - std::size_t position) const { - return position / level_value_spans_[child_level]; - } - - MinCandidate query_child_slots(std::size_t level, - std::size_t node, - std::size_t slot_left, - std::size_t slot_right, - std::size_t left, - std::size_t right) const { - if (slot_left >= slot_right) { - return {}; - } - - const std::size_t count = entry_count(level, node); - const std::size_t slot = - slot_left + 1 == slot_right - ? slot_left - : selector_arg_min(level, node, slot_left, slot_right, count); - if (slot == npos) { - return {}; - } - - const std::size_t child_level = level - 1; - const std::size_t first_child = node * fanout_at_level(level); - const std::size_t child = first_child + slot; - const HighChildMetadata* high_children = - is_high_level(level) ? high_child_metadata_begin(level, node) : nullptr; - const MinCandidate child_min = - high_children != nullptr ? high_child_min_candidate(level, node, slot) - : subtree_min_candidate(child_level, child); - const std::size_t child_begin = high_children != nullptr - ? high_children[slot].value_begin - : node_value_begin(child_level, child); - const std::size_t child_end = high_children != nullptr - ? high_children[slot].value_end - : node_value_end(child_level, child); - if ((left <= child_begin && child_end <= right) || - contains_position(left, right, child_min.position)) { - return child_min; - } - - const std::size_t last_slot = slot_right - 1; - const std::size_t left_child_begin = - high_children != nullptr - ? high_children[slot_left].value_begin - : node_value_begin(child_level, first_child + slot_left); - const std::size_t left_child_end = - high_children != nullptr - ? high_children[slot_left].value_end - : node_value_end(child_level, first_child + slot_left); - MinCandidate answer = query_node(child_level, first_child + slot_left, - std::max(left, left_child_begin), - std::min(right, left_child_end)); - - if (slot_left != last_slot) { - const std::size_t right_child_begin = - high_children != nullptr - ? high_children[last_slot].value_begin - : node_value_begin(child_level, first_child + last_slot); - const std::size_t right_child_end = - high_children != nullptr - ? high_children[last_slot].value_end - : node_value_end(child_level, first_child + last_slot); - answer = better_candidate(answer, - query_node(child_level, first_child + last_slot, - std::max(left, right_child_begin), - std::min(right, right_child_end))); - } - - if (slot_left + 1 < last_slot) { - answer = better_candidate( - answer, - full_child_slot_range_min(level, node, slot_left + 1, last_slot)); - } - - return answer; - } - - MinCandidate full_child_slot_range_min(std::size_t level, - std::size_t node, - std::size_t slot_left, - std::size_t slot_right) const { - if (slot_left >= slot_right) { - return {}; - } - - const std::size_t slot = - slot_left + 1 == slot_right - ? slot_left - : selector_arg_min(level, node, slot_left, slot_right, - entry_count(level, node)); - if (slot == npos) { - return {}; - } - - if (is_high_level(level)) { - return high_child_min_candidate(level, node, slot); - } - return subtree_child_min_candidate(level, node, slot); - } - - MinCandidate query_node(std::size_t level, - std::size_t node, - std::size_t left, - std::size_t right) const { - if (left >= right) { - return {}; - } - const std::size_t begin = node_value_begin(level, node); - const std::size_t end = node_value_end(level, node); - if (left <= begin && end <= right) { - return subtree_min_candidate(level, node); - } - if (level == 0) { - return value_candidate(leaf_range_min(node, left, right)); - } - - const std::size_t child_level = level - 1; - const std::size_t left_child = child_for_value(child_level, left); - const std::size_t right_child = child_for_value(child_level, right - 1); - const std::size_t first_child = node * fanout_at_level(level); - const std::size_t left_slot = left_child - first_child; - const std::size_t right_slot = right_child - first_child + 1; - return query_child_slots(level, node, left_slot, right_slot, left, right); - } - - bool contains_position(std::size_t left, - std::size_t right, - std::size_t position) const { - return !missing_position(position) && left <= position && position < right; - } - - std::size_t level_count() const { return level_sizes_.size(); } - - std::size_t entry_count(std::size_t level, std::size_t node) const { - if (level == 0) { - const std::size_t begin = node_value_begin(0, node); - return std::min(LeafSize, values_.size() - begin); - } - const std::size_t first_child = node * fanout_at_level(level); - return std::min(fanout_at_level(level), - level_sizes_[level - 1] - first_child); - } - - std::size_t node_value_begin(std::size_t level, std::size_t node) const { - return node * level_value_spans_[level]; - } - - std::size_t node_value_end(std::size_t level, std::size_t node) const { - return std::min(values_.size(), - node_value_begin(level, node) + level_value_spans_[level]); - } - - std::size_t subtree_min_position(std::size_t level, std::size_t node) const { - if (level == 0) { - return node_value_begin(0, node) + - leaf_selectors_[node].embedded_min_offset(); - } - if (is_high_level(level)) { - return high_min_positions_[high_flat_index(level, node)]; - } - return selector_at(level, node).embedded_min_position(); - } - - const T& subtree_min_value(std::size_t level, std::size_t node) const { - if (level == 0) { - return values_[subtree_min_position(0, node)]; - } - if (is_high_level(level)) { - return high_min_values_[high_flat_index(level, node)]; - } - return medium_min_values_[medium_flat_index(level, node)]; - } - - const Bp512Selector& selector_at(std::size_t level, std::size_t node) const { - if (is_high_level(level)) { - return high_selectors_[high_flat_index(level, node)]; - } - return medium_selectors_[medium_flat_index(level, node)]; - } - - Bp512Selector& mutable_selector_at(std::size_t level, std::size_t node) { - if (is_high_level(level)) { - return high_selectors_[high_flat_index(level, node)]; - } - return medium_selectors_[medium_flat_index(level, node)]; - } - - std::size_t selector_arg_min(std::size_t level, - std::size_t node, - std::size_t slot_left, - std::size_t slot_right, - std::size_t count) const { - if (is_high_level(level)) { - return high_sparse_arg_min(level, node, slot_left, slot_right, count); - } - const Bp512Selector& selector = selector_at(level, node); - return selector.arg_min_with_zero_prefix(slot_left, slot_right, count); - } - - bool is_high_level(std::size_t level) const { - return level > 0 && level >= high_level_begin_ && level < level_count(); - } - - std::size_t medium_flat_index(std::size_t level, std::size_t node) const { - return medium_level_offsets_[level] + node; - } - - std::size_t fanout_at_level(std::size_t level) const { - return level_fanouts_[level]; - } - - std::size_t high_flat_index(std::size_t level, std::size_t node) const { - return high_level_offsets_[level] + node; - } - - const HighChildMetadata* high_child_metadata_begin(std::size_t level, - std::size_t node) const { - return high_child_metadata_.data() + high_flat_index(level, node) * Fanout; - } - - std::uint8_t* mutable_high_sparse_min_slots_begin(std::size_t high_flat) { - return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; - } - - const std::uint8_t* high_sparse_min_slots_begin(std::size_t high_flat) const { - return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; - } - - const HighChildMetadata& high_child_metadata_at(std::size_t high_flat, - std::size_t slot) const { - return high_child_metadata_[high_flat * Fanout + slot]; - } - - HighChildMetadata& mutable_high_child_metadata_at(std::size_t high_flat, - std::size_t slot) { - return high_child_metadata_[high_flat * Fanout + slot]; - } - - std::size_t better_high_child_slot(std::size_t level, - std::size_t node, - std::size_t left_slot, - std::size_t right_slot) const { - const MinCandidate left = high_child_min_candidate(level, node, left_slot); - const MinCandidate right = - high_child_min_candidate(level, node, right_slot); - return better_candidate(left, right).position == right.position ? right_slot - : left_slot; - } - - void build_high_sparse_min_slots(std::size_t level, - std::size_t node, - std::size_t count) { - const std::size_t high_flat = high_flat_index(level, node); - std::uint8_t* table = mutable_high_sparse_min_slots_begin(high_flat); - for (std::size_t slot = 0; slot < count; ++slot) { - table[slot] = static_cast(slot); - } - - for (std::size_t table_level = 1; table_level < kHighSparseLevels; - ++table_level) { - const std::size_t span = std::size_t{1} << table_level; - if (span > count) { - break; - } - const std::size_t half_span = span >> 1; - const std::uint8_t* previous = table + (table_level - 1) * Fanout; - std::uint8_t* current = table + table_level * Fanout; - for (std::size_t slot = 0; slot + span <= count; ++slot) { - current[slot] = static_cast(better_high_child_slot( - level, node, previous[slot], previous[slot + half_span])); - } - } - } - - std::size_t high_sparse_arg_min(std::size_t level, - std::size_t node, - std::size_t slot_left, - std::size_t slot_right, - std::size_t count) const { - if (slot_left >= slot_right || slot_right > count) { - return npos; - } - const std::size_t length = slot_right - slot_left; - if (length == 1) { - return slot_left; - } - - const std::size_t high_flat = high_flat_index(level, node); - const std::size_t table_level = std::bit_width(length) - 1; - const std::size_t span = std::size_t{1} << table_level; - const std::uint8_t* table = - high_sparse_min_slots_begin(high_flat) + table_level * Fanout; - return better_high_child_slot(level, node, table[slot_left], - table[slot_right - span]); - } - - std::span values_; - Compare compare_; - std::vector leaf_selectors_; - std::vector medium_selectors_; - std::vector medium_min_values_; - std::vector high_selectors_; - std::vector high_min_positions_; - std::vector high_min_values_; - std::vector high_child_metadata_; - std::vector high_sparse_min_slots_; - std::vector medium_level_offsets_; - std::vector high_level_offsets_; - std::vector level_sizes_; - std::vector level_value_spans_; - std::vector level_fanouts_; - std::size_t high_level_begin_ = std::numeric_limits::max(); -}; - -} // namespace pixie::rmq::experimental diff --git a/include/pixie/rmq/segment_btree_xl.h b/include/pixie/rmq/hybrid_btree.h similarity index 76% rename from include/pixie/rmq/segment_btree_xl.h rename to include/pixie/rmq/hybrid_btree.h index 55b1462..9c5f479 100644 --- a/include/pixie/rmq/segment_btree_xl.h +++ b/include/pixie/rmq/hybrid_btree.h @@ -20,15 +20,15 @@ namespace pixie::rmq { /** - * @brief Low-level selector implementation used by SegmentBTreeXL leaves. + * @brief Low-level selector implementation used by HybridBTree leaves. */ -enum class SegmentBTreeXLLeafSelector { +enum class HybridBTreeLeafSelector { PrefixSuffix, BP, }; /** - * @brief Segment B-tree RMQ with compact per-level selectors and XL leaves. + * @brief Hybrid B-tree RMQ with compact per-level selectors. * * @details This is a static, non-owning value-RMQ index over an external value * array. Values are split into leaves, leaves are grouped into a B-tree, and @@ -44,18 +44,18 @@ enum class SegmentBTreeXLLeafSelector { * values. The same implementation also supports 248-value mask leaves and * 252-value BP leaves for controlled experiments. * - * Middle internal nodes use 192-way fanout. Their selector is a 512-bit local + * Internal nodes use 192-way fanout. Their selector is a 512-bit local * Cartesian-tree balanced-parentheses encoding over child minima: 384 BP bits, * a 64-bit absolute subtree-minimum position, and 64 bits of zero-rank prefix - * metadata. Middle-node minimum values are cached in a side vector so comparing + * metadata. Node minimum values are cached in a side vector so comparing * candidates does not require repeatedly descending to leaves. * - * High nodes use 256-way fanout and are limited to the root and its child - * level, so there are at most 257 high nodes. They keep the same local BP - * selector, cache child value ranges and child subtree minima, and add sparse - * tables over child-minimum slots. This spends more space at the top of the - * tree to reduce work on wide queries while keeping the high-level metadata - * small enough to stay cache-resident. + * Wide queries first try a single coarse sparse table over original-value block + * minima. The sparse table uses at least 4096-value blocks and grows the block + * width when needed so the top layer has at most 2^14 blocks. If the padded + * block-cover candidate lies inside the query, it is returned immediately; on + * a miss, the sparse table answers the fully covered middle block range and + * the B-tree handles the two border ranges. * * @code * value array, n entries @@ -64,25 +64,21 @@ enum class SegmentBTreeXLLeafSelector { * | ceil(n / 496) nodes by default * | each leaf covers <=496 values and stores one 64-byte selector * | - * +-- L1..Lm: middle levels, only when the frontier is > 256 * 256 nodes + * +-- T: value-block sparse overlay + * | block width >=4096 values, at most 2^14 blocks + * | stores one original-value minimum position per sparse-table cell + * | + * +-- L1..Lm: internal levels * | fanout 192 * | each node stores one 512-bit BP selector over child minima - * | - * +-- H0: high child level - * | fanout 256, <=256 nodes - * | BP selector + child ranges + child minima + sparse child table - * | - * +-- H1: high root - * fanout 256, one node - * BP selector + child ranges + child minima + sparse child table * * Examples with default 496-value leaves: * * n = 2^24: - * 33,826 leaves -> 133 high nodes -> 1 high root + * 33,826 leaves -> 177 internal nodes -> 1 root * * n = 2^26: - * 135,301 leaves -> 705 middle nodes -> 3 high nodes -> 1 high root + * 135,301 leaves -> 705 internal nodes -> 4 internal nodes -> 1 root * @endcode * * Querying starts at the lowest node that covers both endpoint leaves. At each @@ -98,8 +94,8 @@ enum class SegmentBTreeXLLeafSelector { * @tparam Index Unsigned integer type used for stored positions. * @tparam LeafSize Number of original values per leaf. This backend currently * supports 252 with BP leaves and 248 or 496 with mask leaves. - * @tparam Fanout Number of children per high internal node. This backend - * currently supports only 256. Middle internal nodes use 192. + * @tparam Fanout Template parameter kept fixed at 256 for the current layout; + * internal tree nodes use fixed 192-way fanout. * @tparam LeafSelectorKind Compile-time low-level selector kind. */ template -class SegmentBTreeXL + HybridBTreeLeafSelector LeafSelectorKind = + HybridBTreeLeafSelector::PrefixSuffix> +class HybridBTree : public RmqBase< - SegmentBTreeXL, + HybridBTree, T> { public: static_assert(std::is_unsigned_v, - "SegmentBTreeXL index type must be unsigned"); + "HybridBTree index type must be unsigned"); static constexpr bool kBpLeafSelector = - LeafSelectorKind == SegmentBTreeXLLeafSelector::BP; + LeafSelectorKind == HybridBTreeLeafSelector::BP; static constexpr bool kMaskLeafSelector = - LeafSelectorKind == SegmentBTreeXLLeafSelector::PrefixSuffix; + LeafSelectorKind == HybridBTreeLeafSelector::PrefixSuffix; static_assert((kBpLeafSelector && LeafSize == 252) || (kMaskLeafSelector && (LeafSize == 248 || LeafSize == 496)), - "SegmentBTreeXL requires 252-value BP leaves " + "HybridBTree requires 252-value BP leaves " "or 248/496-value prefix/suffix mask leaves"); static_assert(Fanout == 256, - "SegmentBTreeXL currently requires 256-way internal nodes"); + "HybridBTree currently requires a 256 fanout template " + "argument"); static_assert(kBpLeafSelector || kMaskLeafSelector, - "unsupported SegmentBTreeXL low-level selector kind"); + "unsupported HybridBTree low-level selector kind"); using Self = - SegmentBTreeXL; + HybridBTree; static constexpr std::size_t npos = RmqBase::npos; static constexpr Index invalid_index = std::numeric_limits::max(); static constexpr std::size_t kLeafSize = LeafSize; static constexpr std::size_t kFanout = Fanout; static constexpr std::size_t kMiddleFanout = 192; + static constexpr std::size_t kMinTopSparseBlockSize = 4096; + static constexpr std::size_t kMaxTopSparseBlocks = std::size_t{1} << 14; /** * @brief Construct an empty RMQ index. */ - SegmentBTreeXL() = default; + HybridBTree() = default; /** * @brief Copy an RMQ index while preserving its non-owning value span. */ - SegmentBTreeXL(const SegmentBTreeXL&) = default; + HybridBTree(const HybridBTree&) = default; /** * @brief Move an RMQ index while preserving selector and cache storage. */ - SegmentBTreeXL(SegmentBTreeXL&&) noexcept = default; + HybridBTree(HybridBTree&&) noexcept = default; /** * @brief Copy-assign an RMQ index and its cached metadata. */ - SegmentBTreeXL& operator=(const SegmentBTreeXL&) = default; + HybridBTree& operator=(const HybridBTree&) = default; /** * @brief Move-assign an RMQ index and its cached metadata. */ - SegmentBTreeXL& operator=(SegmentBTreeXL&&) noexcept = default; + HybridBTree& operator=(HybridBTree&&) noexcept = default; /** - * @brief Build a segment B-tree RMQ index over @p values. + * @brief Build a hybrid B-tree RMQ index over @p values. * * @details The values are not copied and must outlive this object. Equal * values keep the smaller original position as the answer. @@ -173,8 +172,7 @@ class SegmentBTreeXL * @param compare Ordering used to choose minima. * @throws std::length_error if @p Index cannot represent all positions. */ - explicit SegmentBTreeXL(std::span values, - Compare compare = Compare()) + explicit HybridBTree(std::span values, Compare compare = Compare()) : values_(values), compare_(compare) { build(); } @@ -201,21 +199,45 @@ class SegmentBTreeXL return npos; } - const std::size_t root_level = level_count() - 1; - if (left == 0 && right == values_.size()) { - return subtree_min_position(root_level, 0); + if (right - left <= top_block_size_) { + return tree_arg_min(left, right); } + const std::size_t top_answer = top_sparse_arg_min(left, right); + return top_answer != npos ? top_answer : tree_arg_min(left, right); + } - const std::size_t left_leaf = leaf_for_value(left); - const std::size_t right_leaf = leaf_for_value(right - 1); - if (left_leaf == right_leaf) { - return leaf_range_min(left_leaf, left, right); + /** + * @brief Return the top sparse-table block width chosen for a value count. + */ + static std::size_t top_sparse_block_size_for(std::size_t value_count) { + if (value_count == 0) { + return kMinTopSparseBlockSize; } + return std::max(kMinTopSparseBlockSize, + 1 + (value_count - 1) / kMaxTopSparseBlocks); + } - const auto [level, node_index] = covering_node(left_leaf, right_leaf); - return query_node(level, node_index, left, right).position; + /** + * @brief Return the number of top sparse-table blocks for a value count. + */ + static std::size_t top_sparse_block_count_for(std::size_t value_count) { + if (value_count == 0) { + return 0; + } + const std::size_t block_size = top_sparse_block_size_for(value_count); + return 1 + (value_count - 1) / block_size; } + /** + * @brief Return the current top sparse-table block width. + */ + std::size_t top_sparse_block_size() const { return top_block_size_; } + + /** + * @brief Return the current number of top sparse-table blocks. + */ + std::size_t top_sparse_block_count() const { return top_block_count_; } + /** * @brief Return owned auxiliary memory usage in bytes. * @@ -227,13 +249,8 @@ class SegmentBTreeXL bytes += pixie::vector_capacity_bytes(leaf_selectors_); bytes += pixie::vector_capacity_bytes(medium_selectors_); bytes += pixie::vector_capacity_bytes(medium_min_values_); - bytes += pixie::vector_capacity_bytes(high_selectors_); - bytes += pixie::vector_capacity_bytes(high_min_positions_); - bytes += pixie::vector_capacity_bytes(high_min_values_); - bytes += pixie::vector_capacity_bytes(high_child_metadata_); - bytes += pixie::vector_capacity_bytes(high_sparse_min_slots_); + bytes += pixie::vector_capacity_bytes(top_sparse_candidates_); bytes += pixie::vector_capacity_bytes(medium_level_offsets_); - bytes += pixie::vector_capacity_bytes(high_level_offsets_); bytes += pixie::vector_capacity_bytes(level_sizes_); bytes += pixie::vector_capacity_bytes(level_value_spans_); bytes += pixie::vector_capacity_bytes(level_fanouts_); @@ -250,10 +267,6 @@ class SegmentBTreeXL static constexpr std::size_t kEmbeddedPositionBit = 2 * kEmbeddedPositionEntries; static constexpr std::size_t kMiddleZeroPrefixWord = 7; - static constexpr std::size_t kHighSparseLevels = - static_cast(std::bit_width(Fanout)); - static constexpr std::size_t kHighSparseSlotsPerNode = - kHighSparseLevels * Fanout; static constexpr std::size_t kLeafLinearScanThreshold = 64; static constexpr std::size_t kLeafAvx2ScanThreshold = 16; static constexpr std::uint64_t kEmbeddedOffsetMask = @@ -266,17 +279,16 @@ class SegmentBTreeXL static_assert(!kBpLeafSelector || LeafSize <= kEmbeddedOffsetEntries); static_assert(kMiddleFanout <= kEmbeddedPositionEntries); - struct HighChildMetadata { - std::size_t value_begin = 0; - std::size_t value_end = 0; - Index min_position = invalid_index; - }; - struct MinCandidate { std::size_t position = npos; const T* value = nullptr; }; + struct TopCandidate { + Index position = invalid_index; + }; + static_assert(sizeof(TopCandidate) == sizeof(Index)); + class alignas(64) Bp512Selector { public: /** @@ -295,7 +307,7 @@ class SegmentBTreeXL template void build(std::size_t entry_count, EntryLess entry_less) { if (entry_count > kSelectorEntries) { - throw std::length_error("SegmentBTreeXL local selector too large"); + throw std::length_error("HybridBTree local selector too large"); } bp_bits_.fill(0); @@ -667,7 +679,7 @@ class SegmentBTreeXL void build(std::size_t entry_count, EntryLess entry_less) { if (entry_count > kMaskEntries) { throw std::length_error( - "SegmentBTreeXL prefix/suffix leaf selector too large"); + "HybridBTree prefix/suffix leaf selector too large"); } words_.fill(0); @@ -857,7 +869,7 @@ class SegmentBTreeXL * @brief Wrap an original value position as a comparable candidate. */ MinCandidate value_candidate(std::size_t position) const { - if (missing_position(position)) { + if (missing_position(position) || position >= values_.size()) { return {}; } return {position, values_.data() + position}; @@ -886,22 +898,6 @@ class SegmentBTreeXL node * fanout_at_level(level) + slot); } - /** - * @brief Return a high-node child candidate using high-child metadata. - */ - MinCandidate high_child_min_candidate(std::size_t level, - std::size_t node, - std::size_t slot) const { - const std::size_t child_level = level - 1; - const std::size_t child = node * fanout_at_level(level) + slot; - const std::size_t position = - high_child_metadata_at(high_flat_index(level, node), slot).min_position; - if (missing_position(position)) { - return {}; - } - return {position, &subtree_min_value(child_level, child)}; - } - /** * @brief Return whether @p left is strictly better than @p right. */ @@ -946,18 +942,6 @@ class SegmentBTreeXL subtree_child_min_candidate(level, node, right_slot)); } - /** - * @brief Compare two high-node child slots while building high metadata. - */ - bool strictly_better_high_child_slot(std::size_t level, - std::size_t node, - std::size_t left_slot, - std::size_t right_slot) const { - return strictly_better_candidate( - high_child_min_candidate(level, node, left_slot), - high_child_min_candidate(level, node, right_slot)); - } - /** * @brief Build all levels, selectors, and cached minimum metadata. */ @@ -965,22 +949,19 @@ class SegmentBTreeXL leaf_selectors_.clear(); medium_selectors_.clear(); medium_min_values_.clear(); - high_selectors_.clear(); - high_min_positions_.clear(); - high_min_values_.clear(); - high_child_metadata_.clear(); - high_sparse_min_slots_.clear(); + top_sparse_candidates_.clear(); + top_block_size_ = kMinTopSparseBlockSize; + top_block_count_ = 0; + top_sparse_levels_ = 0; medium_level_offsets_.clear(); - high_level_offsets_.clear(); level_sizes_.clear(); level_value_spans_.clear(); level_fanouts_.clear(); - high_level_begin_ = std::numeric_limits::max(); if (values_.empty()) { return; } if (values_.size() > static_cast(invalid_index)) { - throw std::length_error("SegmentBTreeXL index type is too small"); + throw std::length_error("HybridBTree index type is too small"); } initialize_layout((values_.size() + LeafSize - 1) / LeafSize); @@ -993,13 +974,14 @@ class SegmentBTreeXL build_internal_node(level, node); } } + build_top_sparse_table(); } /** * @brief Compute level sizes, fanouts, flat offsets, and cache storage. * - * @details The top two tree levels are marked as high levels. Levels below - * them use the middle-node layout. Level zero always stores leaf selectors. + * @details Level zero always stores leaf selectors. Every internal level uses + * the fixed 192-way middle-node layout. */ void initialize_layout(std::size_t leaf_count) { level_sizes_.push_back(leaf_count); @@ -1008,52 +990,28 @@ class SegmentBTreeXL std::size_t current_count = leaf_count; std::size_t current_span = LeafSize; - while (current_count > Fanout * Fanout) { + while (current_count > 1) { level_fanouts_.push_back(kMiddleFanout); current_count = ceil_div(current_count, kMiddleFanout); current_span = saturating_product(current_span, kMiddleFanout); level_sizes_.push_back(current_count); level_value_spans_.push_back(current_span); } - while (current_count > 1) { - level_fanouts_.push_back(Fanout); - current_count = ceil_div(current_count, Fanout); - current_span = saturating_product(current_span, Fanout); - level_sizes_.push_back(current_count); - level_value_spans_.push_back(current_span); - } leaf_selectors_.resize(level_sizes_[0]); medium_level_offsets_.assign(level_count(), 0); - high_level_offsets_.assign(level_count(), 0); if (level_count() <= 1) { - high_level_begin_ = std::numeric_limits::max(); return; } - const std::size_t root_level = level_count() - 1; - high_level_begin_ = root_level > 1 ? root_level - 1 : 1; - std::size_t medium_node_count = 0; - for (std::size_t level = 1; level < high_level_begin_; ++level) { + for (std::size_t level = 1; level < level_count(); ++level) { medium_level_offsets_[level] = medium_node_count; medium_node_count += level_sizes_[level]; } medium_selectors_.resize(medium_node_count); medium_min_values_.reserve(medium_node_count); - - std::size_t high_node_count = 0; - for (std::size_t level = high_level_begin_; level < level_count(); - ++level) { - high_level_offsets_[level] = high_node_count; - high_node_count += level_sizes_[level]; - } - high_selectors_.resize(high_node_count); - high_min_positions_.resize(high_node_count, invalid_index); - high_min_values_.reserve(high_node_count); - high_child_metadata_.resize(high_node_count * Fanout); - high_sparse_min_slots_.resize(high_node_count * kHighSparseSlotsPerNode); } /** @@ -1080,40 +1038,17 @@ class SegmentBTreeXL Bp512Selector& selector = mutable_selector_at(level, node); const std::size_t count = entry_count(level, node); const std::size_t first_child = node * fanout_at_level(level); - const bool high_level = is_high_level(level); - const std::size_t high_flat = high_level ? high_flat_index(level, node) : 0; - if (high_level) { - for (std::size_t slot = 0; slot < count; ++slot) { - const std::size_t child = first_child + slot; - HighChildMetadata& metadata = - mutable_high_child_metadata_at(high_flat, slot); - metadata.value_begin = node_value_begin(level - 1, child); - metadata.value_end = node_value_end(level - 1, child); - metadata.min_position = - static_cast(subtree_min_position(level - 1, child)); - } - build_high_sparse_min_slots(level, node, count); - } selector.build(count, [&](std::size_t left, std::size_t right) { - if (high_level) { - return strictly_better_high_child_slot(level, node, left, right); - } return strictly_better_subtree_child_slot(level, node, left, right); }); const std::size_t slot = selector.arg_min(0, count, count); const std::size_t min_position = - high_level ? high_child_metadata_at(high_flat, slot).min_position - : subtree_min_position(level - 1, first_child + slot); - if (high_level) { - high_min_positions_[high_flat] = static_cast(min_position); - high_min_values_.push_back(values_[min_position]); - } else { - selector.set_embedded_min_position(min_position); - medium_min_values_.push_back(values_[min_position]); - selector.build_zero_prefix_metadata(2 * count); - } + subtree_min_position(level - 1, first_child + slot); + selector.set_embedded_min_position(min_position); + medium_min_values_.push_back(values_[min_position]); + selector.build_zero_prefix_metadata(2 * count); } /** @@ -1130,7 +1065,26 @@ class SegmentBTreeXL * @brief Return `ceil(value / divisor)` for positive divisors. */ static std::size_t ceil_div(std::size_t value, std::size_t divisor) { - return (value + divisor - 1) / divisor; + return value == 0 ? 0 : 1 + (value - 1) / divisor; + } + + /** + * @brief Return the first minimum position through the B-tree fallback. + */ + std::size_t tree_arg_min(std::size_t left, std::size_t right) const { + const std::size_t root_level = level_count() - 1; + if (left == 0 && right == values_.size()) { + return subtree_min_position(root_level, 0); + } + + const std::size_t left_leaf = leaf_for_value(left); + const std::size_t right_leaf = leaf_for_value(right - 1); + if (left_leaf == right_leaf) { + return leaf_range_min(left_leaf, left, right); + } + + const auto [level, node_index] = covering_node(left_leaf, right_leaf); + return query_node(level, node_index, left, right).position; } /** @@ -1325,17 +1279,9 @@ class SegmentBTreeXL const std::size_t child_level = level - 1; const std::size_t first_child = node * fanout_at_level(level); const std::size_t child = first_child + slot; - const HighChildMetadata* high_children = - is_high_level(level) ? high_child_metadata_begin(level, node) : nullptr; - const MinCandidate child_min = - high_children != nullptr ? high_child_min_candidate(level, node, slot) - : subtree_min_candidate(child_level, child); - const std::size_t child_begin = high_children != nullptr - ? high_children[slot].value_begin - : node_value_begin(child_level, child); - const std::size_t child_end = high_children != nullptr - ? high_children[slot].value_end - : node_value_end(child_level, child); + const MinCandidate child_min = subtree_min_candidate(child_level, child); + const std::size_t child_begin = node_value_begin(child_level, child); + const std::size_t child_end = node_value_end(child_level, child); if ((left <= child_begin && child_end <= right) || contains_position(left, right, child_min.position)) { return child_min; @@ -1343,26 +1289,18 @@ class SegmentBTreeXL const std::size_t last_slot = slot_right - 1; const std::size_t left_child_begin = - high_children != nullptr - ? high_children[slot_left].value_begin - : node_value_begin(child_level, first_child + slot_left); + node_value_begin(child_level, first_child + slot_left); const std::size_t left_child_end = - high_children != nullptr - ? high_children[slot_left].value_end - : node_value_end(child_level, first_child + slot_left); + node_value_end(child_level, first_child + slot_left); MinCandidate answer = query_node(child_level, first_child + slot_left, std::max(left, left_child_begin), std::min(right, left_child_end)); if (slot_left != last_slot) { const std::size_t right_child_begin = - high_children != nullptr - ? high_children[last_slot].value_begin - : node_value_begin(child_level, first_child + last_slot); + node_value_begin(child_level, first_child + last_slot); const std::size_t right_child_end = - high_children != nullptr - ? high_children[last_slot].value_end - : node_value_end(child_level, first_child + last_slot); + node_value_end(child_level, first_child + last_slot); answer = better_candidate(answer, query_node(child_level, first_child + last_slot, std::max(left, right_child_begin), @@ -1398,9 +1336,6 @@ class SegmentBTreeXL return {}; } - if (is_high_level(level)) { - return high_child_min_candidate(level, node, slot); - } return subtree_child_min_candidate(level, node, slot); } @@ -1482,9 +1417,6 @@ class SegmentBTreeXL return node_value_begin(0, node) + leaf_selectors_[node].embedded_min_offset(); } - if (is_high_level(level)) { - return high_min_positions_[high_flat_index(level, node)]; - } return selector_at(level, node).embedded_min_position(); } @@ -1495,9 +1427,6 @@ class SegmentBTreeXL if (level == 0) { return values_[subtree_min_position(0, node)]; } - if (is_high_level(level)) { - return high_min_values_[high_flat_index(level, node)]; - } return medium_min_values_[medium_flat_index(level, node)]; } @@ -1505,9 +1434,6 @@ class SegmentBTreeXL * @brief Return an immutable internal-node BP selector. */ const Bp512Selector& selector_at(std::size_t level, std::size_t node) const { - if (is_high_level(level)) { - return high_selectors_[high_flat_index(level, node)]; - } return medium_selectors_[medium_flat_index(level, node)]; } @@ -1515,9 +1441,6 @@ class SegmentBTreeXL * @brief Return a mutable internal-node BP selector while building. */ Bp512Selector& mutable_selector_at(std::size_t level, std::size_t node) { - if (is_high_level(level)) { - return high_selectors_[high_flat_index(level, node)]; - } return medium_selectors_[medium_flat_index(level, node)]; } @@ -1529,20 +1452,10 @@ class SegmentBTreeXL std::size_t slot_left, std::size_t slot_right, std::size_t count) const { - if (is_high_level(level)) { - return high_sparse_arg_min(level, node, slot_left, slot_right, count); - } const Bp512Selector& selector = selector_at(level, node); return selector.arg_min_with_zero_prefix(slot_left, slot_right, count); } - /** - * @brief Return whether a level uses the high-node layout. - */ - bool is_high_level(std::size_t level) const { - return level > 0 && level >= high_level_begin_ && level < level_count(); - } - /** * @brief Map a medium-level node to its flat storage index. */ @@ -1558,115 +1471,168 @@ class SegmentBTreeXL } /** - * @brief Map a high-level node to its flat storage index. + * @brief Build a single top sparse table over original-value block minima. */ - std::size_t high_flat_index(std::size_t level, std::size_t node) const { - return high_level_offsets_[level] + node; + void build_top_sparse_table() { + top_sparse_candidates_.clear(); + top_block_size_ = top_sparse_block_size_for(values_.size()); + top_block_count_ = top_sparse_block_count_for(values_.size()); + top_sparse_levels_ = + top_block_count_ == 0 ? 0 : std::bit_width(top_block_count_); + if (top_block_count_ == 0) { + return; + } + + top_sparse_candidates_.assign(top_sparse_levels_ * top_block_count_, + TopCandidate{}); + for (std::size_t block = 0; block < top_block_count_; ++block) { + const std::size_t begin = block * top_block_size_; + const std::size_t end = std::min(values_.size(), begin + top_block_size_); + std::size_t minimum = begin; + for (std::size_t position = begin + 1; position < end; ++position) { + if (strictly_better_value_position(position, minimum)) { + minimum = position; + } + } + top_sparse_candidates_[block] = make_top_candidate(minimum); + } + + for (std::size_t level = 1; level < top_sparse_levels_; ++level) { + const std::size_t span = std::size_t{1} << level; + const std::size_t half_span = span >> 1; + TopCandidate* current = + top_sparse_candidates_.data() + level * top_block_count_; + const TopCandidate* previous = + top_sparse_candidates_.data() + (level - 1) * top_block_count_; + for (std::size_t block = 0; block + span <= top_block_count_; ++block) { + current[block] = + better_top_candidate(previous[block], previous[block + half_span]); + } + } } /** - * @brief Return the first child-metadata record for a high node. + * @brief Wrap an original value position as a top sparse-table candidate. */ - const HighChildMetadata* high_child_metadata_begin(std::size_t level, - std::size_t node) const { - return high_child_metadata_.data() + high_flat_index(level, node) * Fanout; + TopCandidate make_top_candidate(std::size_t position) const { + if (!valid_value_position(position)) { + return {}; + } + return {static_cast(position)}; } /** - * @brief Return mutable sparse-slot storage for one high node. + * @brief Return whether @p position is a valid original-value index. */ - std::uint8_t* mutable_high_sparse_min_slots_begin(std::size_t high_flat) { - return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; + bool valid_value_position(std::size_t position) const { + return !missing_position(position) && position < values_.size(); } /** - * @brief Return sparse-slot storage for one high node. + * @brief Return whether value position @p left is strictly better than @p + * right. */ - const std::uint8_t* high_sparse_min_slots_begin(std::size_t high_flat) const { - return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; + bool strictly_better_value_position(std::size_t left, + std::size_t right) const { + if (!valid_value_position(left)) { + return false; + } + if (!valid_value_position(right)) { + return true; + } + if (compare_(values_[left], values_[right])) { + return true; + } + if (compare_(values_[right], values_[left])) { + return false; + } + return left < right; } /** - * @brief Return high-child metadata by flat high-node index and slot. + * @brief Choose the better original-value candidate, preserving first ties. */ - const HighChildMetadata& high_child_metadata_at(std::size_t high_flat, - std::size_t slot) const { - return high_child_metadata_[high_flat * Fanout + slot]; + TopCandidate better_top_candidate(TopCandidate left, + TopCandidate right) const { + const std::size_t left_position = static_cast(left.position); + const std::size_t right_position = static_cast(right.position); + return strictly_better_value_position(right_position, left_position) ? right + : left; } /** - * @brief Return mutable high-child metadata while building. + * @brief Return the sparse-table candidate over a top-block range. */ - HighChildMetadata& mutable_high_child_metadata_at(std::size_t high_flat, - std::size_t slot) { - return high_child_metadata_[high_flat * Fanout + slot]; + TopCandidate top_sparse_block_arg_min(std::size_t block_left, + std::size_t block_right) const { + if (block_left >= block_right || block_right > top_block_count_ || + top_sparse_levels_ == 0) { + return {}; + } + const std::size_t length = block_right - block_left; + const std::size_t level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << level; + const TopCandidate* table = + top_sparse_candidates_.data() + level * top_block_count_; + return better_top_candidate(table[block_left], table[block_right - span]); } /** - * @brief Choose the better high-node child slot. + * @brief Return whether a candidate lies inside a half-open value range. */ - std::size_t better_high_child_slot(std::size_t level, - std::size_t node, - std::size_t left_slot, - std::size_t right_slot) const { - const MinCandidate left = high_child_min_candidate(level, node, left_slot); - const MinCandidate right = - high_child_min_candidate(level, node, right_slot); - return better_candidate(left, right).position == right.position ? right_slot - : left_slot; + bool top_candidate_inside(TopCandidate candidate, + std::size_t left, + std::size_t right) const { + const std::size_t position = static_cast(candidate.position); + return valid_value_position(position) && left <= position && + position < right; } /** - * @brief Build sparse tables over high-node child minima. + * @brief Return the top-overlay answer, or `npos` when the tree should run. */ - void build_high_sparse_min_slots(std::size_t level, - std::size_t node, - std::size_t count) { - const std::size_t high_flat = high_flat_index(level, node); - std::uint8_t* table = mutable_high_sparse_min_slots_begin(high_flat); - for (std::size_t slot = 0; slot < count; ++slot) { - table[slot] = static_cast(slot); - } - - for (std::size_t table_level = 1; table_level < kHighSparseLevels; - ++table_level) { - const std::size_t span = std::size_t{1} << table_level; - if (span > count) { - break; - } - const std::size_t half_span = span >> 1; - const std::uint8_t* previous = table + (table_level - 1) * Fanout; - std::uint8_t* current = table + table_level * Fanout; - for (std::size_t slot = 0; slot + span <= count; ++slot) { - current[slot] = static_cast(better_high_child_slot( - level, node, previous[slot], previous[slot + half_span])); - } + std::size_t top_sparse_arg_min(std::size_t left, std::size_t right) const { + if (top_block_count_ <= 1) { + return npos; } - } - /** - * @brief Return the best high-node child slot in a slot range. - */ - std::size_t high_sparse_arg_min(std::size_t level, - std::size_t node, - std::size_t slot_left, - std::size_t slot_right, - std::size_t count) const { - if (slot_left >= slot_right || slot_right > count) { + const std::size_t padded_block_left = left / top_block_size_; + const std::size_t padded_block_right = (right - 1) / top_block_size_ + 1; + if (padded_block_left + 1 >= padded_block_right) { + return npos; + } + + const TopCandidate padded = + top_sparse_block_arg_min(padded_block_left, padded_block_right); + if (top_candidate_inside(padded, left, right)) { + return static_cast(padded.position); + } + + const std::size_t first_full_block = + (left + top_block_size_ - 1) / top_block_size_; + const std::size_t full_block_right = right / top_block_size_; + if (first_full_block >= full_block_right) { return npos; } - const std::size_t length = slot_right - slot_left; - if (length == 1) { - return slot_left; + + TopCandidate answer = + top_sparse_block_arg_min(first_full_block, full_block_right); + + const std::size_t left_border_end = first_full_block * top_block_size_; + if (left < left_border_end) { + answer = better_top_candidate( + answer, make_top_candidate(tree_arg_min(left, left_border_end))); + } + + const std::size_t right_border_begin = full_block_right * top_block_size_; + if (right_border_begin < right) { + answer = better_top_candidate( + answer, make_top_candidate(tree_arg_min(right_border_begin, right))); } - const std::size_t high_flat = high_flat_index(level, node); - const std::size_t table_level = std::bit_width(length) - 1; - const std::size_t span = std::size_t{1} << table_level; - const std::uint8_t* table = - high_sparse_min_slots_begin(high_flat) + table_level * Fanout; - return better_high_child_slot(level, node, table[slot_left], - table[slot_right - span]); + return valid_value_position(static_cast(answer.position)) + ? static_cast(answer.position) + : npos; } std::span values_; @@ -1674,30 +1640,14 @@ class SegmentBTreeXL std::vector leaf_selectors_; std::vector medium_selectors_; std::vector medium_min_values_; - std::vector high_selectors_; - std::vector high_min_positions_; - std::vector high_min_values_; - std::vector high_child_metadata_; - std::vector high_sparse_min_slots_; + std::vector top_sparse_candidates_; std::vector medium_level_offsets_; - std::vector high_level_offsets_; std::vector level_sizes_; std::vector level_value_spans_; std::vector level_fanouts_; - std::size_t high_level_begin_ = std::numeric_limits::max(); + std::size_t top_block_size_ = kMinTopSparseBlockSize; + std::size_t top_block_count_ = 0; + std::size_t top_sparse_levels_ = 0; }; -/** - * @brief Compatibility alias for the historical SegmentBTreeXl spelling. - */ -template , - class Index = std::size_t, - std::size_t LeafSize = 496, - std::size_t Fanout = 256, - SegmentBTreeXLLeafSelector LeafSelectorKind = - SegmentBTreeXLLeafSelector::PrefixSuffix> -using SegmentBTreeXl = - SegmentBTreeXL; - } // namespace pixie::rmq diff --git a/include/pixie/rmq/node_euler_btree_rmq.h b/include/pixie/rmq/node_euler_btree_rmq.h deleted file mode 100644 index cd7292a..0000000 --- a/include/pixie/rmq/node_euler_btree_rmq.h +++ /dev/null @@ -1,1016 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace pixie::rmq { - -/** - * @brief Value RMQ with per-node Cartesian/Euler selectors. - * - * @details Values are split into fixed-size leaves, then grouped by a B-tree. - * Every leaf stores a local Cartesian BP selector over original values. Every - * internal node stores the same kind of selector over child subtree minima. - * Queries first enter the lowest B-tree node covering both endpoints, then ask - * that node's selector over all intersecting children, including partial border - * children. If the selected child is fully covered, or its stored subtree - * minimum is inside the query, the answer is known immediately. Otherwise the - * selected border child is corrected recursively and compared with the - * remaining child slots. - * - * This backend is available as the main per-node Euler B-tree RMQ - * implementation. - * - * @tparam T Value type in the indexed array. - * @tparam Compare Strict weak ordering used to choose minima. - * @tparam Index Unsigned integer type used for stored positions. - * @tparam LeafSize Number of original values per leaf. - * @tparam Fanout Maximum number of children per internal node. - */ -template , - class Index = std::size_t, - std::size_t LeafSize = 256, - std::size_t Fanout = 256> -class NodeEulerBTreeRmq - : public RmqBase, - T> { - public: - static_assert(std::is_unsigned_v, - "NodeEulerBTreeRmq index type must be unsigned"); - static_assert(LeafSize > 0); - static_assert(Fanout > 1); - - static constexpr std::size_t npos = - RmqBase, T>::npos; - static constexpr Index invalid_index = std::numeric_limits::max(); - static constexpr std::size_t kLeafSize = LeafSize; - static constexpr std::size_t kFanout = Fanout; - - /** - * @brief Construct an empty RMQ index. - */ - NodeEulerBTreeRmq() = default; - NodeEulerBTreeRmq(const NodeEulerBTreeRmq&) = default; - NodeEulerBTreeRmq(NodeEulerBTreeRmq&&) noexcept = default; - NodeEulerBTreeRmq& operator=(const NodeEulerBTreeRmq&) = default; - NodeEulerBTreeRmq& operator=(NodeEulerBTreeRmq&&) noexcept = default; - - /** - * @brief Build a B-tree RMQ index over @p values. - * - * @details The values are not copied and must outlive this object. Equal - * values keep the smaller original position as the answer. - * - * @param values Values to index. - * @param compare Ordering used to choose minima. - * @throws std::length_error if @p Index cannot represent all positions. - */ - explicit NodeEulerBTreeRmq(std::span values, - Compare compare = Compare()) - : values_(values), compare_(compare) { - build(); - } - - /** - * @brief Return the number of indexed values. - */ - std::size_t size_impl() const { return values_.size(); } - - /** - * @brief Return the value at an indexed position. - */ - T value_at_impl(std::size_t position) const { return values_[position]; } - - /** - * @brief Return the first minimum position in [@p left, @p right). - * - * @details Empty or invalid ranges return `npos`. Ties are reduced by - * comparing `(value, position)`, so traversal order cannot change first-min - * semantics. - */ - std::size_t arg_min_impl(std::size_t left, std::size_t right) const { - if (left >= right || right > values_.size() || nodes_.empty()) { - return npos; - } - - const NodeMetadata& root = nodes_.front(); - if (left == root.value_begin && right == root.value_end) { - return root.subtree_min_position; - } - - const bool allow_leaf_linear_scan = - right - left <= kLeafLinearScanThreshold; - const std::size_t left_leaf = leaf_for_value(left); - const std::size_t right_leaf = leaf_for_value(right - 1); - if (left_leaf == right_leaf) { - const NodeMetadata& leaf = node_at(0, left_leaf); - if (left == leaf.value_begin && right == leaf.value_end) { - return leaf.subtree_min_position; - } - return leaf_range_min(leaf, selector_at(0, left_leaf), left, right, - allow_leaf_linear_scan); - } - - const auto [level, node_index] = covering_node(left_leaf, right_leaf); - return query_node(level, node_index, left, right, allow_leaf_linear_scan); - } - - private: - static constexpr std::size_t kMaxLocalEntries = - LeafSize > Fanout ? LeafSize : Fanout; - static constexpr std::size_t kMaxLocalBits = 2 * kMaxLocalEntries; - static constexpr std::size_t kMaxLocalWords = (kMaxLocalBits + 63) / 64; - static constexpr std::size_t kMaxDepthBlocks = (kMaxLocalBits + 1 + 63) / 64; - static constexpr std::size_t kInternalScalarSlotThreshold = 4; - static constexpr std::size_t kLeafLinearScanThreshold = 64; - static constexpr std::size_t kLeafAvx2ScanThreshold = 16; - static constexpr bool kLeafSizePowerOfTwo = (LeafSize & (LeafSize - 1)) == 0; - static constexpr bool kFanoutPowerOfTwo = (Fanout & (Fanout - 1)) == 0; - static constexpr bool kPowerOfTwoLayout = - kLeafSizePowerOfTwo && kFanoutPowerOfTwo; - static constexpr bool kInvalidIndexEqualsNpos = - static_cast(invalid_index) == npos; - - static_assert( - kMaxLocalBits <= - static_cast(std::numeric_limits::max())); - static_assert( - kMaxLocalEntries <= - static_cast(std::numeric_limits::max())); - - struct DepthBlockSummary { - std::int16_t base_depth = 0; - std::int16_t min_depth = 0; - std::uint16_t min_offset = 0; - std::uint16_t size = 0; - }; - - class LocalSelector { - public: - LocalSelector() = default; - - template - void build(std::size_t entry_count, EntryLess entry_less) { - if (entry_count > kMaxLocalEntries) { - throw std::length_error("NodeEulerBTreeRmq local selector too large"); - } - - entry_count_ = static_cast(entry_count); - bit_count_ = static_cast(2 * entry_count); - word_count_ = static_cast((bit_count_ + 63) / 64); - depth_block_count_ = - static_cast((bit_count_ + 1 + 63) / 64); - bp_bits_.fill(0); - close_positions_.fill(0); - depth_blocks_.fill({}); - zero_rank_prefix_.fill(0); - - if (entry_count == 0) { - return; - } - - std::array stack{}; - std::size_t stack_size = 0; - std::size_t write_position = bit_count_; - - for (std::size_t i = entry_count; i-- > 0;) { - while (stack_size != 0 && !entry_less(stack[stack_size - 1], i)) { - --stack_size; - prepend_bp_bit(write_position, true); - } - stack[stack_size++] = static_cast(i); - close_positions_[i] = prepend_bp_bit(write_position, false); - } - - while (write_position != 0) { - prepend_bp_bit(write_position, true); - } - build_depth_blocks(); - build_zero_rank_prefix(); - } - - std::size_t arg_min(std::size_t slot_left, - std::size_t slot_right, - bool force_scalar = false) const { - if (slot_left >= slot_right || slot_right > entry_count_) { - return npos; - } - if (slot_left + 1 == slot_right) { - return slot_left; - } - - const std::size_t first_close = close_positions_[slot_left]; - const std::size_t last_close = close_positions_[slot_right - 1]; - if (first_close > last_close) { - return npos; - } - - const std::size_t shifted_min = - depth_arg_min(first_close + 1, last_close + 2, force_scalar); - if (shifted_min == npos || shifted_min == 0) { - return npos; - } - - const std::size_t zero_rank = rank0(shifted_min); - if (zero_rank == 0) { - return npos; - } - const std::size_t entry = zero_rank - 1; - return entry < entry_count_ ? entry : npos; - } - - std::size_t entry_count() const { return entry_count_; } - - std::size_t depth_count() const { return bit_count_ + 1; } - - std::size_t close_position(std::size_t slot) const { - return close_positions_[slot]; - } - - std::int16_t depth_at_position(std::size_t position) const { - return depth_at(position); - } - - std::size_t rank0_at(std::size_t position) const { return rank0(position); } - - private: - std::size_t prepend_bp_bit(std::size_t& write_position, bool bit) { - --write_position; - if (bit) { - bp_bits_[write_position >> 6] |= std::uint64_t{1} - << (write_position & 63); - } - return write_position; - } - - bool bit(std::size_t position) const { - return ((bp_bits_[position >> 6] >> (position & 63)) & 1u) != 0; - } - - static std::uint64_t low_bits_mask(std::size_t count) { - if (count == 0) { - return 0; - } - if (count >= 64) { - return ~std::uint64_t{0}; - } - return (std::uint64_t{1} << count) - 1; - } - - std::int16_t prefix_excess_in_word(std::size_t word, - std::size_t count) const { - if (count == 0 || word >= word_count_) { - return 0; - } - const std::uint64_t bits = bp_bits_[word] & low_bits_mask(count); - const std::size_t ones = std::popcount(bits); - return static_cast(2 * static_cast(ones) - - static_cast(count)); - } - - std::int16_t depth_at(std::size_t position) const { - const std::size_t block = position >> 6; - const std::size_t offset = position & 63; - return static_cast(depth_blocks_[block].base_depth + - prefix_excess_in_word(block, offset)); - } - - void build_depth_blocks() { - const std::size_t depth_count = bit_count_ + 1; - std::int16_t current_depth = 0; - - for (std::size_t block = 0; block < depth_block_count_; ++block) { - const std::size_t block_begin = block * 64; - const std::size_t block_size = - std::min(64, depth_count - block_begin); - DepthBlockSummary summary; - summary.base_depth = current_depth; - summary.min_depth = current_depth; - summary.size = static_cast(block_size); - - for (std::size_t offset = 1; offset < block_size; ++offset) { - current_depth = static_cast( - current_depth + (bit(block_begin + offset - 1) ? 1 : -1)); - if (current_depth < summary.min_depth) { - summary.min_depth = current_depth; - summary.min_offset = static_cast(offset); - } - } - - depth_blocks_[block] = summary; - if (block_begin + block_size < depth_count) { - current_depth = static_cast( - current_depth + (bit(block_begin + block_size - 1) ? 1 : -1)); - } - } - } - - void build_zero_rank_prefix() { - zero_rank_prefix_[0] = 0; - for (std::size_t word = 0; word < word_count_; ++word) { - const std::size_t word_begin = word * 64; - const std::size_t word_bits = - std::min(64, bit_count_ - word_begin); - const std::uint64_t logical_bits = - bp_bits_[word] & low_bits_mask(word_bits); - const std::size_t zeros = word_bits - std::popcount(logical_bits); - zero_rank_prefix_[word + 1] = - static_cast(zero_rank_prefix_[word] + zeros); - } - } - - std::size_t depth_arg_min_scalar(std::size_t left, - std::size_t right) const { - const std::size_t depth_count = bit_count_ + 1; - if (left >= right || right > depth_count) { - return npos; - } - - std::size_t position = left; - std::int16_t best_depth = depth_at(position); - std::size_t best_position = position; - std::int16_t current_depth = best_depth; - ++position; - - while (position < right && (position & 63) != 0) { - current_depth = static_cast(current_depth + - (bit(position - 1) ? 1 : -1)); - if (current_depth < best_depth) { - best_depth = current_depth; - best_position = position; - } - ++position; - } - - while (position + 64 <= right) { - const DepthBlockSummary& summary = depth_blocks_[position >> 6]; - if (summary.min_depth < best_depth) { - best_depth = summary.min_depth; - best_position = position + summary.min_offset; - } - position += 64; - } - - if (position < right) { - current_depth = depth_at(position); - if (current_depth < best_depth) { - best_depth = current_depth; - best_position = position; - } - ++position; - } - while (position < right) { - current_depth = static_cast(current_depth + - (bit(position - 1) ? 1 : -1)); - if (current_depth < best_depth) { - best_depth = current_depth; - best_position = position; - } - ++position; - } - - return best_position; - } - - std::size_t depth_arg_min(std::size_t left, - std::size_t right, - bool force_scalar) const { - const std::size_t depth_count = bit_count_ + 1; - if (left >= right || right > depth_count) { - return npos; - } - if (force_scalar) { - return depth_arg_min_scalar(left, right); - } - - std::size_t position = left; - std::int16_t best_depth = depth_at(position); - std::size_t best_position = position; - - while (position < right) { - const std::size_t chunk_begin = (position / 128) * 128; - const std::size_t local_left = position - chunk_begin; - const std::size_t local_right = - std::min(right - 1, chunk_begin + 128) - chunk_begin; - - if (chunk_begin >= bit_count_) { - const std::int16_t candidate_depth = depth_at(chunk_begin); - if (candidate_depth < best_depth) { - best_depth = candidate_depth; - best_position = chunk_begin; - } - } else { - const std::size_t word = chunk_begin >> 6; - const ExcessResult candidate = - excess_min_128(bp_bits_.data() + word, local_left, local_right); - const std::int16_t candidate_depth = static_cast( - depth_at(chunk_begin) + candidate.min_excess); - const std::size_t candidate_position = chunk_begin + candidate.offset; - if (candidate_depth < best_depth) { - best_depth = candidate_depth; - best_position = candidate_position; - } - } - - position = chunk_begin + local_right + 1; - } - - return best_position; - } - - std::size_t rank0(std::size_t position) const { - position = std::min(position, bit_count_); - const std::size_t full_words = position >> 6; - std::size_t zeros = zero_rank_prefix_[full_words]; - const std::size_t tail_bits = position & 63; - if (tail_bits != 0) { - const std::uint64_t tail = - bp_bits_[full_words] & low_bits_mask(tail_bits); - zeros += tail_bits - std::popcount(tail); - } - return zeros; - } - - std::array bp_bits_{}; - std::array close_positions_{}; - std::array depth_blocks_{}; - std::array zero_rank_prefix_{}; - std::uint16_t entry_count_ = 0; - std::uint16_t bit_count_ = 0; - std::uint16_t word_count_ = 0; - std::uint16_t depth_block_count_ = 0; - }; - - class TopDepthSparseSelector { - public: - TopDepthSparseSelector() = default; - - void build(const LocalSelector& selector) { - const std::size_t depth_count = selector.depth_count(); - depth_count_ = depth_count; - depths_.assign(depth_count, 0); - if (depth_count == 0) { - log_count_ = 0; - sparse_positions_.clear(); - return; - } - - for (std::size_t position = 0; position < depth_count; ++position) { - depths_[position] = selector.depth_at_position(position); - } - - log_count_ = std::bit_width(depth_count); - sparse_positions_.assign(log_count_ * depth_count, 0); - for (std::size_t position = 0; position < depth_count; ++position) { - sparse_positions_[position] = static_cast(position); - } - - for (std::size_t level = 1; level < log_count_; ++level) { - const std::size_t half_span = std::size_t{1} << (level - 1); - const std::size_t span = half_span << 1; - if (span > depth_count) { - break; - } - const std::size_t previous_offset = (level - 1) * depth_count; - const std::size_t current_offset = level * depth_count; - for (std::size_t position = 0; position + span <= depth_count; - ++position) { - sparse_positions_[current_offset + position] = better_depth_position( - sparse_positions_[previous_offset + position], - sparse_positions_[previous_offset + position + half_span]); - } - } - } - - std::size_t arg_min(const LocalSelector& selector, - std::size_t slot_left, - std::size_t slot_right) const { - if (slot_left >= slot_right || slot_right > selector.entry_count()) { - return npos; - } - if (slot_left + 1 == slot_right) { - return slot_left; - } - - const std::size_t first_close = selector.close_position(slot_left); - const std::size_t last_close = selector.close_position(slot_right - 1); - if (first_close > last_close) { - return npos; - } - - const std::size_t shifted_min = - depth_arg_min(first_close + 1, last_close + 2); - if (shifted_min == npos || shifted_min == 0) { - return npos; - } - - const std::size_t zero_rank = selector.rank0_at(shifted_min); - if (zero_rank == 0) { - return npos; - } - const std::size_t entry = zero_rank - 1; - return entry < selector.entry_count() ? entry : npos; - } - - private: - std::uint16_t better_depth_position(std::uint16_t left, - std::uint16_t right) const { - const std::int16_t left_depth = depths_[left]; - const std::int16_t right_depth = depths_[right]; - if (right_depth < left_depth) { - return right; - } - if (left_depth < right_depth) { - return left; - } - return std::min(left, right); - } - - std::size_t depth_arg_min(std::size_t left, std::size_t right) const { - if (left >= right || right > depth_count_) { - return npos; - } - const std::size_t length = right - left; - const std::size_t level = std::bit_width(length) - 1; - const std::size_t span = std::size_t{1} << level; - const std::size_t offset = level * depth_count_; - return better_depth_position(sparse_positions_[offset + left], - sparse_positions_[offset + right - span]); - } - - std::vector depths_; - std::vector sparse_positions_; - std::size_t depth_count_ = 0; - std::size_t log_count_ = 0; - }; - - static constexpr std::size_t log2_exact(std::size_t value) { - std::size_t shift = 0; - while (value > 1) { - value >>= 1; - ++shift; - } - return shift; - } - - static constexpr std::size_t kLeafShift = - kLeafSizePowerOfTwo ? log2_exact(LeafSize) : 0; - static constexpr std::size_t kFanoutShift = - kFanoutPowerOfTwo ? log2_exact(Fanout) : 0; - - struct alignas(64) NodeMetadata { - std::size_t value_begin = 0; - std::size_t value_end = 0; - std::size_t first_entry = 0; - Index subtree_min_position = invalid_index; - std::uint16_t entry_count = 0; - std::array - padding{}; - }; - - static_assert(sizeof(NodeMetadata) == 64); - - bool missing_position(std::size_t position) const { - if constexpr (kInvalidIndexEqualsNpos) { - return position == npos; - } else { - return position == npos || - position == static_cast(invalid_index); - } - } - - bool strictly_better_position(std::size_t left, std::size_t right) const { - if (missing_position(left)) { - return false; - } - if (missing_position(right)) { - return true; - } - return compare_(values_[left], values_[right]); - } - - std::size_t better_position(std::size_t left, std::size_t right) const { - if (missing_position(left)) { - return right; - } - if (missing_position(right)) { - return left; - } - if (compare_(values_[right], values_[left])) { - return right; - } - if (compare_(values_[left], values_[right])) { - return left; - } - return std::min(left, right); - } - - void build() { - nodes_.clear(); - selectors_.clear(); - top_depth_selectors_.clear(); - top_level_offsets_.clear(); - level_offsets_.clear(); - level_sizes_.clear(); - level_value_spans_.clear(); - top_level_begin_ = std::numeric_limits::max(); - if (values_.empty()) { - return; - } - if (values_.size() > static_cast(invalid_index)) { - throw std::length_error("NodeEulerBTreeRmq index type is too small"); - } - - initialize_layout((values_.size() + LeafSize - 1) / LeafSize); - const std::size_t leaf_count = level_sizes_[0]; - for (std::size_t leaf = 0; leaf < leaf_count; ++leaf) { - build_leaf(leaf); - } - - for (std::size_t child_level = 0; child_level + 1 < level_count(); - ++child_level) { - const std::size_t parent_count = level_sizes_[child_level + 1]; - for (std::size_t parent = 0; parent < parent_count; ++parent) { - build_internal_node(child_level, parent); - } - } - } - - void initialize_layout(std::size_t leaf_count) { - level_sizes_.push_back(leaf_count); - level_value_spans_.push_back(LeafSize); - while (level_sizes_.back() > 1) { - level_sizes_.push_back((level_sizes_.back() + Fanout - 1) / Fanout); - level_value_spans_.push_back(saturating_product( - level_value_spans_.back(), static_cast(Fanout))); - } - - level_offsets_.assign(level_sizes_.size(), 0); - std::size_t total_nodes = 0; - for (std::size_t level = level_sizes_.size(); level-- > 0;) { - level_offsets_[level] = total_nodes; - total_nodes += level_sizes_[level]; - } - - nodes_.resize(total_nodes); - selectors_.resize(total_nodes); - initialize_top_depth_selector_layout(); - } - - void initialize_top_depth_selector_layout() { - top_level_begin_ = std::numeric_limits::max(); - top_level_offsets_.assign(level_sizes_.size(), 0); - top_depth_selectors_.clear(); - if (level_sizes_.size() <= 1) { - return; - } - - const std::size_t root_level = level_sizes_.size() - 1; - top_level_begin_ = root_level > 1 ? root_level - 1 : 1; - - std::size_t top_node_count = 0; - for (std::size_t level = top_level_begin_; level < level_sizes_.size(); - ++level) { - top_level_offsets_[level] = top_node_count; - top_node_count += level_sizes_[level]; - } - top_depth_selectors_.resize(top_node_count); - } - - void build_leaf(std::size_t leaf) { - NodeMetadata& node = mutable_node_at(0, leaf); - LocalSelector& selector = mutable_selector_at(0, leaf); - node.first_entry = leaf * LeafSize; - node.value_begin = node.first_entry; - node.entry_count = static_cast( - std::min(LeafSize, values_.size() - node.first_entry)); - node.value_end = node.value_begin + node.entry_count; - selector.build(node.entry_count, [&](std::size_t left, std::size_t right) { - return compare_(values_[node.first_entry + left], - values_[node.first_entry + right]); - }); - - const std::size_t slot = selector.arg_min(0, node.entry_count); - node.subtree_min_position = static_cast(node.first_entry + slot); - } - - void build_internal_node(std::size_t child_level, std::size_t parent) { - const std::size_t parent_level = child_level + 1; - NodeMetadata& node = mutable_node_at(parent_level, parent); - LocalSelector& selector = mutable_selector_at(parent_level, parent); - node.first_entry = parent * Fanout; - node.entry_count = static_cast(std::min( - Fanout, level_sizes_[child_level] - node.first_entry)); - node.value_begin = node_at(child_level, node.first_entry).value_begin; - node.value_end = - node_at(child_level, node.first_entry + node.entry_count - 1).value_end; - selector.build(node.entry_count, [&](std::size_t left, std::size_t right) { - return strictly_better_position( - node_at(child_level, node.first_entry + left).subtree_min_position, - node_at(child_level, node.first_entry + right).subtree_min_position); - }); - - const std::size_t slot = selector.arg_min(0, node.entry_count); - node.subtree_min_position = - node_at(child_level, node.first_entry + slot).subtree_min_position; - if (is_top_internal_level(parent_level)) { - mutable_top_depth_selector_at(parent_level, parent).build(selector); - } - } - - static std::size_t saturating_product(std::size_t left, std::size_t right) { - if (left != 0 && right > std::numeric_limits::max() / left) { - return std::numeric_limits::max(); - } - return left * right; - } - - std::size_t leaf_range_min(const NodeMetadata& node, - const LocalSelector& selector, - std::size_t left, - std::size_t right, - bool allow_linear_scan) const { - if (allow_linear_scan && right - left <= kLeafLinearScanThreshold) { - return linear_range_min(left, right); - } - const std::size_t slot_left = left - node.value_begin; - const std::size_t slot_right = right - node.value_begin; - const std::size_t slot = selector.arg_min(slot_left, slot_right); - if (slot == npos) { - return npos; - } - return node.value_begin + slot; - } - - std::size_t linear_range_min(std::size_t left, std::size_t right) const { - if (left >= right) { - return npos; - } -#ifdef PIXIE_AVX2_SUPPORT - if constexpr (std::is_same_v && - std::is_same_v>) { - if (right - left >= kLeafAvx2ScanThreshold) { - return linear_range_min_i64_avx2(left, right); - } - } -#endif - std::size_t best = left; - for (std::size_t position = left + 1; position < right; ++position) { - if (compare_(values_[position], values_[best])) { - best = position; - } - } - return best; - } - -#ifdef PIXIE_AVX2_SUPPORT - std::size_t linear_range_min_i64_avx2(std::size_t left, - std::size_t right) const { - const std::int64_t* data = values_.data(); - std::size_t position = left; - - __m256i best_values = - _mm256_loadu_si256(reinterpret_cast(data + position)); - __m256i best_positions = _mm256_set_epi64x( - static_cast(position + 3), - static_cast(position + 2), - static_cast(position + 1), static_cast(position)); - position += 4; - - for (; position + 4 <= right; position += 4) { - const __m256i values = - _mm256_loadu_si256(reinterpret_cast(data + position)); - const __m256i positions = - _mm256_set_epi64x(static_cast(position + 3), - static_cast(position + 2), - static_cast(position + 1), - static_cast(position)); - const __m256i take_new = _mm256_cmpgt_epi64(best_values, values); - best_values = _mm256_blendv_epi8(best_values, values, take_new); - best_positions = _mm256_blendv_epi8(best_positions, positions, take_new); - } - - alignas(32) std::int64_t value_lanes[4]; - alignas(32) std::uint64_t position_lanes[4]; - _mm256_store_si256(reinterpret_cast<__m256i*>(value_lanes), best_values); - _mm256_store_si256(reinterpret_cast<__m256i*>(position_lanes), - best_positions); - - std::int64_t best_value = value_lanes[0]; - std::size_t best_position = static_cast(position_lanes[0]); - for (std::size_t lane = 1; lane < 4; ++lane) { - const std::size_t lane_position = - static_cast(position_lanes[lane]); - if (value_lanes[lane] < best_value || - (value_lanes[lane] == best_value && lane_position < best_position)) { - best_value = value_lanes[lane]; - best_position = lane_position; - } - } - - for (; position < right; ++position) { - if (data[position] < best_value) { - best_value = data[position]; - best_position = position; - } - } - return best_position; - } -#endif - - std::pair covering_node( - std::size_t left_leaf, - std::size_t right_leaf) const { - std::size_t level = 0; - std::size_t left_node = left_leaf; - std::size_t right_node = right_leaf; - while (left_node != right_node) { - ++level; - if constexpr (kFanoutPowerOfTwo) { - left_node >>= kFanoutShift; - right_node >>= kFanoutShift; - } else { - left_node /= Fanout; - right_node /= Fanout; - } - } - return {level, left_node}; - } - - std::size_t leaf_for_value(std::size_t position) const { - if constexpr (kLeafSizePowerOfTwo) { - return position >> kLeafShift; - } else { - return position / LeafSize; - } - } - - std::size_t child_for_value(std::size_t child_level, - std::size_t position) const { - if constexpr (kPowerOfTwoLayout) { - return position >> (kLeafShift + child_level * kFanoutShift); - } else { - return position / level_value_spans_[child_level]; - } - } - - bool contains_position(std::size_t left, - std::size_t right, - std::size_t position) const { - return !missing_position(position) && left <= position && position < right; - } - - std::size_t query_child_slots(std::size_t level, - std::size_t node_index, - std::size_t slot_left, - std::size_t slot_right, - std::size_t left, - std::size_t right, - bool allow_leaf_linear_scan) const { - if (slot_left >= slot_right) { - return npos; - } - - const NodeMetadata& node = node_at(level, node_index); - const std::size_t child_level = level - 1; - const std::size_t slot_count = slot_right - slot_left; - const std::size_t slot = - slot_count == 1 - ? slot_left - : selector_arg_min(level, node_index, slot_left, slot_right, - slot_count <= kInternalScalarSlotThreshold); - if (slot == npos) { - return npos; - } - - const std::size_t child_index = node.first_entry + slot; - const NodeMetadata& child = node_at(child_level, child_index); - if ((left <= child.value_begin && child.value_end <= right) || - contains_position(left, right, child.subtree_min_position)) { - return child.subtree_min_position; - } - - std::size_t answer = - query_node(child_level, child_index, std::max(left, child.value_begin), - std::min(right, child.value_end), allow_leaf_linear_scan); - answer = better_position( - answer, query_child_slots(level, node_index, slot_left, slot, left, - right, allow_leaf_linear_scan)); - answer = better_position( - answer, query_child_slots(level, node_index, slot + 1, slot_right, left, - right, allow_leaf_linear_scan)); - return answer; - } - - std::size_t query_node(std::size_t level, - std::size_t node_index, - std::size_t left, - std::size_t right, - bool allow_leaf_linear_scan) const { - if (left >= right) { - return npos; - } - const NodeMetadata& node = node_at(level, node_index); - if (left <= node.value_begin && node.value_end <= right) { - return node.subtree_min_position; - } - if (level == 0) { - return leaf_range_min(node, selector_at(level, node_index), left, right, - allow_leaf_linear_scan); - } - - const std::size_t child_level = level - 1; - const std::size_t left_child = child_for_value(child_level, left); - const std::size_t right_child = child_for_value(child_level, right - 1); - const std::size_t left_slot = left_child - node.first_entry; - const std::size_t right_slot = right_child - node.first_entry + 1; - return query_child_slots(level, node_index, left_slot, right_slot, left, - right, allow_leaf_linear_scan); - } - - std::size_t level_count() const { return level_offsets_.size(); } - - std::size_t flat_index(std::size_t level, std::size_t node_index) const { - return level_offsets_[level] + node_index; - } - - bool is_top_internal_level(std::size_t level) const { - return level > 0 && level >= top_level_begin_ && level < level_count(); - } - - std::size_t top_flat_index(std::size_t level, std::size_t node_index) const { - return top_level_offsets_[level] + node_index; - } - - std::size_t selector_arg_min(std::size_t level, - std::size_t node_index, - std::size_t slot_left, - std::size_t slot_right, - bool force_scalar) const { - const LocalSelector& selector = selector_at(level, node_index); - if (is_top_internal_level(level)) { - return top_depth_selector_at(level, node_index) - .arg_min(selector, slot_left, slot_right); - } - return selector.arg_min(slot_left, slot_right, force_scalar); - } - - const NodeMetadata& node_at(std::size_t level, std::size_t node_index) const { - return nodes_[flat_index(level, node_index)]; - } - - NodeMetadata& mutable_node_at(std::size_t level, std::size_t node_index) { - return nodes_[flat_index(level, node_index)]; - } - - const LocalSelector& selector_at(std::size_t level, - std::size_t node_index) const { - return selectors_[flat_index(level, node_index)]; - } - - LocalSelector& mutable_selector_at(std::size_t level, - std::size_t node_index) { - return selectors_[flat_index(level, node_index)]; - } - - const TopDepthSparseSelector& top_depth_selector_at( - std::size_t level, - std::size_t node_index) const { - return top_depth_selectors_[top_flat_index(level, node_index)]; - } - - TopDepthSparseSelector& mutable_top_depth_selector_at( - std::size_t level, - std::size_t node_index) { - return top_depth_selectors_[top_flat_index(level, node_index)]; - } - - std::span values_; - Compare compare_; - std::vector nodes_; - std::vector selectors_; - std::vector top_depth_selectors_; - std::vector level_offsets_; - std::vector top_level_offsets_; - std::vector level_sizes_; - std::vector level_value_spans_; - std::size_t top_level_begin_ = std::numeric_limits::max(); -}; - -} // namespace pixie::rmq diff --git a/include/pixie/rmq/rmq_one_interval_btree.h b/include/pixie/rmq/rmq_one_interval_btree.h deleted file mode 100644 index 0453b49..0000000 --- a/include/pixie/rmq/rmq_one_interval_btree.h +++ /dev/null @@ -1,833 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace pixie::rmq { - -/** - * @brief Interval B-tree backend for arrays with adjacent differences +/-1. - * - * @details The indexed depth sequence is represented by BP deltas: bit 1 means - * the next depth is current + 1, and bit 0 means current - 1. The structure - * keeps 128-depth-position lower blocks separate and builds a cache-aligned - * interval hierarchy above those blocks. Queries scan partial edge blocks with - * the existing 128-bit excess-min primitives and cover the aligned middle with - * B-tree summary nodes. - * - * @tparam Index Unsigned integer type used to validate representable - * positions. - * @tparam HighCacheLines Number of 64-byte cache lines assigned to one high - * summary node. - * @tparam LowFanout Number of lower-block summaries stored in one low node. - * @tparam UseBoundaryRecords Whether to precompute left-to-right and - * right-to-left block minimum record positions for cross-block edge queries. - */ -template -class OneIntervalBTreeRmq { - public: - static_assert(std::is_unsigned_v, - "OneIntervalBTreeRmq index type must be unsigned"); - static_assert(HighCacheLines > 0); - static_assert(LowFanout > 0); - - static constexpr std::size_t npos = std::numeric_limits::max(); - static constexpr Index invalid_index = std::numeric_limits::max(); - static constexpr std::size_t kBlockSize = 128; - static constexpr std::size_t kChunkSize = 128; - static constexpr std::size_t kBlockChunks = kBlockSize / kChunkSize; - static constexpr std::size_t kChunkWords = kChunkSize / 64; - static constexpr bool kUseBoundaryRecords = UseBoundaryRecords; - static constexpr std::size_t kCacheLineBytes = 64; - static constexpr std::size_t kLowFanout = LowFanout; - static constexpr std::size_t kHighFanout = - std::max(2, (512 * HighCacheLines) / (2 * 64)); - static constexpr std::size_t kMaxFanout = std::max(kLowFanout, kHighFanout); - - static_assert(kBlockSize % kChunkSize == 0); - static_assert(kChunkSize == 128); - static_assert(kMaxFanout <= 64); - static_assert( - kBlockSize * kLowFanout <= - static_cast(std::numeric_limits::max())); - - /** - * @brief Construct an empty +/-1 RMQ index. - */ - OneIntervalBTreeRmq() = default; - OneIntervalBTreeRmq(const OneIntervalBTreeRmq&) = default; - OneIntervalBTreeRmq(OneIntervalBTreeRmq&&) noexcept = default; - OneIntervalBTreeRmq& operator=(const OneIntervalBTreeRmq&) = default; - OneIntervalBTreeRmq& operator=(OneIntervalBTreeRmq&&) noexcept = default; - - /** - * @brief Build an interval B-tree over a BP-encoded depth-delta sequence. - * - * @details The bit span is not copied and must outlive this object. A - * sequence with @p depth_count depth positions requires - * @p depth_count - 1 delta bits. - * - * @param bits Packed delta bits in little-endian bit order within each word. - * @param depth_count Number of depth positions indexed by the RMQ. - * @throws std::length_error if @p Index cannot represent all positions. - * @throws std::invalid_argument if @p bits does not contain enough words. - */ - OneIntervalBTreeRmq(std::span bits, - std::size_t depth_count) - : input_bits_(bits), depth_count_(depth_count) { - build(); - } - - /** - * @brief Return the number of indexed depth positions. - */ - std::size_t size() const { return depth_count_; } - - /** - * @brief Whether the indexed depth sequence is empty. - */ - bool empty() const { return depth_count_ == 0; } - - /** - * @brief Return the first minimum depth position in [@p left, @p right). - * - * @details The query range is half-open over depth positions, not delta-bit - * positions. Empty or invalid ranges return `npos`. - */ - std::size_t arg_min(std::size_t left, std::size_t right) const { - if (left >= right || right > depth_count_) { - return npos; - } - - const std::size_t last = right - 1; - const std::size_t left_block = left / kBlockSize; - const std::size_t right_block = last / kBlockSize; - if (left_block == right_block) { - return scan_block_range_position(left_block, left % kBlockSize, - last % kBlockSize); - } - - Candidate best; - std::int64_t prefix = 0; - NodeRef best_node; - std::int64_t best_node_target = 0; - bool best_is_node = false; - - const auto consider_position = [&](std::size_t position, - std::int64_t value) { - if (position == npos) { - return; - } - if (value < best.value) { - best = {position, value}; - best_is_node = false; - } - }; - - const auto consider_node = [&](NodeRef node, std::int64_t target, - std::int64_t value) { - if (value < best.value) { - best = {npos, value}; - best_node = node; - best_node_target = target; - best_is_node = true; - } - }; - - const auto consider_excess_result = - [&](std::size_t block, ExcessResult result, std::int64_t value) { - if (result.offset >= block_size(block)) { - return; - } - consider_position(block * kBlockSize + result.offset, value); - }; - - const std::size_t left_offset = left % kBlockSize; - const std::size_t right_offset = last % kBlockSize; - ExcessResult fused_right_boundary; - bool has_fused_right_boundary = false; - - const std::size_t first_full_block = (left + kBlockSize - 1) / kBlockSize; - const std::size_t full_begin = - std::min(right, first_full_block * kBlockSize); - if constexpr (UseBoundaryRecords) { - if (left < full_begin) { - const ScanResult scan = - scan_suffix_boundary_record(left_block, left_offset); - consider_position(scan.position, scan.min_value); - prefix += scan.block_excess; - } - } else { - const bool use_fused_boundaries = left_offset > right_offset && - left_offset < kBlockSize - 1 && - right_offset > 0; - if (use_fused_boundaries) { - const auto left_bits = block_bits(left_block); - const auto right_bits = block_bits(right_block); - const ExcessBoundaryPairResult result = - excess_min_128_disjoint_suffix_prefix( - left_bits.data(), left_offset, right_bits.data(), right_offset); - const std::int64_t left_base = - block_prefix_excess(left_block, left_offset); - consider_excess_result(left_block, result.suffix, - result.suffix.min_excess - left_base); - prefix += block_prefix_excess(left_block, kBlockSize) - left_base; - fused_right_boundary = result.prefix; - has_fused_right_boundary = true; - } else if (left < full_begin) { - const ScanResult scan = scan_range(left, full_begin); - consider_position(scan.position, scan.min_value); - prefix += scan.block_excess; - } - } - - const std::size_t last_full_block_exclusive = right / kBlockSize; - const std::size_t middle_begin = full_begin; - const std::size_t middle_end = - std::max(middle_begin, last_full_block_exclusive * kBlockSize); - if (middle_begin < middle_end) { - Cover cover; - collect_cover(middle_begin, middle_end, cover); - for (std::size_t i = 0; i < cover.size; ++i) { - const NodeRef node = cover.nodes[i]; - const Summary summary = summary_at(node.level, node.index); - consider_node(node, summary.min_excess, prefix + summary.min_excess); - prefix += summary.block_excess; - } - } - - if (middle_end < right) { - if constexpr (UseBoundaryRecords) { - const ExcessResult result = - scan_prefix_boundary_record(right_block, right_offset); - consider_excess_result(right_block, result, prefix + result.min_excess); - } else { - if (has_fused_right_boundary) { - consider_excess_result(right_block, fused_right_boundary, - prefix + fused_right_boundary.min_excess); - } else { - const ScanResult scan = scan_range(middle_end, right); - consider_position(scan.position, prefix + scan.min_value); - } - } - } - - if (best_is_node) { - return descend_first_min(best_node.level, best_node.index, - best_node_target); - } - return best.position; - } - - private: - struct Summary { - std::uint64_t size_positions = 0; - std::int64_t block_excess = 0; - std::int64_t min_excess = 0; - }; - - struct BlockSummary { - Summary summary; - std::uint16_t min_offset = 0; - }; - - struct BoundaryRecords { - std::array prefix_records{}; - std::array suffix_records{}; - }; - - static_assert(sizeof(BoundaryRecords) == 4 * sizeof(std::uint64_t)); - - template - struct alignas(kCacheLineBytes) SummaryNode { - using ExcessType = Excess; - static constexpr std::size_t kFanout = Fanout; - - std::array prefix_excess{}; - std::array min_excess{}; - }; - - using LowNode = SummaryNode; - using HighNode = SummaryNode; - static_assert(alignof(LowNode) == kCacheLineBytes); - static_assert(alignof(HighNode) == kCacheLineBytes); - static_assert(sizeof(LowNode) % kCacheLineBytes == 0); - static_assert(sizeof(HighNode) % kCacheLineBytes == 0); - - struct Candidate { - std::size_t position = npos; - std::int64_t value = std::numeric_limits::max(); - }; - - struct ScanResult { - std::size_t position = npos; - std::int64_t min_value = std::numeric_limits::max(); - std::int64_t block_excess = 0; - }; - - struct NodeRef { - std::size_t level = 0; - std::size_t index = 0; - }; - - struct ChildSearchResult { - bool found = false; - std::size_t index = 0; - std::int64_t target = 0; - }; - - static constexpr std::size_t kMaxCoverItems = 512; - - struct Cover { - std::array nodes{}; - std::size_t size = 0; - - void push(NodeRef node) { - if (size < nodes.size()) { - nodes[size++] = node; - } - } - }; - - /** - * @brief Build lower block summaries and interval B-tree levels. - */ - void build() { - level_counts_.clear(); - low_levels_.clear(); - high_levels_.clear(); - block_summaries_.clear(); - boundary_records_.clear(); - top_summary_ = Summary{}; - block_count_ = 0; - - if (depth_count_ == 0) { - return; - } - if (depth_count_ > static_cast(invalid_index)) { - throw std::length_error("RMQ +/-1 interval B-tree index is too small"); - } - const std::size_t delta_count = depth_count_ - 1; - if (input_bits_.size() < (delta_count + 63) / 64) { - throw std::invalid_argument( - "RMQ +/-1 interval B-tree bit span is too small"); - } - - block_count_ = (depth_count_ + kBlockSize - 1) / kBlockSize; - block_summaries_.resize(block_count_); - if constexpr (UseBoundaryRecords) { - boundary_records_.resize(block_count_); - } - std::vector block_summaries(block_count_); - for (std::size_t block = 0; block < block_count_; ++block) { - block_summaries_[block] = summarize_block(block); - if constexpr (UseBoundaryRecords) { - boundary_records_[block] = build_boundary_records(block); - } - block_summaries[block] = block_summaries_[block].summary; - } - build_levels(block_summaries); - } - - /** - * @brief Build low and high summary levels from 128-position blocks. - */ - void build_levels(const std::vector& block_summaries) { - level_counts_.push_back(block_summaries.size()); - if (block_summaries.empty()) { - return; - } - - std::vector current = block_summaries; - current = build_parent_level(current, low_levels_.emplace_back()); - level_counts_.push_back(current.size()); - current = - build_parent_level(current, high_levels_.emplace_back()); - level_counts_.push_back(current.size()); - while (current.size() > 1) { - current = - build_parent_level(current, high_levels_.emplace_back()); - level_counts_.push_back(current.size()); - } - top_summary_ = current.front(); - } - - template - static std::vector build_parent_level(const std::vector& in, - std::vector& nodes) { - constexpr std::size_t fanout = Node::kFanout; - std::vector out((in.size() + fanout - 1) / fanout); - nodes.resize(out.size()); - for (std::size_t parent = 0; parent < out.size(); ++parent) { - const std::size_t begin = parent * fanout; - const std::size_t end = std::min(in.size(), begin + fanout); - Summary combined; - for (std::size_t i = begin; i < end; ++i) { - store_child_summary(nodes[parent], i - begin, combined.block_excess, - in[i]); - combined = append(combined, in[i]); - } - out[parent] = combined; - } - return out; - } - - template - static void store_child_summary(Node& node, - std::size_t slot, - std::int64_t prefix_excess, - const Summary& summary) { - node.prefix_excess[slot] = - static_cast( - prefix_excess + summary.block_excess); - node.min_excess[slot] = - static_cast( - summary.min_excess); - } - - BlockSummary summarize_block(std::size_t block) const { - const std::size_t size = block_size(block); - BlockSummary block_summary; - Summary& summary = block_summary.summary; - summary.size_positions = size; - if (size == 0) { - return block_summary; - } - const auto bits = block_bits(block); - const ExcessResult scan = excess_min_128(bits.data(), 0, size - 1); - summary.min_excess = scan.min_excess; - block_summary.min_offset = static_cast(scan.offset); - summary.block_excess = - block_prefix_excess(block, next_block_delta_count(block)); - return block_summary; - } - - BoundaryRecords build_boundary_records(std::size_t block) const { - const std::size_t size = block_size(block); - BoundaryRecords records; - if (size == 0) { - return records; - } - - const auto bits = block_bits(block); - std::array local_excess{}; - std::int16_t current = 0; - std::int16_t prefix_best = 0; - set_record_bit(records.prefix_records, 0); - - for (std::size_t offset = 1; offset < size; ++offset) { - current += local_bit(bits, offset - 1) ? 1 : -1; - local_excess[offset] = current; - if (current < prefix_best) { - prefix_best = current; - set_record_bit(records.prefix_records, offset); - } - } - - std::int16_t suffix_best = std::numeric_limits::max(); - for (std::size_t offset = size; offset-- > 0;) { - if (local_excess[offset] <= suffix_best) { - suffix_best = local_excess[offset]; - set_record_bit(records.suffix_records, offset); - } - } - return records; - } - - static Summary append(Summary left, const Summary& right) { - if (left.size_positions == 0) { - return right; - } - if (right.size_positions == 0) { - return left; - } - Summary result; - result.size_positions = left.size_positions + right.size_positions; - result.block_excess = left.block_excess + right.block_excess; - result.min_excess = - std::min(left.min_excess, left.block_excess + right.min_excess); - return result; - } - - std::size_t block_size(std::size_t block) const { - const std::size_t begin = block * kBlockSize; - return begin >= depth_count_ ? 0 - : std::min(kBlockSize, depth_count_ - begin); - } - - std::size_t next_block_delta_count(std::size_t block) const { - const std::size_t begin = block * kBlockSize; - if (begin >= depth_count_ - 1) { - return 0; - } - const std::size_t next_begin = begin + kBlockSize; - return std::min(next_begin, depth_count_ - 1) - begin; - } - - std::uint64_t word_or_zero(std::size_t word) const { - return word < input_bits_.size() ? input_bits_[word] : 0; - } - - std::array block_bits(std::size_t block) const { - const std::size_t first_word = (block * kBlockSize) / 64; - return {word_or_zero(first_word), word_or_zero(first_word + 1)}; - } - - std::array chunk_bits(std::size_t block, - std::size_t chunk) const { - const std::size_t first_word = - (block * kBlockSize + chunk * kChunkSize) / 64; - return {word_or_zero(first_word), word_or_zero(first_word + 1)}; - } - - std::int64_t block_prefix_excess(std::size_t block, - std::size_t prefix_offset) const { - prefix_offset = std::min(prefix_offset, next_block_delta_count(block)); - const auto bits = block_bits(block); - return prefix_excess_128(bits.data(), prefix_offset); - } - - static bool local_bit(const std::array& bits, - std::size_t offset) { - return ((bits[offset >> 6] >> (offset & 63)) & 1u) != 0; - } - - static void set_record_bit(std::array& records, - std::size_t offset) { - records[offset >> 6] |= std::uint64_t{1} << (offset & 63); - } - - ExcessResult scan_prefix_boundary_record(std::size_t block, - std::size_t right_offset) const { - const std::size_t size = block_size(block); - if (size == 0 || block >= boundary_records_.size()) { - return {}; - } - right_offset = std::min(right_offset, size - 1); - const BoundaryRecords& records = boundary_records_[block]; - - std::uint64_t candidates = records.prefix_records[right_offset >> 6] & - first_bits_mask((right_offset & 63) + 1); - if (candidates != 0) { - const std::size_t offset = (right_offset & ~std::size_t{63}) + - (63 - std::countl_zero(candidates)); - const auto bits = block_bits(block); - return {prefix_excess_128(bits.data(), offset), offset}; - } - if ((right_offset >> 6) != 0 && records.prefix_records[0] != 0) { - const std::size_t offset = - 63 - std::countl_zero(records.prefix_records[0]); - const auto bits = block_bits(block); - return {prefix_excess_128(bits.data(), offset), offset}; - } - return {}; - } - - ExcessResult scan_suffix_boundary_record_result( - std::size_t block, - std::size_t left_offset) const { - const std::size_t size = block_size(block); - if (size == 0 || block >= boundary_records_.size()) { - return {}; - } - left_offset = std::min(left_offset, size - 1); - const BoundaryRecords& records = boundary_records_[block]; - - const std::size_t word = left_offset >> 6; - std::uint64_t candidates = - records.suffix_records[word] & ~first_bits_mask(left_offset & 63); - if (candidates != 0) { - const std::size_t offset = - word * 64 + static_cast(std::countr_zero(candidates)); - const auto bits = block_bits(block); - return {prefix_excess_128(bits.data(), offset), offset}; - } - if (word == 0 && records.suffix_records[1] != 0) { - const std::size_t offset = - 64 + - static_cast(std::countr_zero(records.suffix_records[1])); - const auto bits = block_bits(block); - return {prefix_excess_128(bits.data(), offset), offset}; - } - return {}; - } - - ScanResult scan_suffix_boundary_record(std::size_t block, - std::size_t left_offset) const { - const std::size_t size = block_size(block); - if (size == 0) { - return {}; - } - left_offset = std::min(left_offset, size - 1); - const std::int64_t base = block_prefix_excess(block, left_offset); - - ScanResult scan; - const ExcessResult result = - scan_suffix_boundary_record_result(block, left_offset); - if (result.offset == npos || result.offset >= size) { - return scan; - } - - scan.position = block * kBlockSize + result.offset; - scan.min_value = result.min_excess - base; - scan.block_excess = block_prefix_excess(block, kBlockSize) - base; - return scan; - } - - ScanResult scan_range(std::size_t begin, std::size_t end) const { - const std::size_t block = begin / kBlockSize; - const std::size_t block_begin = block * kBlockSize; - return scan_block_range(block, begin - block_begin, end - block_begin - 1); - } - - std::size_t scan_block_range_position(std::size_t block, - std::size_t left_offset, - std::size_t right_offset) const { - const std::size_t size = block_size(block); - right_offset = std::min(right_offset, size - 1); - const auto bits = block_bits(block); - const ExcessResult result = - excess_min_128(bits.data(), left_offset, right_offset); - if (result.offset == npos || result.offset >= size) { - return npos; - } - return block * kBlockSize + result.offset; - } - - ScanResult scan_block_range(std::size_t block, - std::size_t left_offset, - std::size_t right_offset) const { - const std::size_t size = block_size(block); - right_offset = std::min(right_offset, size - 1); - const std::int64_t base = block_prefix_excess(block, left_offset); - - ScanResult scan; - const auto bits = block_bits(block); - const ExcessResult result = - excess_min_128(bits.data(), left_offset, right_offset); - if (result.offset == npos || result.offset >= size) { - return scan; - } - - scan.position = block * kBlockSize + result.offset; - scan.min_value = result.min_excess - base; - scan.block_excess = block_prefix_excess(block, right_offset + 1) - base; - return scan; - } - - std::size_t total_levels() const { return level_counts_.size(); } - - std::size_t level_count(std::size_t level) const { - return level < level_counts_.size() ? level_counts_[level] : 0; - } - - Summary summary_at(std::size_t level, std::size_t index) const { - if (level == 0) { - return block_summaries_[index].summary; - } - if (level + 1 >= total_levels()) { - return top_summary_; - } - const std::size_t parent_level = level + 1; - const std::size_t fanout = fanout_to_parent(level); - const std::size_t parent = index / fanout; - const std::size_t slot = index % fanout; - Summary summary; - summary.size_positions = node_size_positions(level, index); - if (parent_level == 1) { - const LowNode& node = low_levels_[0][parent]; - summary.block_excess = child_excess(node, slot); - summary.min_excess = node.min_excess[slot]; - } else { - const HighNode& node = high_levels_[parent_level - 2][parent]; - summary.block_excess = child_excess(node, slot); - summary.min_excess = node.min_excess[slot]; - } - return summary; - } - - static std::size_t fanout_to_parent(std::size_t level) { - return level == 0 ? kLowFanout : kHighFanout; - } - - static std::size_t mul_clamped(std::size_t lhs, std::size_t rhs) { - if (lhs != 0 && rhs > std::numeric_limits::max() / lhs) { - return std::numeric_limits::max(); - } - return lhs * rhs; - } - - static std::size_t level_span_positions(std::size_t level) { - std::size_t span = kBlockSize; - if (level >= 1) { - span = mul_clamped(span, kLowFanout); - } - if (level >= 2) { - for (std::size_t i = 2; i <= level; ++i) { - span = mul_clamped(span, kHighFanout); - } - } - return span; - } - - std::size_t node_start_position(std::size_t level, std::size_t index) const { - const std::size_t span = level_span_positions(level); - if (span != 0 && index > std::numeric_limits::max() / span) { - return depth_count_; - } - return std::min(depth_count_, index * span); - } - - std::size_t node_size_positions(std::size_t level, std::size_t index) const { - const std::size_t start = node_start_position(level, index); - if (start >= depth_count_) { - return 0; - } - return std::min(level_span_positions(level), depth_count_ - start); - } - - template - static std::int64_t prefix_excess_at(const Node& node, std::size_t slot) { - if (slot == 0) { - return 0; - } - return static_cast(node.prefix_excess[slot - 1]); - } - - template - static std::int64_t child_excess(const Node& node, std::size_t slot) { - return static_cast(node.prefix_excess[slot]) - - prefix_excess_at(node, slot); - } - - template - ChildSearchResult find_first_child_with_min(const Node& node, - std::size_t child_level, - std::size_t parent, - std::size_t child_count, - std::int64_t target) const { - for (std::size_t slot = 0; slot < child_count; ++slot) { - const std::int64_t candidate = - prefix_excess_at(node, slot) + - static_cast(node.min_excess[slot]); - if (candidate == target) { - return {true, parent * fanout_to_parent(child_level) + slot, - static_cast(node.min_excess[slot])}; - } - } - return {}; - } - - ChildSearchResult find_first_child_with_min(std::size_t child_level, - std::size_t parent, - std::size_t child_count, - std::int64_t target) const { - if (child_level == 0) { - return find_first_child_with_min(low_levels_[0][parent], child_level, - parent, child_count, target); - } - return find_first_child_with_min(high_levels_[child_level - 1][parent], - child_level, parent, child_count, target); - } - - void collect_cover(std::size_t begin, std::size_t end, Cover& out) const { - if (begin >= end || total_levels() == 0 || (begin % kBlockSize) != 0 || - (end % kBlockSize) != 0) { - return; - } - - Cover right_cover; - std::size_t level = 0; - std::size_t left = begin / kBlockSize; - std::size_t right = end / kBlockSize; - - while (left < right) { - if (!has_parent_level(level)) { - for (std::size_t index = left; index < right; ++index) { - out.push({level, index}); - } - break; - } - - const std::size_t fanout = fanout_to_parent(level); - while (left < right && (left % fanout) != 0) { - out.push({level, left}); - ++left; - } - while (left < right && (right % fanout) != 0) { - --right; - right_cover.push({level, right}); - } - left /= fanout; - right /= fanout; - ++level; - } - - while (right_cover.size > 0) { - out.push(right_cover.nodes[--right_cover.size]); - } - } - - bool has_parent_level(std::size_t level) const { - return level + 1 < total_levels() && level_count(level + 1) != 0; - } - - std::size_t descend_first_min(std::size_t level, - std::size_t index, - std::int64_t target) const { - while (level > 0) { - const std::size_t child_level = level - 1; - const std::size_t fanout = fanout_to_parent(child_level); - const std::size_t child_begin = index * fanout; - const std::size_t child_end = - std::min(level_count(child_level), child_begin + fanout); - const ChildSearchResult child = find_first_child_with_min( - child_level, index, child_end - child_begin, target); - if (!child.found) { - return npos; - } - index = child.index; - level = child_level; - target = child.target; - } - - if (index >= block_summaries_.size()) { - return npos; - } - const BlockSummary& block = block_summaries_[index]; - return block.summary.min_excess == target - ? index * kBlockSize + block.min_offset - : npos; - } - - std::span input_bits_; - std::size_t depth_count_ = 0; - std::size_t block_count_ = 0; - Summary top_summary_; - std::vector block_summaries_; - std::vector boundary_records_; - std::vector level_counts_; - std::vector> low_levels_; - std::vector> high_levels_; -}; - -template -using OneIntervalBTreeRmqBoundaryRecords = - OneIntervalBTreeRmq; - -} // namespace pixie::rmq diff --git a/include/pixie/rmq/sdsl_sct_rmq.h b/include/pixie/rmq/sdsl_sct.h similarity index 92% rename from include/pixie/rmq/sdsl_sct_rmq.h rename to include/pixie/rmq/sdsl_sct.h index 51466f6..d9043d1 100644 --- a/include/pixie/rmq/sdsl_sct_rmq.h +++ b/include/pixie/rmq/sdsl_sct.h @@ -1,7 +1,7 @@ #pragma once #ifndef SDSL_SUPPORT -#error "pixie/rmq/sdsl_sct_rmq.h requires SDSL_SUPPORT" +#error "pixie/rmq/sdsl_sct.h requires SDSL_SUPPORT" #endif #include @@ -46,18 +46,18 @@ namespace pixie::rmq { * @tparam Index Interface compatibility parameter for benchmark templates. */ template , class Index = std::size_t> -class SdslSctRmq : public RmqBase, T> { +class SdslSct : public RmqBase, T> { public: static_assert(std::is_same_v>, "SDSL SCT RMQ wrapper supports only std::less"); static constexpr std::size_t npos = - RmqBase, T>::npos; + RmqBase, T>::npos; /** * @brief Construct an empty SDSL SCT RMQ adapter. */ - SdslSctRmq() = default; + SdslSct() = default; /** * @brief Build the SDSL SCT index over @p values. @@ -68,7 +68,7 @@ class SdslSctRmq : public RmqBase, T> { * @param values Values to index. * @param compare Must be the default `std::less` comparator. */ - explicit SdslSctRmq(std::span values, Compare compare = Compare()) + explicit SdslSct(std::span values, Compare compare = Compare()) : values_(values), rmq_(&values_) { (void)compare; } diff --git a/include/pixie/rmq/utils/succinct_monotone_stack.h b/include/pixie/rmq/utils/succinct_monotone_stack.h new file mode 100644 index 0000000..acd8552 --- /dev/null +++ b/include/pixie/rmq/utils/succinct_monotone_stack.h @@ -0,0 +1,213 @@ +#pragma once + +/** + * Monotone-stack construction experiment, 2026-06-13. + * + * SIMD investigation: for the RMQ benchmark's uniform-random value arrays, the + * Cartesian construction pop-run distribution is short: roughly 50% of + * iterations pop 0 entries, 25% pop 1, 12.5% pop 2, and only about 6.25% pop + * 4 or more. A SIMD path that gathers several stack-top values would therefore + * pay gather/setup overhead on mostly short runs. The retained improvement is + * bit-parallel rather than SIMD: 64-bit key blocks plus compact non-empty block + * summaries replace the old 63-bit data/pointer-word layout. + * + * Command shape: + * taskset -c 0 ./build/release/bench_rmq + * --benchmark_filter='^rmq_build_(cartesian_rmm|cartesian_hybrid_btree|sdsl_sct)/(4194304|67108864)$' + * --benchmark_repetitions=5 + * + * CPU mean, milliseconds. + * + * | N | row | before | after | + * | ---: | ------------------------ | -----: | ------: | + * | 2^22 | CartesianRmM | 44.731 | 40.274 | + * | 2^22 | CartesianHybridBTree | 47.943 | 45.019 | + * | 2^22 | SdslSct control | 39.911 | 39.628 | + * | ---- | ------------------------ | ------ | ------- | + * | 2^26 | CartesianRmM | 750.316 | 678.925 | + * | 2^26 | CartesianHybridBTree | 772.157 | 732.582 | + * | 2^26 | SdslSct control | 642.540 | 644.627 | + */ + +#include +#include +#include +#include +#include + +namespace pixie::rmq::utils { + +/** + * @brief Construction-only stack for monotonically increasing integer keys. + * + * @details The stack stores the currently live keys in 64-bit blocks and keeps + * compact summary bitsets for non-empty blocks. It is intended for + * Cartesian-tree construction passes where pushed keys are strictly increasing. + * Under that precondition the logical stack top is the largest stored key. + * + * The helper keeps construction memory succinct in the worst case: an input of + * `capacity` keys uses one bit per key plus about 1/64 extra summary bits, + * instead of one full index per live stack entry. It does not own or inspect + * input values and deliberately does not define the Cartesian-tree tie rule; + * callers keep their existing pop predicate. + */ +class SuccinctIncreasingStack { + public: + explicit SuccinctIncreasingStack(std::size_t capacity) + : words_(word_count(capacity)), + nonempty_words_(word_count(words_.size())), + nonempty_super_words_(word_count(nonempty_words_.size())), + capacity_(capacity) {} + + /** + * @brief Whether the logical stack is empty. + */ + bool empty() const { return size_ == 0; } + + /** + * @brief Return the current top key. + * + * @pre The stack is not empty. + */ + std::size_t top() const { + assert(!empty()); + return top_key_; + } + + /** + * @brief Push a key larger than the current top key. + * + * @details Keys are expected to be pushed in strictly increasing order over + * the lifetime of this stack. The assertion checks the live top, which is the + * only ordering constraint needed by this bitset representation. + */ + void push(std::size_t key) { + assert(key < capacity_); + assert(empty() || top() < key); + + const std::size_t block = block_index(key); + words_[block] |= std::uint64_t{1} << block_position(key); + set_nonempty(block); + top_key_ = key; + ++size_; + } + + /** + * @brief Remove the current top key. + * + * @pre The stack is not empty. + */ + void pop() { + assert(!empty()); + + const std::size_t block = block_index(top_key_); + std::uint64_t word = words_[block]; + word &= ~(std::uint64_t{1} << block_position(top_key_)); + words_[block] = word; + + --size_; + if (size_ == 0) { + if (word == 0) { + clear_nonempty(block); + } + top_key_ = 0; + return; + } + + if (word != 0) { + top_key_ = block * kBlockBits + highest_bit(word); + return; + } + + clear_nonempty(block); + const std::size_t previous_block = previous_nonempty_block(block); + const std::uint64_t previous_word = words_[previous_block]; + assert(previous_word != 0); + top_key_ = previous_block * kBlockBits + highest_bit(previous_word); + } + + private: + static constexpr std::size_t kBlockBits = 64; + static constexpr std::size_t kBlockShift = 6; + static constexpr std::size_t kBlockMask = kBlockBits - 1; + + static std::size_t word_count(std::size_t bit_count) { + return (bit_count + kBlockBits - 1) / kBlockBits; + } + + static std::size_t block_index(std::size_t key) { return key >> kBlockShift; } + + static std::size_t block_position(std::size_t key) { + return key & kBlockMask; + } + + static std::size_t highest_bit(std::uint64_t word) { + assert(word != 0); + return static_cast(std::bit_width(word) - 1); + } + + static std::uint64_t low_bits(std::size_t count) { + return count == 0 ? 0 : ((std::uint64_t{1} << count) - 1); + } + + void set_nonempty(std::size_t block) { + const std::size_t summary = block_index(block); + const std::uint64_t bit = std::uint64_t{1} << block_position(block); + if ((nonempty_words_[summary] & bit) == 0) { + nonempty_words_[summary] |= bit; + nonempty_super_words_[block_index(summary)] |= std::uint64_t{1} + << block_position(summary); + } + } + + void clear_nonempty(std::size_t block) { + const std::size_t summary = block_index(block); + nonempty_words_[summary] &= ~(std::uint64_t{1} << block_position(block)); + if (nonempty_words_[summary] == 0) { + nonempty_super_words_[block_index(summary)] &= + ~(std::uint64_t{1} << block_position(summary)); + } + } + + std::size_t previous_nonempty_block(std::size_t block) const { + assert(block > 0); + std::size_t summary = block_index(block); + const std::size_t summary_position = block_position(block); + if (summary_position != 0) { + const std::uint64_t same_summary = + nonempty_words_[summary] & low_bits(summary_position); + if (same_summary != 0) { + return summary * kBlockBits + highest_bit(same_summary); + } + } + + std::size_t super = block_index(summary); + const std::size_t super_position = block_position(summary); + std::uint64_t super_mask = + super_position == 0 + ? 0 + : nonempty_super_words_[super] & low_bits(super_position); + while (super_mask == 0) { + assert(super > 0); + --super; + super_mask = nonempty_super_words_[super]; + } + + const std::size_t previous_summary = + super * kBlockBits + highest_bit(super_mask); + assert(previous_summary < nonempty_words_.size()); + const std::uint64_t previous_summary_word = + nonempty_words_[previous_summary]; + assert(previous_summary_word != 0); + return previous_summary * kBlockBits + highest_bit(previous_summary_word); + } + + std::vector words_; + std::vector nonempty_words_; + std::vector nonempty_super_words_; + std::size_t capacity_ = 0; + std::size_t size_ = 0; + std::size_t top_key_ = 0; +}; + +} // namespace pixie::rmq::utils diff --git a/src/benchmarks/bench_rmq.cpp b/src/benchmarks/bench_rmq.cpp index 3e5fea0..3b60633 100644 --- a/src/benchmarks/bench_rmq.cpp +++ b/src/benchmarks/bench_rmq.cpp @@ -1,33 +1,22 @@ #include -#include #include -#include -#include -#include -#include +#include +#include #ifdef PIXIE_THIRD_PARTY_BENCHMARKS -#include -#endif - -#ifdef __linux__ -#include -#include -#include -#include +#include #endif #include -#include -#include #include #include -#include #include #include +#include #include #include #include +#include #include #include @@ -37,19 +26,10 @@ constexpr std::uint64_t kSeed = 42; constexpr std::size_t kQueryPoolBytes = 512 * 1024; constexpr std::size_t kQueryCount = kQueryPoolBytes / sizeof(std::pair); -constexpr std::size_t kBpBlockSize = 128; -constexpr std::size_t kCacheLineBytes = 64; -constexpr std::size_t kSparseFootprintCycles = 16; -// Value-RMQ rows rotate between several arrays so query/build timings do not -// depend on one accidental global-minimum position. Large rows use fewer -// variants to keep multi-index query benchmarks within practical memory limits. -constexpr std::size_t kDefaultValueDatasetVariants = 4; -constexpr std::size_t kLargeValueDatasetVariants = 2; -constexpr std::size_t kValueDatasetVariantLargeSize = 1ull << 18; -constexpr std::size_t kMaxValueDatasetVariants = 16; constexpr std::int64_t kValueDatasetGlobalMinimum = std::numeric_limits::min() / 4; -constexpr const char* kValueDatasetVariantsEnv = "PIXIE_BENCH_VALUE_VARIANTS"; +constexpr double kBenchmarkWarmupSeconds = 0.2; +constexpr double kBenchmarkMinSeconds = 2.0; using Index = std::size_t; static_assert(kQueryPoolBytes % sizeof(std::pair) == @@ -60,101 +40,93 @@ concept HasRmqMemoryUsage = requires(const Rmq& rmq) { { rmq.memory_usage_bytes() } -> std::convertible_to; }; -struct Dataset { - std::size_t size = 0; - std::size_t max_width = 0; - std::vector values; -}; - -struct DepthDataset { - std::size_t size = 0; - std::size_t max_width = 0; - std::vector depths; - std::vector bits; - std::vector> ranges; -}; - -struct DeltaBitDataset { - std::size_t size = 0; - std::vector bits; -}; - -struct BpQueryShape { - std::size_t same_block = 0; - std::size_t cross_block = 0; - std::size_t middle_block = 0; - std::size_t disjoint_boundary = 0; - std::size_t fused_boundary = 0; - std::size_t excess_calls = 0; - std::size_t left_boundary_width = 0; - std::size_t right_boundary_width = 0; - std::size_t same_block_width = 0; -}; - -struct BlockRange { - std::size_t block = 0; - std::size_t left = 0; - std::size_t right = 0; -}; - -struct MacroRange { - std::size_t left = 0; - std::size_t right = 0; -}; - -struct alignas(16) BenchBlockSummary { - static constexpr std::uint64_t kSigned48Mask = (std::uint64_t{1} << 48) - 1; - static constexpr std::uint64_t kSigned48SignBit = std::uint64_t{1} << 47; - - std::uint64_t word0 = 0; - std::uint64_t word1 = 0; +std::uint64_t splitmix64(std::uint64_t value) { + value += 0x9E3779B97F4A7C15ull; + value = (value ^ (value >> 30)) * 0xBF58476D1CE4E5B9ull; + value = (value ^ (value >> 27)) * 0x94D049BB133111EBull; + return value ^ (value >> 31); +} - static BenchBlockSummary make(std::int64_t base_depth, - std::int64_t min_value, - std::size_t min_offset) { - const std::uint64_t packed_base = pack_signed48(base_depth); - const std::uint64_t ordered_min = pack_ordered48(min_value); - return {ordered_min | (static_cast(min_offset) << 48), - packed_base}; - } +std::uint64_t mix_seed(std::uint64_t seed, std::uint64_t value) { + return splitmix64(seed ^ splitmix64(value)); +} - std::int64_t min_value() const { - return unpack_ordered48(ordered_min_value()); +class BenchmarkRepetitionSeeds { + public: + std::uint64_t repetition_index(const benchmark::State& state, + std::size_t size, + std::size_t max_width) { + std::lock_guard lock(mutex_); + Entry& entry = entries_[key_for(state, size, max_width)]; + const benchmark::IterationCount max_iterations = state.max_iterations; + if (!entry.seen) { + entry.seen = true; + entry.last_max_iterations = max_iterations; + return entry.repetition_index; + } + + if (max_iterations == entry.last_max_iterations) { + if (!entry.saw_iteration_count_change && + !entry.skipped_initial_warmup_equal) { + entry.skipped_initial_warmup_equal = true; + } else { + ++entry.repetition_index; + } + } else { + entry.saw_iteration_count_change = true; + } + entry.last_max_iterations = max_iterations; + return entry.repetition_index; } - std::uint64_t ordered_min_value() const { return word0 & kSigned48Mask; } - private: - static std::uint64_t pack_signed48(std::int64_t value) { - return static_cast(value) & kSigned48Mask; - } - - static std::uint64_t pack_ordered48(std::int64_t value) { - return pack_signed48(value) ^ kSigned48SignBit; - } + struct Entry { + benchmark::IterationCount last_max_iterations = 0; + std::uint64_t repetition_index = 0; + bool seen = false; + bool saw_iteration_count_change = false; + bool skipped_initial_warmup_equal = kBenchmarkWarmupSeconds == 0.0; + }; - static std::int64_t unpack_signed48(std::uint64_t value) { - if ((value & kSigned48SignBit) != 0) { - value |= ~kSigned48Mask; - } - return static_cast(value); + std::string key_for(const benchmark::State& state, + std::size_t size, + std::size_t max_width) const { + return state.name() + "/" + std::to_string(size) + "/" + + std::to_string(max_width); } - static std::int64_t unpack_ordered48(std::uint64_t value) { - return unpack_signed48(value ^ kSigned48SignBit); - } + std::mutex mutex_; + std::unordered_map entries_; }; -static_assert(sizeof(BenchBlockSummary) == 16); -static_assert(alignof(BenchBlockSummary) == 16); +BenchmarkRepetitionSeeds& repetition_seeds() { + static BenchmarkRepetitionSeeds seeds; + return seeds; +} -struct BenchBlockSummaryMinLess { - bool operator()(const BenchBlockSummary& left, - const BenchBlockSummary& right) const { - return left.ordered_min_value() < right.ordered_min_value(); - } +struct SeedContext { + std::uint64_t repetition_index = 0; + std::uint64_t value_seed = 0; + std::uint64_t query_seed = 0; }; +SeedContext make_seed_context(const benchmark::State& state, + std::size_t size, + std::size_t max_width = 0) { + const std::uint64_t repetition_index = + repetition_seeds().repetition_index(state, size, max_width); + std::uint64_t seed = mix_seed(kSeed, static_cast(size)); + seed = mix_seed(seed, repetition_index); + + SeedContext context; + context.repetition_index = repetition_index; + context.value_seed = mix_seed(seed, 0x4F1BBCDCBFA54005ull); + context.query_seed = + mix_seed(mix_seed(seed, static_cast(max_width)), + 0x9D2C5680D3F6C58Bull); + return context; +} + class RandomQueryGenerator { public: RandomQueryGenerator(std::size_t size, @@ -212,615 +184,34 @@ class RandomQueryGenerator { std::size_t shift_ = 0; }; -class PerfCounterGroup { +/** + * @brief Single generated value dataset for value-RMQ benchmarks. + */ +class ValueDataset { public: - struct Values { - std::uint64_t cycles = 0; - std::uint64_t instructions = 0; - std::uint64_t cache_misses = 0; - }; - - PerfCounterGroup() { -#ifdef __linux__ - if (!enabled()) { - return; - } - - leader_fd_ = open_hardware_event(PERF_COUNT_HW_CPU_CYCLES, -1, true); - if (leader_fd_ < 0) { - return; - } - instructions_fd_ = - open_hardware_event(PERF_COUNT_HW_INSTRUCTIONS, leader_fd_, false); - cache_misses_fd_ = - open_hardware_event(PERF_COUNT_HW_CACHE_MISSES, leader_fd_, false); - if (instructions_fd_ < 0 || cache_misses_fd_ < 0) { - close_all(); - return; - } - - ioctl(leader_fd_, PERF_EVENT_IOC_RESET, PERF_IOC_FLAG_GROUP); - ioctl(leader_fd_, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP); - active_ = true; -#endif - } - - PerfCounterGroup(const PerfCounterGroup&) = delete; - PerfCounterGroup& operator=(const PerfCounterGroup&) = delete; - - ~PerfCounterGroup() { close_all(); } + explicit ValueDataset(std::size_t size, std::uint64_t seed) : values_(size) { + size_ = size; - bool stop(Values* values) { -#ifdef __linux__ - if (!active_) { - return false; - } - ioctl(leader_fd_, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP); - active_ = false; - - struct ReadGroup { - std::uint64_t count = 0; - std::uint64_t values[3] = {}; - }; - ReadGroup group; - const ssize_t bytes = read(leader_fd_, &group, sizeof(group)); - if (bytes != static_cast(sizeof(group)) || group.count != 3) { - close_all(); - return false; - } - - values->cycles = group.values[0]; - values->instructions = group.values[1]; - values->cache_misses = group.values[2]; - close_all(); - return true; -#else - (void)values; - return false; -#endif - } - - private: - static bool enabled() { - const char* value = std::getenv("PIXIE_BENCH_PERF_COUNTERS"); - return value != nullptr && value[0] != '\0' && - !(value[0] == '0' && value[1] == '\0'); - } - -#ifdef __linux__ - static int open_hardware_event(std::uint64_t config, - int group_fd, - bool disabled) { - perf_event_attr attr = {}; - attr.type = PERF_TYPE_HARDWARE; - attr.size = sizeof(attr); - attr.config = config; - attr.disabled = disabled ? 1 : 0; - attr.exclude_kernel = 1; - attr.exclude_hv = 1; - attr.read_format = PERF_FORMAT_GROUP; - return static_cast( - syscall(__NR_perf_event_open, &attr, 0, -1, group_fd, 0)); - } - - void close_fd(int* fd) { - if (*fd >= 0) { - close(*fd); - *fd = -1; - } - } - - void close_all() { - if (active_ && leader_fd_ >= 0) { - ioctl(leader_fd_, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP); - active_ = false; - } - close_fd(&cache_misses_fd_); - close_fd(&instructions_fd_); - close_fd(&leader_fd_); - } - - int leader_fd_ = -1; - int instructions_fd_ = -1; - int cache_misses_fd_ = -1; - bool active_ = false; -#else - void close_all() {} -#endif -}; - -void set_perf_counters(benchmark::State& state, - const PerfCounterGroup::Values& values) { - const double iterations = static_cast(state.iterations()); - if (iterations == 0.0) { - return; - } - state.counters["cycles_per_query"] = - static_cast(values.cycles) / iterations; - state.counters["instructions_per_query"] = - static_cast(values.instructions) / iterations; - state.counters["ipc"] = values.cycles == 0 - ? 0.0 - : static_cast(values.instructions) / - static_cast(values.cycles); - state.counters["cache_misses_per_query"] = - static_cast(values.cache_misses) / iterations; -} - -Dataset make_dataset(std::size_t size, std::size_t max_width) { - Dataset dataset; - dataset.size = size; - dataset.max_width = max_width; - dataset.values.resize(size); - - std::mt19937_64 rng(kSeed ^ (size * 0x9E3779B185EBCA87ull) ^ - (max_width * 0xBF58476D1CE4E5B9ull)); - std::uniform_int_distribution value_dist(-1'000'000, 1'000'000); - std::generate(dataset.values.begin(), dataset.values.end(), - [&] { return value_dist(rng); }); - return dataset; -} - -std::size_t value_dataset_variant_count(std::size_t size) { - if (const char* value = std::getenv(kValueDatasetVariantsEnv); - value != nullptr && value[0] != '\0') { - char* end = nullptr; - const unsigned long long parsed = std::strtoull(value, &end, 10); - if (end != value && *end == '\0' && parsed != 0) { - return std::min(static_cast(parsed), - kMaxValueDatasetVariants); - } - } - return size > kValueDatasetVariantLargeSize ? kLargeValueDatasetVariants - : kDefaultValueDatasetVariants; -} - -std::size_t value_dataset_global_min_position(std::size_t size, - std::size_t variant, - std::size_t variant_count) { - return ((2 * variant + 1) * size) / (2 * variant_count); -} - -std::vector make_value_dataset_variants(std::size_t size, - std::size_t max_width, - std::size_t variant_count) { - variant_count = std::max( - 1, std::min(variant_count, std::max(size, 1))); - - std::vector base_values(size); - std::mt19937_64 rng(kSeed ^ (size * 0x9E3779B185EBCA87ull)); - std::uniform_int_distribution value_dist(-1'000'000, 1'000'000); - std::generate(base_values.begin(), base_values.end(), - [&] { return value_dist(rng); }); - - std::vector variants; - variants.reserve(variant_count); - for (std::size_t variant = 0; variant < variant_count; ++variant) { - Dataset dataset; - dataset.size = size; - dataset.max_width = max_width; - dataset.values = base_values; - if (!dataset.values.empty()) { - dataset.values[value_dataset_global_min_position( - size, variant, variant_count)] = kValueDatasetGlobalMinimum; - } - variants.push_back(std::move(dataset)); - } - return variants; -} - -DeltaBitDataset make_delta_bit_dataset(std::size_t size) { - DeltaBitDataset dataset; - dataset.size = size; - if (size == 0) { - return dataset; - } - dataset.bits.assign((size - 1 + 63) / 64, 0); + std::mt19937_64 rng(seed); + std::uniform_int_distribution value_dist(-1'000'000, + 1'000'000); + std::generate(values_.begin(), values_.end(), + [&] { return value_dist(rng); }); - std::mt19937_64 rng(kSeed ^ (size * 0xD6E8FEB86659FD93ull)); - for (std::size_t i = 0; i + 1 < size; ++i) { - if ((rng() & 1u) != 0) { - dataset.bits[i >> 6] |= std::uint64_t{1} << (i & 63); + if (!values_.empty()) { + values_[values_.size() / 2] = kValueDatasetGlobalMinimum; } } - return dataset; -} - -struct SparseTableFootprintStats { - std::size_t query_count = 0; - std::size_t max_observed_width = 0; - std::size_t over_max_width = 0; - std::uint64_t total_width = 0; - std::uint64_t total_level = 0; - std::uint64_t same_table_line = 0; - std::uint64_t same_value_line = 0; - std::array level_counts = {}; - std::vector table_lines; - std::vector value_lines; - - void finalize() { - sort_unique(table_lines); - sort_unique(value_lines); - } - double avg_width() const { - return query_count == 0 ? 0.0 - : static_cast(total_width) / query_count; - } - - double avg_level() const { - return query_count == 0 ? 0.0 - : static_cast(total_level) / query_count; - } + std::size_t size() const { return size_; } - double ratio(std::uint64_t count) const { - return query_count == 0 ? 0.0 : static_cast(count) / query_count; - } - - std::size_t percentile_level(std::uint64_t numerator, - std::uint64_t denominator) const { - if (query_count == 0) { - return 0; - } - const std::uint64_t target = - (static_cast(query_count) * numerator + denominator - - 1) / - denominator; - std::uint64_t cumulative = 0; - for (std::size_t level = 0; level < level_counts.size(); ++level) { - cumulative += level_counts[level]; - if (cumulative >= target) { - return level; - } - } - return level_counts.size() - 1; - } + std::span values() const { return values_; } private: - static void sort_unique(std::vector& values) { - std::sort(values.begin(), values.end()); - values.erase(std::unique(values.begin(), values.end()), values.end()); - } + std::size_t size_ = 0; + std::vector values_; }; -SparseTableFootprintStats compute_sparse_table_footprint( - const pixie::rmq::SparseTable, Index>& - rmq, - std::size_t size, - std::size_t max_width) { - constexpr std::size_t kIndexEntriesPerLine = kCacheLineBytes / sizeof(Index); - constexpr std::size_t kValueEntriesPerLine = - kCacheLineBytes / sizeof(std::int64_t); - - SparseTableFootprintStats stats; - stats.query_count = kSparseFootprintCycles * kQueryCount; - stats.table_lines.reserve(stats.query_count * 2); - stats.value_lines.reserve(stats.query_count * 2); - - RandomQueryGenerator queries(size, max_width, - kSeed ^ (size * 0xC2B2AE3D27D4EB4Full) ^ - (max_width * 0x165667B19E3779F9ull)); - - for (std::size_t i = 0; i < stats.query_count; ++i) { - const auto [left, right] = queries.get_random_query(); - const std::size_t width = right - left; - const std::size_t level = std::bit_width(width) - 1; - const std::size_t span = std::size_t{1} << level; - const std::size_t first_table_position = left; - const std::size_t second_table_position = right - span; - const std::uint64_t first_table_line = - (static_cast(level) << 56) | - (first_table_position / kIndexEntriesPerLine); - const std::uint64_t second_table_line = - (static_cast(level) << 56) | - (second_table_position / kIndexEntriesPerLine); - - const std::size_t first_value_position = - rmq.arg_min(first_table_position, first_table_position + span); - const std::size_t second_value_position = - rmq.arg_min(second_table_position, second_table_position + span); - const std::uint64_t first_value_line = - first_value_position / kValueEntriesPerLine; - const std::uint64_t second_value_line = - second_value_position / kValueEntriesPerLine; - - stats.total_width += width; - stats.max_observed_width = std::max(stats.max_observed_width, width); - stats.over_max_width += width > max_width ? 1 : 0; - stats.total_level += level; - ++stats.level_counts[level]; - stats.same_table_line += first_table_line == second_table_line ? 1 : 0; - stats.same_value_line += first_value_line == second_value_line ? 1 : 0; - stats.table_lines.push_back(first_table_line); - stats.table_lines.push_back(second_table_line); - stats.value_lines.push_back(first_value_line); - stats.value_lines.push_back(second_value_line); - } - - stats.finalize(); - return stats; -} - -void set_sparse_table_footprint_counters( - benchmark::State& state, - const SparseTableFootprintStats& stats) { - const double query_count = static_cast(stats.query_count); - state.counters["sample_queries"] = query_count; - state.counters["sample_cycles"] = static_cast(kSparseFootprintCycles); - state.counters["avg_width"] = stats.avg_width(); - state.counters["max_observed_width"] = - static_cast(stats.max_observed_width); - state.counters["over_max_width_ratio"] = stats.ratio(stats.over_max_width); - state.counters["avg_level"] = stats.avg_level(); - state.counters["level_p50"] = - static_cast(stats.percentile_level(50, 100)); - state.counters["level_p90"] = - static_cast(stats.percentile_level(90, 100)); - state.counters["level_p99"] = - static_cast(stats.percentile_level(99, 100)); - state.counters["table_distinct_lines_per_query"] = - 2.0 - stats.ratio(stats.same_table_line); - state.counters["value_distinct_lines_per_query"] = - 2.0 - stats.ratio(stats.same_value_line); - state.counters["table_working_set_lines_per_query"] = - static_cast(stats.table_lines.size()) / query_count; - state.counters["value_working_set_lines_per_query"] = - static_cast(stats.value_lines.size()) / query_count; - state.counters["total_working_set_lines_per_query"] = - static_cast(stats.table_lines.size() + stats.value_lines.size()) / - query_count; - - for (std::size_t level = 0; level < stats.level_counts.size(); ++level) { - if (stats.level_counts[level] == 0) { - continue; - } - state.counters["level_" + std::to_string(level) + "_ratio"] = - stats.ratio(stats.level_counts[level]); - } -} - -DepthDataset make_depth_dataset(std::size_t size, std::size_t max_width) { - DepthDataset dataset; - dataset.size = size; - dataset.max_width = max_width; - dataset.depths.resize(size); - dataset.ranges.resize(kQueryCount); - - std::mt19937_64 rng(kSeed ^ (size * 0xD6E8FEB86659FD93ull) ^ - (max_width * 0xA5A3564E27F88695ull)); - for (std::size_t i = 1; i < dataset.depths.size(); ++i) { - dataset.depths[i] = dataset.depths[i - 1] + ((rng() & 1u) ? 1 : -1); - } - dataset.bits.assign((size - 1 + 63) / 64, 0); - for (std::size_t i = 1; i < dataset.depths.size(); ++i) { - if (dataset.depths[i] - dataset.depths[i - 1] == 1) { - dataset.bits[(i - 1) >> 6] |= std::uint64_t{1} << ((i - 1) & 63); - } - } - - RandomQueryGenerator queries(size, max_width, - kSeed ^ (size * 0xA24BAED4963EE407ull) ^ - (max_width * 0x9FB21C651E98DF25ull)); - for (auto& range : dataset.ranges) { - range = queries.get_random_query(); - } - return dataset; -} - -std::size_t bp_block_size(const DepthDataset& dataset, std::size_t block) { - const std::size_t begin = block * kBpBlockSize; - return std::min(kBpBlockSize, dataset.depths.size() - begin); -} - -std::array bp_block_bits_stack(const DepthDataset& dataset, - std::size_t block) { - const std::size_t first_word = block * (kBpBlockSize / 64); - const std::uint64_t lo = - first_word < dataset.bits.size() ? dataset.bits[first_word] : 0; - const std::uint64_t hi = - first_word + 1 < dataset.bits.size() ? dataset.bits[first_word + 1] : 0; - return {lo, hi}; -} - -const std::uint64_t* bp_block_bits_direct(const DepthDataset& dataset, - std::size_t block) { - return dataset.bits.data() + block * (kBpBlockSize / 64); -} - -BpQueryShape compute_bp_query_shape(const DepthDataset& dataset, - std::size_t block_size) { - BpQueryShape shape; - for (const auto [left, right] : dataset.ranges) { - const std::size_t last = right - 1; - const std::size_t left_block = left / block_size; - const std::size_t right_block = last / block_size; - if (left_block == right_block) { - ++shape.same_block; - ++shape.excess_calls; - shape.same_block_width += right - left; - continue; - } - - ++shape.cross_block; - const std::size_t left_offset = left % block_size; - const std::size_t right_offset = last % block_size; - const std::size_t left_width = block_size - left_offset; - const std::size_t right_width = right_offset + 1; - if (left_offset > right_offset) { - ++shape.disjoint_boundary; - if (left_offset < block_size - 1 && right_offset > 0) { - ++shape.fused_boundary; - ++shape.excess_calls; - } else { - shape.excess_calls += 2; - } - } else { - shape.excess_calls += 2; - } - shape.left_boundary_width += left_width; - shape.right_boundary_width += right_width; - if (left_block + 1 < right_block) { - ++shape.middle_block; - } - } - return shape; -} - -void set_depth_counters(benchmark::State& state, - const DepthDataset& dataset, - bool include_shape, - std::size_t block_size = kBpBlockSize) { - state.counters["N"] = static_cast(dataset.size); - state.counters["max_width"] = static_cast(dataset.max_width); - state.counters["index_bytes"] = static_cast(sizeof(Index)); - - if (!include_shape) { - return; - } - - const BpQueryShape shape = compute_bp_query_shape(dataset, block_size); - const double query_count = static_cast(dataset.ranges.size()); - const double cross_count = static_cast(shape.cross_block); - const double same_count = static_cast(shape.same_block); - state.counters["same_block_ratio"] = - static_cast(shape.same_block) / query_count; - state.counters["cross_block_ratio"] = - static_cast(shape.cross_block) / query_count; - state.counters["middle_block_ratio"] = - static_cast(shape.middle_block) / query_count; - state.counters["disjoint_boundary_ratio"] = - static_cast(shape.disjoint_boundary) / query_count; - state.counters["fused_boundary_ratio"] = - static_cast(shape.fused_boundary) / query_count; - state.counters["excess_calls_per_query"] = - static_cast(shape.excess_calls) / query_count; - state.counters["avg_same_width"] = - same_count == 0 - ? 0.0 - : static_cast(shape.same_block_width) / same_count; - state.counters["avg_left_boundary_width"] = - cross_count == 0 - ? 0.0 - : static_cast(shape.left_boundary_width) / cross_count; - state.counters["avg_right_boundary_width"] = - cross_count == 0 - ? 0.0 - : static_cast(shape.right_boundary_width) / cross_count; -} - -std::vector make_same_block_ranges(const DepthDataset& dataset) { - std::vector out; - out.reserve(dataset.ranges.size()); - for (const auto [left, right] : dataset.ranges) { - const std::size_t last = right - 1; - const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = last / kBpBlockSize; - if (left_block == right_block) { - out.push_back({left_block, left % kBpBlockSize, last % kBpBlockSize}); - } - } - return out; -} - -std::vector make_left_boundary_ranges(const DepthDataset& dataset) { - std::vector out; - out.reserve(dataset.ranges.size()); - for (const auto [left, right] : dataset.ranges) { - const std::size_t last = right - 1; - const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = last / kBpBlockSize; - if (left_block != right_block) { - out.push_back({left_block, left % kBpBlockSize, - bp_block_size(dataset, left_block) - 1}); - } - } - return out; -} - -std::vector make_right_boundary_ranges( - const DepthDataset& dataset) { - std::vector out; - out.reserve(dataset.ranges.size()); - for (const auto [left, right] : dataset.ranges) { - const std::size_t last = right - 1; - const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = last / kBpBlockSize; - if (left_block != right_block) { - out.push_back({right_block, 0, last % kBpBlockSize}); - } - } - return out; -} - -std::vector> make_boundary_pairs( - const DepthDataset& dataset) { - std::vector> out; - out.reserve(dataset.ranges.size()); - for (const auto [left, right] : dataset.ranges) { - const std::size_t last = right - 1; - const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = last / kBpBlockSize; - if (left_block != right_block) { - out.push_back({{left_block, left % kBpBlockSize, - bp_block_size(dataset, left_block) - 1}, - {right_block, 0, last % kBpBlockSize}}); - } - } - return out; -} - -std::vector> make_disjoint_boundary_pairs( - const DepthDataset& dataset) { - std::vector> out; - out.reserve(dataset.ranges.size()); - for (const auto [left, right] : dataset.ranges) { - const std::size_t last = right - 1; - const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = last / kBpBlockSize; - const std::size_t left_offset = left % kBpBlockSize; - const std::size_t right_offset = last % kBpBlockSize; - if (left_block != right_block && left_offset > right_offset) { - out.push_back( - {{left_block, left_offset, bp_block_size(dataset, left_block) - 1}, - {right_block, 0, right_offset}}); - } - } - return out; -} - -std::vector make_macro_ranges(const DepthDataset& dataset) { - std::vector out; - out.reserve(dataset.ranges.size()); - for (const auto [left, right] : dataset.ranges) { - const std::size_t last = right - 1; - const std::size_t left_block = left / kBpBlockSize; - const std::size_t right_block = last / kBpBlockSize; - if (left_block + 1 < right_block) { - out.push_back({left_block + 1, right_block}); - } - } - return out; -} - -std::vector make_block_summaries( - const DepthDataset& dataset) { - const std::size_t block_count = - (dataset.depths.size() + kBpBlockSize - 1) / kBpBlockSize; - std::vector summaries; - summaries.reserve(block_count); - for (std::size_t block = 0; block < block_count; ++block) { - const std::size_t begin = block * kBpBlockSize; - const std::size_t end = - std::min(begin + kBpBlockSize, dataset.depths.size()); - auto min_it = std::min_element(dataset.depths.begin() + begin, - dataset.depths.begin() + end); - summaries.push_back(BenchBlockSummary::make( - dataset.depths[begin], *min_it, - static_cast(min_it - dataset.depths.begin() - begin))); - } - return summaries; -} - void set_aux_memory_counters(benchmark::State& state, std::size_t size, double aux_bytes); @@ -828,49 +219,33 @@ void set_aux_memory_counters(benchmark::State& state, template void set_query_aux_memory_counters(benchmark::State& state, std::size_t size, - const std::vector& rmqs); + const Rmq& rmq); +/** + * @brief Run value-RMQ query benchmarks over one live RMQ instance. + */ template void run_queries(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); const std::size_t max_width = static_cast(state.range(1)); - const std::vector datasets = make_value_dataset_variants( - size, max_width, value_dataset_variant_count(size)); - std::vector rmqs; - rmqs.reserve(datasets.size()); - for (const Dataset& dataset : datasets) { - rmqs.emplace_back(std::span(dataset.values)); - } - - const std::uint64_t query_seed = kSeed ^ (size * 0xC2B2AE3D27D4EB4Full) ^ - (max_width * 0x165667B19E3779F9ull); - std::vector queries; - queries.reserve(datasets.size()); - for (std::size_t i = 0; i < datasets.size(); ++i) { - queries.emplace_back(size, max_width, query_seed); - } + const SeedContext seeds = make_seed_context(state, size, max_width); + ValueDataset dataset(size, seeds.value_seed); + Rmq rmq(dataset.values()); + RandomQueryGenerator queries(size, max_width, seeds.query_seed); - std::size_t variant = 0; - PerfCounterGroup perf_counters; for (auto _ : state) { - const auto [left, right] = queries[variant].get_random_query(); - std::size_t result = rmqs[variant].arg_min(left, right); + const auto [left, right] = queries.get_random_query(); + std::size_t result = rmq.arg_min(left, right); benchmark::DoNotOptimize(result); - ++variant; - if (variant == rmqs.size()) { - variant = 0; - } - } - PerfCounterGroup::Values perf_values; - if (perf_counters.stop(&perf_values)) { - set_perf_counters(state, perf_values); } state.counters["N"] = static_cast(size); state.counters["max_width"] = static_cast(max_width); state.counters["index_bytes"] = static_cast(sizeof(Index)); - state.counters["value_arrays"] = static_cast(datasets.size()); - set_query_aux_memory_counters(state, size, rmqs); + state.counters["seed_repetition"] = + static_cast(seeds.repetition_index); + state.counters["value_arrays"] = 1.0; + set_query_aux_memory_counters(state, size, rmq); } void set_build_counters(benchmark::State& state, @@ -895,278 +270,46 @@ void set_aux_memory_counters(benchmark::State& state, template void set_query_aux_memory_counters(benchmark::State& state, std::size_t size, - const std::vector& rmqs) { + const Rmq& rmq) { if constexpr (HasRmqMemoryUsage) { - double total = 0.0; - for (const Rmq& rmq : rmqs) { - total += static_cast(rmq.memory_usage_bytes()); - } - const double average = rmqs.empty() ? 0.0 : total / rmqs.size(); - set_aux_memory_counters(state, size, average); + set_aux_memory_counters(state, size, + static_cast(rmq.memory_usage_bytes())); } } +/** + * @brief Run value-RMQ construction benchmarks with one value array. + */ template void run_value_rmq_build(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); - const std::vector datasets = make_value_dataset_variants( - size, size, value_dataset_variant_count(size)); + const SeedContext seeds = make_seed_context(state, size); + ValueDataset dataset(size, seeds.value_seed); - std::size_t variant = 0; for (auto _ : state) { - const Dataset& dataset = datasets[variant]; - Rmq rmq(std::span(dataset.values)); + Rmq rmq(dataset.values()); std::size_t built_size = rmq.size(); benchmark::DoNotOptimize(built_size); benchmark::ClobberMemory(); - ++variant; - if (variant == datasets.size()) { - variant = 0; - } } - set_build_counters(state, size, - datasets.front().values.size() * sizeof(std::int64_t)); - state.counters["value_arrays"] = static_cast(datasets.size()); + set_build_counters(state, size, dataset.size() * sizeof(std::int64_t)); + state.counters["seed_repetition"] = + static_cast(seeds.repetition_index); + state.counters["value_arrays"] = 1.0; if constexpr (HasRmqMemoryUsage) { - const Rmq rmq(std::span(datasets.front().values)); + const Rmq rmq(dataset.values()); const std::size_t aux_bytes = rmq.memory_usage_bytes(); set_aux_memory_counters(state, size, static_cast(aux_bytes)); } } -template -void run_bp_rmq_build(benchmark::State& state) { - const std::size_t size = static_cast(state.range(0)); - const DeltaBitDataset dataset = make_delta_bit_dataset(size); - - for (auto _ : state) { - Rmq rmq(std::span(dataset.bits), dataset.size); - std::size_t built_size = rmq.size(); - benchmark::DoNotOptimize(built_size); - benchmark::ClobberMemory(); - } - - set_build_counters(state, size, dataset.bits.size() * sizeof(std::uint64_t)); - if constexpr (HasRmqMemoryUsage) { - const Rmq rmq(std::span(dataset.bits), dataset.size); - const std::size_t aux_bytes = rmq.memory_usage_bytes(); - set_aux_memory_counters(state, size, static_cast(aux_bytes)); - } -} - -void run_sparse_table_footprint(benchmark::State& state) { - const std::size_t size = static_cast(state.range(0)); - const std::size_t max_width = static_cast(state.range(1)); - const Dataset dataset = make_dataset(size, max_width); - const pixie::rmq::SparseTable, Index> - rmq(std::span(dataset.values)); - const SparseTableFootprintStats stats = - compute_sparse_table_footprint(rmq, size, max_width); - - for (auto _ : state) { - std::size_t query_count = stats.query_count; - benchmark::DoNotOptimize(query_count); - } - - state.counters["N"] = static_cast(size); - state.counters["max_width"] = static_cast(max_width); - state.counters["index_bytes"] = static_cast(sizeof(Index)); - set_sparse_table_footprint_counters(state, stats); -} - -void run_bp_boundary_stack( - benchmark::State& state, - std::vector (*make_ranges)(const DepthDataset&)) { - const std::size_t size = static_cast(state.range(0)); - const std::size_t max_width = static_cast(state.range(1)); - const DepthDataset dataset = make_depth_dataset(size, max_width); - const std::vector ranges = make_ranges(dataset); - if (ranges.empty()) { - state.SkipWithError("no diagnostic ranges for this size/width"); - return; - } - - std::size_t query_index = 0; - for (auto _ : state) { - const BlockRange range = ranges[query_index++ % ranges.size()]; - const auto bits = bp_block_bits_stack(dataset, range.block); - ExcessResult result = excess_min_128(bits.data(), range.left, range.right); - benchmark::DoNotOptimize(result.min_excess); - benchmark::DoNotOptimize(result.offset); - } - - set_depth_counters(state, dataset, false); - state.counters["diagnostic_ranges"] = static_cast(ranges.size()); -} - -void run_bp_boundary_pair_stack(benchmark::State& state) { - const std::size_t size = static_cast(state.range(0)); - const std::size_t max_width = static_cast(state.range(1)); - const DepthDataset dataset = make_depth_dataset(size, max_width); - const auto ranges = make_boundary_pairs(dataset); - if (ranges.empty()) { - state.SkipWithError("no cross-block boundary pairs for this size/width"); - return; - } - - std::size_t query_index = 0; - for (auto _ : state) { - const auto [left, right] = ranges[query_index++ % ranges.size()]; - const auto left_bits = bp_block_bits_stack(dataset, left.block); - const auto right_bits = bp_block_bits_stack(dataset, right.block); - ExcessResult left_result = - excess_min_128(left_bits.data(), left.left, left.right); - ExcessResult right_result = - excess_min_128(right_bits.data(), right.left, right.right); - benchmark::DoNotOptimize(left_result.min_excess); - benchmark::DoNotOptimize(left_result.offset); - benchmark::DoNotOptimize(right_result.min_excess); - benchmark::DoNotOptimize(right_result.offset); - } - - set_depth_counters(state, dataset, false); - state.counters["diagnostic_ranges"] = static_cast(ranges.size()); -} - -void run_bp_boundary_pair_direct(benchmark::State& state) { - const std::size_t size = static_cast(state.range(0)); - const std::size_t max_width = static_cast(state.range(1)); - const DepthDataset dataset = make_depth_dataset(size, max_width); - const auto ranges = make_boundary_pairs(dataset); - if (ranges.empty()) { - state.SkipWithError("no cross-block boundary pairs for this size/width"); - return; - } - - std::size_t query_index = 0; - for (auto _ : state) { - const auto [left, right] = ranges[query_index++ % ranges.size()]; - ExcessResult left_result = excess_min_128( - bp_block_bits_direct(dataset, left.block), left.left, left.right); - ExcessResult right_result = excess_min_128( - bp_block_bits_direct(dataset, right.block), right.left, right.right); - benchmark::DoNotOptimize(left_result.min_excess); - benchmark::DoNotOptimize(left_result.offset); - benchmark::DoNotOptimize(right_result.min_excess); - benchmark::DoNotOptimize(right_result.offset); - } - - set_depth_counters(state, dataset, false); - state.counters["diagnostic_ranges"] = static_cast(ranges.size()); -} - -void run_bp_boundary_pair_disjoint_direct(benchmark::State& state) { - const std::size_t size = static_cast(state.range(0)); - const std::size_t max_width = static_cast(state.range(1)); - const DepthDataset dataset = make_depth_dataset(size, max_width); - const auto ranges = make_disjoint_boundary_pairs(dataset); - if (ranges.empty()) { - state.SkipWithError("no disjoint cross-block boundary pairs"); - return; - } - - std::size_t query_index = 0; - for (auto _ : state) { - const auto [left, right] = ranges[query_index++ % ranges.size()]; - ExcessResult left_result = excess_min_128( - bp_block_bits_direct(dataset, left.block), left.left, left.right); - ExcessResult right_result = excess_min_128( - bp_block_bits_direct(dataset, right.block), right.left, right.right); - benchmark::DoNotOptimize(left_result.min_excess); - benchmark::DoNotOptimize(left_result.offset); - benchmark::DoNotOptimize(right_result.min_excess); - benchmark::DoNotOptimize(right_result.offset); - } - - set_depth_counters(state, dataset, false); - state.counters["diagnostic_ranges"] = static_cast(ranges.size()); -} - -void run_bp_boundary_pair_fused(benchmark::State& state) { - const std::size_t size = static_cast(state.range(0)); - const std::size_t max_width = static_cast(state.range(1)); - const DepthDataset dataset = make_depth_dataset(size, max_width); - const auto ranges = make_disjoint_boundary_pairs(dataset); - if (ranges.empty()) { - state.SkipWithError("no disjoint cross-block boundary pairs"); - return; - } - - std::size_t query_index = 0; - for (auto _ : state) { - const auto [left, right] = ranges[query_index++ % ranges.size()]; - ExcessBoundaryPairResult result = excess_min_128_disjoint_suffix_prefix( - bp_block_bits_direct(dataset, left.block), left.left, - bp_block_bits_direct(dataset, right.block), right.right); - benchmark::DoNotOptimize(result.suffix.min_excess); - benchmark::DoNotOptimize(result.suffix.offset); - benchmark::DoNotOptimize(result.prefix.min_excess); - benchmark::DoNotOptimize(result.prefix.offset); - } - - set_depth_counters(state, dataset, false); - state.counters["diagnostic_ranges"] = static_cast(ranges.size()); -} - -void run_bp_macro_only(benchmark::State& state) { - const std::size_t size = static_cast(state.range(0)); - const std::size_t max_width = static_cast(state.range(1)); - const DepthDataset dataset = make_depth_dataset(size, max_width); - const std::vector block_summaries = - make_block_summaries(dataset); - const pixie::rmq::SparseTable - macro_rmq{std::span(block_summaries)}; - const std::vector ranges = make_macro_ranges(dataset); - if (ranges.empty()) { - state.SkipWithError("no middle-block macro ranges for this size/width"); - return; - } - - std::size_t query_index = 0; - for (auto _ : state) { - const MacroRange range = ranges[query_index++ % ranges.size()]; - std::size_t result = macro_rmq.arg_min(range.left, range.right); - benchmark::DoNotOptimize(result); - } - - set_depth_counters(state, dataset, false); - state.counters["diagnostic_ranges"] = static_cast(ranges.size()); -} - -template -void run_depth_queries(benchmark::State& state) { - const std::size_t size = static_cast(state.range(0)); - const std::size_t max_width = static_cast(state.range(1)); - const DepthDataset dataset = make_depth_dataset(size, max_width); - const Rmq rmq(std::span(dataset.bits), - dataset.depths.size()); - RandomQueryGenerator queries(size, max_width, - kSeed ^ (size * 0xD1B54A32D192ED03ull) ^ - (max_width * 0x94D049BB133111EBull)); - - PerfCounterGroup perf_counters; - for (auto _ : state) { - const auto [left, right] = queries.get_random_query(); - std::size_t result = rmq.arg_min(left, right); - benchmark::DoNotOptimize(result); - } - PerfCounterGroup::Values perf_values; - if (perf_counters.stop(&perf_values)) { - set_perf_counters(state, perf_values); - } - - set_depth_counters(state, dataset, true, BlockSize); - if constexpr (HasRmqMemoryUsage) { - set_aux_memory_counters(state, size, - static_cast(rmq.memory_usage_bytes())); - } -} - void register_benchmarks() { - using CartesianXl = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq< - std::int64_t, std::less, Index>; + using CartesianRmM = + pixie::rmq::CartesianRmM, Index>; + using CartesianHybrid = + pixie::rmq::CartesianHybridBTree, + Index>; const std::vector sizes = {1ull << 10, 1ull << 14, 1ull << 18, 1ull << 22, 1ull << 24, 1ull << 26}; @@ -1177,7 +320,14 @@ void register_benchmarks() { // Pure sparse-table rows materialize O(n log n) indexes; above 2^22 they // dominate memory/runtime and make large benchmark passes noisy. constexpr std::size_t kSparseTableBenchmarkMaxSize = 1ull << 22; - constexpr std::size_t kFocusedCartesianXlLargeSize = 1ull << 28; + constexpr std::size_t kFocusedCartesianRmMLargeSize = 1ull << 28; + constexpr std::size_t kFocusedCartesianHybridLargeSize = 1ull << 28; + constexpr std::size_t kFocusedHybridBTreeLargeSize = 1ull << 28; + // Focused 2^28/2^30 rows are intentionally limited to the RMQ variants whose + // construction and query memory stay bounded at those sizes. CartesianRmM is + // included here because its Cartesian BP construction now uses a succinct + // monotone bit-stack instead of an n-entry index stack. + constexpr std::size_t kVeryLargeHybridSize = 1ull << 30; auto effective_widths_for = [&](std::size_t size) { std::vector effective_widths; @@ -1202,85 +352,100 @@ void register_benchmarks() { &run_value_rmq_build, Index>>) ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); } benchmark::RegisterBenchmark( "rmq_build_segment_tree", &run_value_rmq_build, Index>>) ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); - benchmark::RegisterBenchmark( - "rmq_build_cartesian_tree", - &run_value_rmq_build, Index>>) - ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); - benchmark::RegisterBenchmark( - "rmq_build_cartesian_tree_rmm_btree", - &run_value_rmq_build, Index>>) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_build_cartesian_rmm", + &run_value_rmq_build) ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); - benchmark::RegisterBenchmark("rmq_build_cartesian_tree_segment_btree_xl", - &run_value_rmq_build) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_build_cartesian_hybrid_btree", + &run_value_rmq_build) ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); #ifdef PIXIE_THIRD_PARTY_BENCHMARKS benchmark::RegisterBenchmark( "rmq_build_sdsl_sct", - &run_value_rmq_build, Index>>) + &run_value_rmq_build< + pixie::rmq::SdslSct, Index>>) ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); #endif benchmark::RegisterBenchmark( - "rmq_build_node_euler_btree", - &run_value_rmq_build, Index>>) - ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); - benchmark::RegisterBenchmark( - "rmq_build_node_euler_btree_exp", - &run_value_rmq_build, Index>>) ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); - benchmark::RegisterBenchmark( - "rmq_build_node_euler_btree_exp_mask_leaf", - &run_value_rmq_build, Index, 248, 256, - pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>>) - ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); - benchmark::RegisterBenchmark( - "rmq_build_segment_btree_xl", - &run_value_rmq_build, Index>>) - ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); - benchmark::RegisterBenchmark( - "rmq_build_bp_plus_minus_one", - &run_bp_rmq_build>) - ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); - benchmark::RegisterBenchmark( - "rmq_build_bp_rmm_btree", - &run_bp_rmq_build< - pixie::rmq::experimental::RmMBTreePlusMinusOneRmq>) - ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); - benchmark::RegisterBenchmark( - "rmq_build_bp_one_interval_btree", - &run_bp_rmq_build>) - ->Arg(static_cast(size)) - ->Unit(benchmark::kMillisecond); - } - - benchmark::RegisterBenchmark("rmq_build_cartesian_tree_segment_btree_xl", - &run_value_rmq_build) - ->Arg(static_cast(kFocusedCartesianXlLargeSize)) - ->Unit(benchmark::kMillisecond); + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + + benchmark::RegisterBenchmark("rmq_build_cartesian_rmm", + &run_value_rmq_build) + ->Arg(static_cast(kFocusedCartesianRmMLargeSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_build_cartesian_rmm", + &run_value_rmq_build) + ->Arg(static_cast(kVeryLargeHybridSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_build_cartesian_hybrid_btree", + &run_value_rmq_build) + ->Arg(static_cast(kFocusedCartesianHybridLargeSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_build_cartesian_hybrid_btree", + &run_value_rmq_build) + ->Arg(static_cast(kVeryLargeHybridSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); +#ifdef PIXIE_THIRD_PARTY_BENCHMARKS + benchmark::RegisterBenchmark( + "rmq_build_sdsl_sct", + &run_value_rmq_build< + pixie::rmq::SdslSct, Index>>) + ->Arg(static_cast(kFocusedCartesianRmMLargeSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); +#endif + benchmark::RegisterBenchmark( + "rmq_build_hybrid_btree", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(kFocusedHybridBTreeLargeSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark( + "rmq_build_hybrid_btree", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(kVeryLargeHybridSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); for (const std::size_t size : sizes) { const std::vector effective_widths = @@ -1288,28 +453,15 @@ void register_benchmarks() { for (const std::size_t width : effective_widths) { if (size <= kSparseTableBenchmarkMaxSize) { - if (std::getenv("PIXIE_BENCH_SPARSE_FOOTPRINT") != nullptr) { - benchmark::RegisterBenchmark("rmq_sparse_table_footprint", - run_sparse_table_footprint) - ->Args({static_cast(size), - static_cast(width)}) - ->Iterations(1) - ->Unit(benchmark::kNanosecond); - } benchmark::RegisterBenchmark( "rmq_sparse_table", &run_queries, Index>>) ->Args({static_cast(size), static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_sparse_table_unaligned", - &run_queries, Index, 0>>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); } benchmark::RegisterBenchmark( "rmq_segment_tree", @@ -1317,205 +469,106 @@ void register_benchmarks() { std::less, Index>>) ->Args({static_cast(size), static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_cartesian_tree", - &run_queries, Index>>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_cartesian_tree_rmm_btree", - &run_queries, Index>>) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_cartesian_rmm", + &run_queries) ->Args({static_cast(size), static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark("rmq_cartesian_tree_segment_btree_xl", - &run_queries) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_cartesian_hybrid_btree", + &run_queries) ->Args({static_cast(size), static_cast(width)}) - ->Unit(benchmark::kNanosecond); + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); #ifdef PIXIE_THIRD_PARTY_BENCHMARKS benchmark::RegisterBenchmark( "rmq_sdsl_sct", - &run_queries, Index>>) + &run_queries, Index>>) ->Args({static_cast(size), static_cast(width)}) - ->Unit(benchmark::kNanosecond); + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); #endif benchmark::RegisterBenchmark( - "rmq_node_euler_btree", - &run_queries, Index>>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_node_euler_btree_exp", - &run_queries, Index>>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_node_euler_btree_exp_mask_leaf", - &run_queries, Index, 248, 256, - pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_segment_btree_xl", - &run_queries, Index>>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_bp_plus_minus_one", - &run_depth_queries, 128>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_bp_rmm_btree", - &run_depth_queries< - pixie::rmq::experimental::RmMBTreePlusMinusOneRmq, - pixie::rmq::experimental::RmMBTreePlusMinusOneRmq< - Index>::kBlockSize>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_bp_one_interval_btree", - &run_depth_queries< - pixie::rmq::OneIntervalBTreeRmq, - pixie::rmq::OneIntervalBTreeRmq::kBlockSize>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_bp_one_interval_btree_h1", - &run_depth_queries< - pixie::rmq::OneIntervalBTreeRmq, - pixie::rmq::OneIntervalBTreeRmq::kBlockSize>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_bp_one_interval_btree_h2", - &run_depth_queries< - pixie::rmq::OneIntervalBTreeRmq, - pixie::rmq::OneIntervalBTreeRmq::kBlockSize>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_bp_one_interval_btree_h4", - &run_depth_queries< - pixie::rmq::OneIntervalBTreeRmq, - pixie::rmq::OneIntervalBTreeRmq::kBlockSize>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_bp_one_interval_btree_records", - &run_depth_queries< - pixie::rmq::OneIntervalBTreeRmqBoundaryRecords, - pixie::rmq::OneIntervalBTreeRmqBoundaryRecords< - Index>::kBlockSize>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_bp_one_interval_btree_records_h1", - &run_depth_queries< - pixie::rmq::OneIntervalBTreeRmqBoundaryRecords, - pixie::rmq::OneIntervalBTreeRmqBoundaryRecords::kBlockSize>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_bp_one_interval_btree_records_h2", - &run_depth_queries< - pixie::rmq::OneIntervalBTreeRmqBoundaryRecords, - pixie::rmq::OneIntervalBTreeRmqBoundaryRecords::kBlockSize>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark( - "rmq_bp_one_interval_btree_records_h4", - &run_depth_queries< - pixie::rmq::OneIntervalBTreeRmqBoundaryRecords, - pixie::rmq::OneIntervalBTreeRmqBoundaryRecords::kBlockSize>) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark("rmq_bp_diag_same_block_boundary", - [](benchmark::State& state) { - run_bp_boundary_stack( - state, make_same_block_ranges); - }) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark("rmq_bp_diag_left_boundary", - [](benchmark::State& state) { - run_bp_boundary_stack( - state, make_left_boundary_ranges); - }) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark("rmq_bp_diag_right_boundary", - [](benchmark::State& state) { - run_bp_boundary_stack( - state, make_right_boundary_ranges); - }) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark("rmq_bp_diag_boundary_pair_stack", - run_bp_boundary_pair_stack) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark("rmq_bp_diag_boundary_pair_direct", - run_bp_boundary_pair_direct) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark("rmq_bp_diag_boundary_pair_disjoint_direct", - run_bp_boundary_pair_disjoint_direct) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark("rmq_bp_diag_boundary_pair_fused", - run_bp_boundary_pair_fused) - ->Args({static_cast(size), - static_cast(width)}) - ->Unit(benchmark::kNanosecond); - benchmark::RegisterBenchmark("rmq_bp_diag_macro_only", run_bp_macro_only) + "rmq_hybrid_btree", + &run_queries, Index>>) ->Args({static_cast(size), static_cast(width)}) - ->Unit(benchmark::kNanosecond); + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); } } for (const std::size_t width : - effective_widths_for(kFocusedCartesianXlLargeSize)) { - benchmark::RegisterBenchmark("rmq_cartesian_tree_segment_btree_xl", - &run_queries) - ->Args({static_cast(kFocusedCartesianXlLargeSize), + effective_widths_for(kFocusedCartesianRmMLargeSize)) { + benchmark::RegisterBenchmark("rmq_cartesian_rmm", + &run_queries) + ->Args({static_cast(kFocusedCartesianRmMLargeSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + for (const std::size_t width : effective_widths_for(kVeryLargeHybridSize)) { + benchmark::RegisterBenchmark("rmq_cartesian_rmm", + &run_queries) + ->Args({static_cast(kVeryLargeHybridSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + for (const std::size_t width : + effective_widths_for(kFocusedCartesianHybridLargeSize)) { + benchmark::RegisterBenchmark("rmq_cartesian_hybrid_btree", + &run_queries) + ->Args({static_cast(kFocusedCartesianHybridLargeSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + for (const std::size_t width : effective_widths_for(kVeryLargeHybridSize)) { + benchmark::RegisterBenchmark("rmq_cartesian_hybrid_btree", + &run_queries) + ->Args({static_cast(kVeryLargeHybridSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + for (const std::size_t width : + effective_widths_for(kFocusedHybridBTreeLargeSize)) { + benchmark::RegisterBenchmark( + "rmq_hybrid_btree", + &run_queries, Index>>) + ->Args({static_cast(kFocusedHybridBTreeLargeSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + for (const std::size_t width : effective_widths_for(kVeryLargeHybridSize)) { + benchmark::RegisterBenchmark( + "rmq_hybrid_btree", + &run_queries, Index>>) + ->Args({static_cast(kVeryLargeHybridSize), static_cast(width)}) - ->Unit(benchmark::kNanosecond); + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); } } diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index 7ed60b5..67ff50e 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -1,21 +1,22 @@ #include +#include #include -#include -#include -#include -#include +#include +#include +#include #ifdef SDSL_SUPPORT -#include +#include #endif #include -#include #include #include #include +#include #include #include +#include #include #include @@ -68,41 +69,10 @@ void check_all_arg_min_ranges(const Rmq& rmq, } } -std::vector pack_depth_deltas( - std::span depths) { - if (depths.size() <= 1) { - return {}; - } - std::vector bits((depths.size() - 1 + 63) / 64, 0); - for (std::size_t i = 1; i < depths.size(); ++i) { - if (depths[i] - depths[i - 1] == 1) { - bits[(i - 1) >> 6] |= std::uint64_t{1} << ((i - 1) & 63); - } - } - return bits; -} - bool packed_bit(std::span words, std::size_t position) { return ((words[position >> 6] >> (position & 63)) & 1u) != 0; } -std::size_t naive_select0(std::span words, - std::size_t bit_count, - std::size_t rank) { - if (rank == 0) { - return pixie::rmq::BpPlusMinusOneRmq<>::npos; - } - for (std::size_t position = 0; position < bit_count; ++position) { - if (!packed_bit(words, position)) { - --rank; - if (rank == 0) { - return position; - } - } - } - return pixie::rmq::BpPlusMinusOneRmq<>::npos; -} - template void expect_valid_bp_encoding(const Rmq& rmq, std::size_t value_count) { const std::size_t bit_count = rmq.bp_bit_count(); @@ -129,6 +99,25 @@ void expect_valid_bp_encoding(const Rmq& rmq, std::size_t value_count) { EXPECT_EQ(excess, 0); } +template +std::string bp_string(const Rmq& rmq) { + std::string bits; + bits.reserve(rmq.bp_bit_count()); + const std::span words = rmq.bp_words(); + for (std::size_t position = 0; position < rmq.bp_bit_count(); ++position) { + bits.push_back(packed_bit(words, position) ? '1' : '0'); + } + return bits; +} + +template +void expect_cartesian_bp_shape(const std::vector& values, + const std::string& expected) { + const Rmq rmq{std::span(values)}; + EXPECT_EQ(bp_string(rmq), expected); + check_all_arg_min_ranges(rmq, std::span(values), std::less()); +} + struct SparseTableCase { using Rmq = pixie::rmq::SparseTable; using MaxRmq = pixie::rmq::SparseTable>; @@ -147,219 +136,43 @@ struct SegmentTreeCase { static MaxRmq make_max(std::span values) { return MaxRmq(values); } }; -struct CartesianTreeCase { - using Rmq = pixie::rmq::CartesianTreeRmq; - using MaxRmq = pixie::rmq::CartesianTreeRmq>; - - static Rmq make(std::span values) { return Rmq(values); } - - static MaxRmq make_max(std::span values) { return MaxRmq(values); } -}; - -struct ExperimentalCartesianTreeRmMBTreeCase { - using Rmq = pixie::rmq::experimental::CartesianTreeRmMBTreeRmq; - using MaxRmq = - pixie::rmq::experimental::CartesianTreeRmMBTreeRmq>; - - static Rmq make(std::span values) { return Rmq(values); } - - static MaxRmq make_max(std::span values) { return MaxRmq(values); } -}; - -struct ExperimentalCartesianTreeSegmentBTreeXLCase { - using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq; - using MaxRmq = pixie::rmq::experimental:: - CartesianTreeSegmentBTreeXLRmq>; - - static Rmq make(std::span values) { return Rmq(values); } - - static MaxRmq make_max(std::span values) { return MaxRmq(values); } -}; - -struct NodeEulerBTreeCase { - using Rmq = pixie::rmq::NodeEulerBTreeRmq; - using MaxRmq = pixie::rmq::NodeEulerBTreeRmq>; - - static Rmq make(std::span values) { return Rmq(values); } - - static MaxRmq make_max(std::span values) { return MaxRmq(values); } -}; - -struct ExperimentalNodeEulerBTreeCase { - using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; - using MaxRmq = - pixie::rmq::experimental::NodeEulerBTreeRmq>; +struct CartesianRmMCase { + using Rmq = pixie::rmq::CartesianRmM; + using MaxRmq = pixie::rmq::CartesianRmM>; static Rmq make(std::span values) { return Rmq(values); } static MaxRmq make_max(std::span values) { return MaxRmq(values); } }; -struct ExperimentalNodeEulerBTreeMaskLeafCase { - using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq< - int, - std::less, - std::size_t, - 248, - 256, - pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>; - using MaxRmq = pixie::rmq::experimental::NodeEulerBTreeRmq< - int, - std::greater, - std::size_t, - 248, - 256, - pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>; +struct CartesianHybridBTreeCase { + using Rmq = pixie::rmq::CartesianHybridBTree; + using MaxRmq = pixie::rmq::CartesianHybridBTree>; static Rmq make(std::span values) { return Rmq(values); } static MaxRmq make_max(std::span values) { return MaxRmq(values); } }; -struct SegmentBTreeXlCase { - using Rmq = pixie::rmq::SegmentBTreeXl; - using MaxRmq = pixie::rmq::SegmentBTreeXl>; +struct HybridBTreeCase { + using Rmq = pixie::rmq::HybridBTree; + using MaxRmq = pixie::rmq::HybridBTree>; static Rmq make(std::span values) { return Rmq(values); } static MaxRmq make_max(std::span values) { return MaxRmq(values); } }; -struct BpPlusMinusOne128Case { - using Rmq = pixie::rmq::BpPlusMinusOneRmq<>; - static constexpr std::size_t kBlockSize = 128; - - static Rmq make(std::span bits, - std::size_t depth_count) { - return Rmq(bits, depth_count); - } -}; - -struct BpPlusMinusOne64Case { - using Rmq = pixie::rmq::BpPlusMinusOneRmq; - static constexpr std::size_t kBlockSize = 64; - - static Rmq make(std::span bits, - std::size_t depth_count) { - return Rmq(bits, depth_count); - } -}; - -struct ExperimentalRmMBTreePlusMinusOneCase { - using Rmq = pixie::rmq::experimental::RmMBTreePlusMinusOneRmq<>; - static constexpr std::size_t kBlockSize = Rmq::kBlockSize; - - static Rmq make(std::span bits, - std::size_t depth_count) { - return Rmq(bits, depth_count); - } -}; - -struct ExperimentalSegmentBTreeXLPlusMinusOneCase { - using Rmq = pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq<>; - static constexpr std::size_t kBlockSize = Rmq::kLeafSize; - - static Rmq make(std::span bits, - std::size_t depth_count) { - return Rmq(bits, depth_count); - } -}; - -struct OneIntervalBTreeCase { - using Rmq = pixie::rmq::OneIntervalBTreeRmq<>; - static constexpr std::size_t kBlockSize = Rmq::kBlockSize; - - static Rmq make(std::span bits, - std::size_t depth_count) { - return Rmq(bits, depth_count); - } -}; - -struct OneIntervalBTreeBoundaryRecordsCase { - using Rmq = pixie::rmq::OneIntervalBTreeRmqBoundaryRecords<>; - static constexpr std::size_t kBlockSize = Rmq::kBlockSize; - - static Rmq make(std::span bits, - std::size_t depth_count) { - return Rmq(bits, depth_count); - } -}; - -template -typename DepthCase::Rmq make_depth_rmq(const std::vector& bits, - std::size_t depth_count) { - return DepthCase::make(std::span(bits), depth_count); -} - -template -void check_depths_all_arg_min(std::span depths) { - const std::vector bits = pack_depth_deltas(depths); - const typename DepthCase::Rmq rmq = - make_depth_rmq(bits, depths.size()); - check_all_arg_min_ranges(rmq, depths, std::less()); -} - -template -void check_depth_ranges( - const Rmq& rmq, - std::span depths, - std::span> ranges) { - for (const auto [left, right] : ranges) { - EXPECT_EQ(rmq.arg_min(left, right), - naive_arg_min(depths, left, right, std::less())) - << "range=[" << left << "," << right << ")"; - } -} - -template -void check_depth_case_ranges( - std::span depths, - std::span> ranges) { - const std::vector bits = pack_depth_deltas(depths); - const typename DepthCase::Rmq rmq = - make_depth_rmq(bits, depths.size()); - check_depth_ranges(rmq, depths, ranges); -} - -void check_one_interval_btree_ranges( - std::span depths, - std::span> ranges) { - const std::vector bits = pack_depth_deltas(depths); - const pixie::rmq::OneIntervalBTreeRmq<> rmq(bits, depths.size()); - const pixie::rmq::OneIntervalBTreeRmqBoundaryRecords<> records(bits, - depths.size()); - const pixie::rmq::BpPlusMinusOneRmq<> baseline(bits, depths.size()); - for (const auto [left, right] : ranges) { - const std::size_t expected = - naive_arg_min(std::span(depths), left, right, - std::less()); - EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - EXPECT_EQ(rmq.arg_min(left, right), baseline.arg_min(left, right)) - << "range=[" << left << "," << right << ")"; - EXPECT_EQ(records.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - EXPECT_EQ(records.arg_min(left, right), baseline.arg_min(left, right)) - << "range=[" << left << "," << right << ")"; - } -} - } // namespace template class ValueRmqContractTest : public ::testing::Test {}; -using ValueRmqCases = - ::testing::Types; +using ValueRmqCases = ::testing::Types; TYPED_TEST_SUITE(ValueRmqContractTest, ValueRmqCases); TYPED_TEST(ValueRmqContractTest, ExhaustiveSmallArray) { @@ -369,6 +182,61 @@ TYPED_TEST(ValueRmqContractTest, ExhaustiveSmallArray) { check_all_ranges(rmq, std::span(values), std::less()); } +TEST(RmqCartesianBuildShape, SuccinctStackPreservesBpEncoding) { + using Hybrid = pixie::rmq::CartesianHybridBTree; + using RmM = pixie::rmq::CartesianRmM; + + const std::vector, std::string>> cases = { + {{1}, "10"}, + {{1, 2}, "1010"}, + {{2, 1}, "1100"}, + {{1, 1}, "1010"}, + {{3, 1, 2}, "110010"}, + {{1, 3, 2}, "101100"}, + {{3, 2, 2, 2, 1, 1, 2, 1, 3}, "111001010010110010"}, + {{4, 1, 3, 1, 5, 0, 0, 2}, "1110011001001010"}, + }; + + for (const auto& [values, expected] : cases) { + SCOPED_TRACE(expected); + expect_cartesian_bp_shape(values, expected); + expect_cartesian_bp_shape(values, expected); + } +} + +TEST(RmqSuccinctIncreasingStack, PopsAcrossEmptyBlocks) { + pixie::rmq::utils::SuccinctIncreasingStack stack(256); + EXPECT_TRUE(stack.empty()); + + stack.push(0); + EXPECT_EQ(stack.top(), 0u); + stack.push(62); + stack.push(63); + EXPECT_EQ(stack.top(), 63u); + stack.pop(); + EXPECT_EQ(stack.top(), 62u); + stack.pop(); + EXPECT_EQ(stack.top(), 0u); + + stack.push(130); + EXPECT_EQ(stack.top(), 130u); + stack.pop(); + EXPECT_EQ(stack.top(), 0u); + stack.push(131); + stack.pop(); + EXPECT_EQ(stack.top(), 0u); + + stack.push(190); + stack.push(191); + EXPECT_EQ(stack.top(), 191u); + stack.pop(); + EXPECT_EQ(stack.top(), 190u); + stack.pop(); + EXPECT_EQ(stack.top(), 0u); + stack.pop(); + EXPECT_TRUE(stack.empty()); +} + TYPED_TEST(ValueRmqContractTest, FirstMinimumTieBreaking) { const std::vector values = {7, 2, 2, 3, 2}; const typename TypeParam::Rmq rmq = @@ -439,15 +307,14 @@ TYPED_TEST(ValueRmqContractTest, DifferentialRandom) { #ifdef SDSL_SUPPORT TEST(RmqSdslSct, MatchesPixieMinContractForSignedValuesAndDuplicates) { const std::vector values = {4, -3, 7, -3, 0, -8, -8, 2, 2}; - const pixie::rmq::SdslSctRmq rmq{std::span(values)}; + const pixie::rmq::SdslSct rmq{std::span(values)}; check_all_ranges(rmq, std::span(values), std::less()); EXPECT_EQ(rmq.arg_min(0, values.size()), 5u); EXPECT_EQ(rmq.arg_min(5, 7), 5u); EXPECT_EQ(rmq.arg_min(6, 7), 6u); - EXPECT_EQ(rmq.arg_min(3, 3), pixie::rmq::SdslSctRmq::npos); - EXPECT_EQ(rmq.arg_min(0, values.size() + 1), - pixie::rmq::SdslSctRmq::npos); + EXPECT_EQ(rmq.arg_min(3, 3), pixie::rmq::SdslSct::npos); + EXPECT_EQ(rmq.arg_min(0, values.size() + 1), pixie::rmq::SdslSct::npos); } TEST(RmqSdslSct, DifferentialRandom) { @@ -458,104 +325,117 @@ TEST(RmqSdslSct, DifferentialRandom) { std::generate(values.begin(), values.end(), [&] { return value_dist(rng); }); - const pixie::rmq::SdslSctRmq rmq{std::span(values)}; + const pixie::rmq::SdslSct rmq{std::span(values)}; check_all_ranges(rmq, std::span(values), std::less()); } const std::vector empty_values; - const pixie::rmq::SdslSctRmq empty{std::span(empty_values)}; + const pixie::rmq::SdslSct empty{std::span(empty_values)}; EXPECT_TRUE(empty.empty()); - EXPECT_EQ(empty.arg_min(0, 0), pixie::rmq::SdslSctRmq::npos); + EXPECT_EQ(empty.arg_min(0, 0), pixie::rmq::SdslSct::npos); } #endif -TEST(RmqNodeEulerBTree, BoundarySizesAroundLeavesAndInternalNodes) { - using Rmq = pixie::rmq::NodeEulerBTreeRmq; +TEST(RmqHybridBTree, BoundaryAndFallbackRanges) { + using Rmq = pixie::rmq::HybridBTree; constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kFanout = Rmq::kFanout; - const std::vector sizes = { - kLeaf - 1, - kLeaf, - kLeaf + 1, - kLeaf * kFanout - 1, - kLeaf * kFanout, - kLeaf * kFanout + 1, - kLeaf * kFanout + kLeaf + 1, + static_assert(kLeaf == 496); + + std::vector values(2 * kLeaf + 17, 2000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 7 + i / 3) % 31); + } + + values[32] = 120; + values[96] = 70; + values[160] = 55; + values[240] = -100; + values[241] = -100; + values[320] = 30; + values[440] = 5; + values[kLeaf - 1] = 5; + + values[kLeaf + 11] = -9; + values[kLeaf + 173] = -17; + values[2 * kLeaf + 5] = -21; + + const Rmq rmq{std::span(values)}; + const std::vector> ranges = { + {0, kLeaf}, {0, 97}, + {0, 240}, {96, 241}, + {250, kLeaf}, {441, kLeaf}, + {300, 450}, {kLeaf - 7, kLeaf + 12}, + {kLeaf, 2 * kLeaf}, {2 * kLeaf - 5, values.size()}, + {0, values.size()}, }; - for (const std::size_t size : sizes) { - SCOPED_TRACE(size); - std::vector values(size); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 17 + i / 5) % 31); - if (i % 9 == 0 || i % 32 == 3) { - values[i] = 4; - } - } + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(rmq.range_min(left, right), values[expected]) + << "range=[" << left << "," << right << ")"; + } +} - const Rmq rmq{std::span(values)}; - const auto check_range = [&](std::size_t left, std::size_t right) { - ASSERT_LT(left, right); - ASSERT_LE(right, values.size()); - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - EXPECT_EQ(rmq.range_min(left, right), values[expected]) - << "range=[" << left << "," << right << ")"; - }; +TEST(RmqHybridBTree, LeafSelectorEnumVariants) { + using MaskRmq = pixie::rmq::HybridBTree< + int, std::less, std::size_t, 248, 256, + pixie::rmq::HybridBTreeLeafSelector::PrefixSuffix>; + using BpRmq = + pixie::rmq::HybridBTree, std::size_t, 252, 256, + pixie::rmq::HybridBTreeLeafSelector::BP>; + + std::vector values(4099, 10000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 37 + i / 7) % 257); + } + values[13] = -50; + values[251] = -70; + values[252] = -70; + values[747] = -90; + values[2048] = -120; + values[4098] = -110; - check_range(0, 1); - check_range(0, size); - check_range(size - 1, size); + const MaskRmq mask_rmq{std::span(values)}; + const BpRmq bp_rmq{std::span(values)}; + const std::vector> ranges = { + {0, 1}, {0, 252}, {1, 251}, {248, 253}, {251, 753}, + {700, 2100}, {2048, 2049}, {0, values.size()}, {3000, 4099}, + }; - const std::vector boundaries = { - kLeaf, - kLeaf * kFanout, - }; - for (const std::size_t boundary : boundaries) { - if (boundary >= size) { - continue; - } - const std::size_t near_left = boundary > 19 ? boundary - 19 : 0; - const std::size_t wide_left = - boundary > kLeaf + 3 ? boundary - kLeaf - 3 : 0; - check_range(boundary - 1, boundary); - check_range(boundary, boundary + 1); - check_range(boundary - 1, boundary + 1); - check_range(near_left, std::min(size, boundary + 23)); - check_range(wide_left, std::min(size, boundary + kLeaf + 5)); - check_range(1, std::min(size, boundary + 1)); - check_range(boundary, size); - } + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(mask_rmq.arg_min(left, right), expected) + << "mask range=[" << left << "," << right << ")"; + EXPECT_EQ(bp_rmq.arg_min(left, right), expected) + << "bp range=[" << left << "," << right << ")"; } } -TEST(RmqNodeEulerBTree, CommonAncestorQueryShapes) { - using Rmq = pixie::rmq::NodeEulerBTreeRmq; +TEST(RmqHybridBTree, MiddleFanoutBoundaryRanges) { + using Rmq = pixie::rmq::HybridBTree; constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kFanout = Rmq::kFanout; - const std::size_t first_level_span = kLeaf * kFanout; - std::vector values(first_level_span + 3 * kLeaf + 17); + constexpr std::size_t kMiddleBoundary = kLeaf * Rmq::kMiddleFanout; + + std::vector values(kMiddleBoundary + 2 * kLeaf + 17, 10000); for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 29 + i / 13) % 101); - if (i % 257 == 0) { - values[i] = -7; - } - if (i % 4099 == 3) { - values[i] = -9; - } + values[i] += static_cast((i * 17 + i / 11) % 257); } + values[13] = -100; + values[kMiddleBoundary - 29] = -90; + values[kMiddleBoundary + kLeaf + 7] = -110; const Rmq rmq{std::span(values)}; const std::vector> ranges = { - {7, 99}, - {kLeaf - 3, 2 * kLeaf + 5}, - {17, 18 * kLeaf + 111}, - {first_level_span - 4, first_level_span + kLeaf + 9}, - {kLeaf * (kFanout - 7) + 13, first_level_span + 2 * kLeaf + 19}, - {0, first_level_span}, - {kLeaf, first_level_span + kLeaf}, + {0, values.size()}, + {kMiddleBoundary - 2 * kLeaf, kMiddleBoundary + 2 * kLeaf}, + {kMiddleBoundary - 31, kMiddleBoundary + kLeaf + 31}, + {kMiddleBoundary + 1, values.size()}, + {kLeaf * (Rmq::kMiddleFanout - 1) + 9, kMiddleBoundary + kLeaf + 13}, + {values.size() - 3 * kLeaf, values.size()}, }; for (const auto [left, right] : ranges) { @@ -566,21 +446,30 @@ TEST(RmqNodeEulerBTree, CommonAncestorQueryShapes) { } } -TEST(RmqNodeEulerBTree, SelectorFirstCorrectsInvalidBorderMinima) { - using Rmq = pixie::rmq::NodeEulerBTreeRmq; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - std::vector values(3 * kLeaf, 1000); +TEST(RmqHybridBTree, TopSparseOverlayBoundaryRanges) { + using Rmq = pixie::rmq::HybridBTree; + constexpr std::size_t kTopBlock = Rmq::kMinTopSparseBlockSize; + + std::vector values(3 * kTopBlock + 77, 100000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 23 + i / 7) % 311); + } values[0] = -1000; - values[kLeaf / 2] = 50; - values[kLeaf + 44] = -10; - values[2 * kLeaf + 10] = 60; - values[2 * kLeaf + 200] = -900; + values[kTopBlock + 17] = -900; + values[kTopBlock + 18] = -900; + values[2 * kTopBlock + 41] = -950; + values[3 * kTopBlock + 40] = -800; const Rmq rmq{std::span(values)}; + EXPECT_EQ(rmq.top_sparse_block_size(), kTopBlock); + EXPECT_EQ(rmq.top_sparse_block_count(), 4u); + const std::vector> ranges = { - {kLeaf / 2, 2 * kLeaf + 100}, - {kLeaf / 2, kLeaf + 100}, - {kLeaf - 50, 3 * kLeaf - 100}, + {1, 2 * kTopBlock + 100}, + {kTopBlock, kTopBlock + 100}, + {kTopBlock + 18, values.size()}, + {kTopBlock - 5, 2 * kTopBlock + 5}, + {0, values.size()}, }; for (const auto [left, right] : ranges) { @@ -591,14 +480,67 @@ TEST(RmqNodeEulerBTree, SelectorFirstCorrectsInvalidBorderMinima) { } } -TEST(RmqNodeEulerBTree, DuplicateHeavyRandomDifferentialTo8193) { - using Rmq = pixie::rmq::NodeEulerBTreeRmq; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kFanout = Rmq::kFanout; - std::mt19937_64 rng(811); +TEST(RmqHybridBTree, TopSparseOverlayCapsBlockCount) { + using Rmq = pixie::rmq::HybridBTree; + constexpr std::size_t kMinBlock = Rmq::kMinTopSparseBlockSize; + constexpr std::size_t kMaxBlocks = Rmq::kMaxTopSparseBlocks; + constexpr std::size_t kLargestFixedBlockInput = kMinBlock * kMaxBlocks; + + EXPECT_EQ(Rmq::top_sparse_block_size_for(0), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(0), 0u); + EXPECT_EQ(Rmq::top_sparse_block_size_for(1), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(1), 1u); + EXPECT_EQ(Rmq::top_sparse_block_size_for(kLargestFixedBlockInput), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(kLargestFixedBlockInput), + kMaxBlocks); + EXPECT_GT(Rmq::top_sparse_block_size_for(kLargestFixedBlockInput + 1), + kMinBlock); + EXPECT_LE(Rmq::top_sparse_block_count_for(kLargestFixedBlockInput + 1), + kMaxBlocks); + + const std::size_t huge = std::numeric_limits::max() / 4; + EXPECT_LE(Rmq::top_sparse_block_count_for(huge), kMaxBlocks); + + std::vector values(3 * kMinBlock + 7, 0); + const Rmq rmq(values); + EXPECT_EQ(rmq.top_sparse_block_size(), kMinBlock); + EXPECT_EQ(rmq.top_sparse_block_count(), 4u); +} + +TEST(RmqHybridBTree, TopSparseOverlayComparatorMaximum) { + using Rmq = pixie::rmq::HybridBTree>; + constexpr std::size_t kTopBlock = Rmq::kMinTopSparseBlockSize; + + std::vector values(2 * kTopBlock + 33, -1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = -1000 + static_cast((i * 13 + i / 5) % 71); + } + values[0] = 1000; + values[kTopBlock + 17] = 900; + values[kTopBlock + 18] = 900; + + const Rmq rmq(values); + const std::vector> ranges = { + {1, values.size()}, + {kTopBlock, kTopBlock + 100}, + {kTopBlock + 18, values.size()}, + {0, values.size()}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min( + std::span(values), left, right, std::greater()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqHybridBTree, DuplicateHeavyRandomTo8193) { + using Rmq = pixie::rmq::HybridBTree; + std::mt19937_64 rng(901496); std::uniform_int_distribution value_dist(-3, 3); const std::vector sizes = { - 1, 2, 17, 255, 256, 257, 1024, 8191, 8192, 8193, kLeaf * kFanout + 1, + 1, 2, 17, 495, 496, 497, 1024, 4096, 8191, 8192, 8193, }; for (const std::size_t size : sizes) { @@ -606,7 +548,7 @@ TEST(RmqNodeEulerBTree, DuplicateHeavyRandomDifferentialTo8193) { std::vector values(size); for (std::size_t i = 0; i < values.size(); ++i) { values[i] = value_dist(rng); - if ((i % 11) < 5) { + if ((i % 11) < 6) { values[i] = 0; } } @@ -626,414 +568,50 @@ TEST(RmqNodeEulerBTree, DuplicateHeavyRandomDifferentialTo8193) { } } -TEST(RmqNodeEulerBTree, TopDepthSparseSelectorLevelShapes) { - using MinRmq = - pixie::rmq::NodeEulerBTreeRmq, std::size_t, 4, 4>; - using MaxRmq = - pixie::rmq::NodeEulerBTreeRmq, std::size_t, 4, 4>; +TEST(RmqCartesianHybridBTree, BoundarySizesAndBpEncoding) { + using Rmq = pixie::rmq::CartesianHybridBTree; const std::vector sizes = { - 4, // no internal level - 5, // one internal level - 17, // two internal levels - 65, // three internal levels - 257, // more than three internal levels + 1u, 255u, 256u, 257u, 511u, 512u, 513u, 8191u, 8192u, 8193u, }; for (const std::size_t size : sizes) { SCOPED_TRACE(size); std::vector values(size); for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 19 + i / 3) % 23); - if (i % 7 == 0 || i % 11 == 3) { - values[i] = -2; + values[i] = static_cast((i * 37 + i / 5) % 53); + if (i % 17 == 0 || i % 257 == 3) { + values[i] = -4; } - if (i % 29 == 5) { - values[i] = 31; - } - } - - const MinRmq min_rmq{std::span(values)}; - const MaxRmq max_rmq{std::span(values)}; - check_all_arg_min_ranges(min_rmq, std::span(values), - std::less()); - check_all_arg_min_ranges(max_rmq, std::span(values), - std::greater()); - } -} - -TEST(RmqNodeEulerBTree, TopDepthSparseSelectorDefaultFanoutRandom) { - using Rmq = pixie::rmq::NodeEulerBTreeRmq; - using MaxRmq = pixie::rmq::NodeEulerBTreeRmq>; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kFanout = Rmq::kFanout; - const std::size_t size = kLeaf * kFanout + kLeaf + 37; - - std::vector values(size); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 31 + i / 9) % 127); - if (i % 257 == 3 || i % 8191 == 17) { - values[i] = -9; - } - if (i % 4099 == 19) { - values[i] = 211; } - } - const Rmq rmq{std::span(values)}; - const MaxRmq max_rmq{std::span(values)}; - std::mt19937_64 rng(424242); - std::uniform_int_distribution width_dist(1, values.size()); - for (std::size_t query = 0; query < 3000; ++query) { - const std::size_t width = width_dist(rng); - std::uniform_int_distribution left_dist(0, - values.size() - width); - const std::size_t left = left_dist(rng); - const std::size_t right = left + width; - EXPECT_EQ(rmq.arg_min(left, right), - naive_arg_min(std::span(values), left, right, - std::less())) - << "range=[" << left << "," << right << ")"; - EXPECT_EQ(max_rmq.arg_min(left, right), - naive_arg_min(std::span(values), left, right, - std::greater())) - << "range=[" << left << "," << right << ")"; - } -} - -TEST(RmqNodeEulerBTree, CopyAndMovePreserveSelectors) { - using Rmq = pixie::rmq::NodeEulerBTreeRmq; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kFanout = Rmq::kFanout; - std::vector values(kLeaf * kFanout + 512); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 43 + i / 7) % 97); - if (i % 13 == 0) { - values[i] = -5; - } - } - - const Rmq original{std::span(values)}; - Rmq copied(original); - Rmq assigned; - assigned = original; - Rmq moved(std::move(copied)); - Rmq move_assigned; - move_assigned = std::move(assigned); - - const std::vector> ranges = { - {0, 1}, {0, values.size()}, - {255, 257}, {kLeaf * kFanout - 5, kLeaf * kFanout + 6}, - {1234, 5678}, {values.size() - 17, values.size()}, - }; + const Rmq rmq(values); + expect_valid_bp_encoding(rmq, values.size()); - const std::array rmqs = {&original, &moved, &move_assigned}; - for (const Rmq* rmq : rmqs) { - ASSERT_EQ(rmq->size(), values.size()); + const std::vector> ranges = { + {0, 1}, + {0, size}, + {size - 1, size}, + {size / 3, std::min(size, size / 3 + 19)}, + {size / 2, std::min(size, size / 2 + 37)}, + }; for (const auto [left, right] : ranges) { - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(rmq->arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - } - } -} - -TEST(RmqExperimentalNodeEulerBTree, SplitLevelBoundarySizes) { - using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kFanout = Rmq::kFanout; - constexpr std::size_t kMediumSpan = kLeaf * kFanout; - const std::vector sizes = { - kLeaf - 1, - kLeaf, - kLeaf + 1, - kMediumSpan - 1, - kMediumSpan, - kMediumSpan + 1, - kMediumSpan + 3 * kLeaf + 17, - }; - - for (const std::size_t size : sizes) { - SCOPED_TRACE(size); - std::vector values(size); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 37 + i / 11) % 113); - if (i % 17 == 0 || i % 257 == 3) { - values[i] = -4; - } - if (i % 4099 == 19) { - values[i] = -11; - } - } - - const Rmq rmq{std::span(values)}; - const auto check_range = [&](std::size_t left, std::size_t right) { ASSERT_LT(left, right); - ASSERT_LE(right, values.size()); - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - EXPECT_EQ(rmq.range_min(left, right), values[expected]) + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())) << "range=[" << left << "," << right << ")"; - }; - - check_range(0, 1); - check_range(0, size); - check_range(size - 1, size); - - for (const std::size_t boundary : {kLeaf, kMediumSpan}) { - if (boundary == 0 || boundary >= size) { - continue; - } - check_range(boundary - 1, boundary); - check_range(boundary, boundary + 1); - check_range(boundary - 1, boundary + 1); - check_range(boundary > 67 ? boundary - 67 : 0, - std::min(size, boundary + 71)); - check_range(boundary > kLeaf + 5 ? boundary - kLeaf - 5 : 0, - std::min(size, boundary + kLeaf + 9)); } } } -TEST(RmqExperimentalNodeEulerBTree, LeafEmbeddedOffsetBoundaryRanges) { - using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - - std::vector values(2 * kLeaf + 5); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 29 + i / 7) % 97) + 10; - } - values[kLeaf - 1] = -7; - values[kLeaf] = -7; - values[2 * kLeaf] = -9; - - const Rmq rmq{std::span(values)}; - const std::vector> ranges = { - {0, kLeaf}, {0, kLeaf + 1}, - {kLeaf, kLeaf + 1}, {kLeaf - 3, kLeaf + 4}, - {kLeaf, kLeaf + 4}, {2 * kLeaf - 2, 2 * kLeaf + 3}, - {0, values.size()}, - }; - - for (const auto [left, right] : ranges) { - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - EXPECT_EQ(rmq.range_min(left, right), values[expected]) - << "range=[" << left << "," << right << ")"; - } -} - -TEST(RmqExperimentalNodeEulerBTree, MaskLeafBoundaryAndFallbackRanges) { - using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq< - int, std::less, std::size_t, 248, 256, - pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - static_assert(kLeaf == 248); - - std::vector values(2 * kLeaf + 13, 1000); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] += static_cast(i % 17); - } - - values[20] = 80; - values[40] = 70; - values[80] = 60; - values[90] = 60; - values[123] = -100; - values[160] = 10; - values[200] = 5; - values[220] = 0; - values[kLeaf - 1] = 0; - - values[kLeaf + 7] = -7; - values[kLeaf + 91] = -11; - values[2 * kLeaf + 3] = -13; - - const Rmq rmq{std::span(values)}; - const std::vector> ranges = { - {0, kLeaf}, - {0, 41}, - {0, 100}, - {0, 123}, - {90, 160}, - {130, kLeaf}, - {221, kLeaf}, - {130, 210}, - {kLeaf - 5, kLeaf + 8}, - {kLeaf, 2 * kLeaf}, - {2 * kLeaf - 3, values.size()}, - {0, values.size()}, - }; - - for (const auto [left, right] : ranges) { - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - EXPECT_EQ(rmq.range_min(left, right), values[expected]) - << "range=[" << left << "," << right << ")"; - } -} - -TEST(RmqSegmentBTreeXl, BoundaryAndFallbackRanges) { - using Rmq = pixie::rmq::SegmentBTreeXl; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - static_assert(kLeaf == 496); - - std::vector values(2 * kLeaf + 17, 2000); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] += static_cast((i * 7 + i / 3) % 31); - } - - values[32] = 120; - values[96] = 70; - values[160] = 55; - values[240] = -100; - values[241] = -100; - values[320] = 30; - values[440] = 5; - values[kLeaf - 1] = 5; - - values[kLeaf + 11] = -9; - values[kLeaf + 173] = -17; - values[2 * kLeaf + 5] = -21; - - const Rmq rmq{std::span(values)}; - const std::vector> ranges = { - {0, kLeaf}, {0, 97}, - {0, 240}, {96, 241}, - {250, kLeaf}, {441, kLeaf}, - {300, 450}, {kLeaf - 7, kLeaf + 12}, - {kLeaf, 2 * kLeaf}, {2 * kLeaf - 5, values.size()}, - {0, values.size()}, - }; - - for (const auto [left, right] : ranges) { - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - EXPECT_EQ(rmq.range_min(left, right), values[expected]) - << "range=[" << left << "," << right << ")"; - } -} - -TEST(RmqSegmentBTreeXl, LeafSelectorEnumVariants) { - using MaskRmq = pixie::rmq::SegmentBTreeXL< - int, std::less, std::size_t, 248, 256, - pixie::rmq::SegmentBTreeXLLeafSelector::PrefixSuffix>; - using BpRmq = - pixie::rmq::SegmentBTreeXL, std::size_t, 252, 256, - pixie::rmq::SegmentBTreeXLLeafSelector::BP>; - - std::vector values(4099, 10000); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 37 + i / 7) % 257); - } - values[13] = -50; - values[251] = -70; - values[252] = -70; - values[747] = -90; - values[2048] = -120; - values[4098] = -110; - - const MaskRmq mask_rmq{std::span(values)}; - const BpRmq bp_rmq{std::span(values)}; - const std::vector> ranges = { - {0, 1}, {0, 252}, {1, 251}, {248, 253}, {251, 753}, - {700, 2100}, {2048, 2049}, {0, values.size()}, {3000, 4099}, - }; - - for (const auto [left, right] : ranges) { - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(mask_rmq.arg_min(left, right), expected) - << "mask range=[" << left << "," << right << ")"; - EXPECT_EQ(bp_rmq.arg_min(left, right), expected) - << "bp range=[" << left << "," << right << ")"; - } -} - -TEST(RmqExperimentalNodeEulerBTree, DuplicateHeavyRandomDifferentialTo8193) { - using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; - std::mt19937_64 rng(9127); +TEST(RmqCartesianHybridBTree, DuplicateHeavyRandomDifferentialTo8193) { + using Rmq = pixie::rmq::CartesianHybridBTree; + std::mt19937_64 rng(20260610); std::uniform_int_distribution value_dist(-3, 3); const std::vector sizes = { 1, 2, 17, 255, 256, 257, 1024, 4096, 8191, 8192, 8193, }; - for (const std::size_t size : sizes) { - SCOPED_TRACE(size); - std::vector values(size); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = value_dist(rng); - if ((i % 13) < 7) { - values[i] = 0; - } - } - - const Rmq rmq{std::span(values)}; - std::uniform_int_distribution width_dist(1, size); - for (std::size_t query = 0; query < 2000; ++query) { - const std::size_t width = width_dist(rng); - std::uniform_int_distribution left_dist(0, size - width); - const std::size_t left = left_dist(rng); - const std::size_t right = left + width; - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - } - } -} - -TEST(RmqExperimentalNodeEulerBTree, MaskLeafDuplicateHeavyRandomTo8193) { - using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq< - int, std::less, std::size_t, 248, 256, - pixie::rmq::experimental::PrefixSuffixMaskLeafSelectorTag>; - std::mt19937_64 rng(901248); - std::uniform_int_distribution value_dist(-3, 3); - const std::vector sizes = { - 1, 2, 17, 247, 248, 249, 1024, 4096, 8191, 8192, 8193, - }; - - for (const std::size_t size : sizes) { - SCOPED_TRACE(size); - std::vector values(size); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = value_dist(rng); - if ((i % 11) < 6) { - values[i] = 0; - } - } - - const Rmq rmq{std::span(values)}; - std::uniform_int_distribution width_dist(1, size); - for (std::size_t query = 0; query < 2000; ++query) { - const std::size_t width = width_dist(rng); - std::uniform_int_distribution left_dist(0, size - width); - const std::size_t left = left_dist(rng); - const std::size_t right = left + width; - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - } - } -} - -TEST(RmqSegmentBTreeXl, DuplicateHeavyRandomTo8193) { - using Rmq = pixie::rmq::SegmentBTreeXl; - std::mt19937_64 rng(901496); - std::uniform_int_distribution value_dist(-3, 3); - const std::vector sizes = { - 1, 2, 17, 495, 496, 497, 1024, 4096, 8191, 8192, 8193, - }; - for (const std::size_t size : sizes) { SCOPED_TRACE(size); std::vector values(size); @@ -1044,7 +622,7 @@ TEST(RmqSegmentBTreeXl, DuplicateHeavyRandomTo8193) { } } - const Rmq rmq{std::span(values)}; + const Rmq rmq(values); std::uniform_int_distribution width_dist(1, size); for (std::size_t query = 0; query < 2000; ++query) { const std::size_t width = width_dist(rng); @@ -1059,502 +637,48 @@ TEST(RmqSegmentBTreeXl, DuplicateHeavyRandomTo8193) { } } -TEST(RmqExperimentalNodeEulerBTree, HighSparseSelectorWithMiddleLevels) { - using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kFanout = Rmq::kFanout; - constexpr std::size_t kMiddleBoundary = kLeaf * kFanout * kFanout; - const std::size_t size = kMiddleBoundary + 2 * kLeaf + 17; - - std::vector values(size); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 53 + i / 17) % 251); - if (i % 257 == 3 || i % 65537 == 11) { - values[i] = -5; - } - } - values[13] = -100; - values[kMiddleBoundary - 29] = -90; - values[kMiddleBoundary + kLeaf + 7] = -110; - - const Rmq rmq{std::span(values)}; - const std::vector> ranges = { - {0, values.size()}, - {kMiddleBoundary - 2 * kLeaf, kMiddleBoundary + 2 * kLeaf}, - {kMiddleBoundary - 31, kMiddleBoundary + kLeaf + 31}, - {kMiddleBoundary + 1, values.size()}, - {kLeaf * (kFanout - 1) + 9, kMiddleBoundary + kLeaf + 13}, - {values.size() - 3 * kLeaf, values.size()}, - }; - - for (const auto [left, right] : ranges) { - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - } -} - -TEST(RmqExperimentalNodeEulerBTree, CopyAndMovePreserveSplitStorage) { - using Rmq = pixie::rmq::experimental::NodeEulerBTreeRmq; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kFanout = Rmq::kFanout; - std::vector values(kLeaf * kFanout + 2 * kLeaf + 9); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 41 + i / 3) % 127); - if (i % 23 == 0) { - values[i] = -8; - } - } - - const Rmq original{std::span(values)}; - Rmq copied(original); - Rmq assigned; - assigned = original; - Rmq moved(std::move(copied)); - Rmq move_assigned; - move_assigned = std::move(assigned); - - const std::vector> ranges = { - {0, 1}, - {0, values.size()}, - {255, 257}, - {kLeaf * kFanout - 5, kLeaf * kFanout + 6}, - {values.size() - 513, values.size() - 3}, - {values.size() - 17, values.size()}, - }; - - const std::array rmqs = {&original, &moved, &move_assigned}; - for (const Rmq* rmq : rmqs) { - ASSERT_EQ(rmq->size(), values.size()); - for (const auto [left, right] : ranges) { - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(rmq->arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - } - } -} - -template -class DepthRmqContractTest : public ::testing::Test {}; - -using DepthRmqCases = - ::testing::Types; -TYPED_TEST_SUITE(DepthRmqContractTest, DepthRmqCases); - -TYPED_TEST(DepthRmqContractTest, EmptyAndSingleDepthInputs) { - using Rmq = typename TypeParam::Rmq; - - const std::vector bits; - const Rmq empty = TypeParam::make(std::span(bits), 0); - EXPECT_TRUE(empty.empty()); - EXPECT_EQ(empty.size(), 0u); - EXPECT_EQ(empty.arg_min(0, 0), Rmq::npos); - - const Rmq single = TypeParam::make(std::span(bits), 1); - EXPECT_FALSE(single.empty()); - EXPECT_EQ(single.size(), 1u); - EXPECT_EQ(single.arg_min(0, 1), 0u); - EXPECT_EQ(single.arg_min(1, 1), Rmq::npos); - EXPECT_EQ(single.arg_min(0, 2), Rmq::npos); -} - -TYPED_TEST(DepthRmqContractTest, ExhaustiveSmallDepthArray) { - const std::vector depths = {0, 1, 0, 1, 2, 1, 2, 1, 0}; - check_depths_all_arg_min(std::span(depths)); -} - -TYPED_TEST(DepthRmqContractTest, CrossBlockRanges) { - constexpr std::size_t kBlock = TypeParam::kBlockSize; - std::vector depths(3 * kBlock + 1); - for (std::size_t i = 1; i < depths.size(); ++i) { - const bool up = (i % 7 == 0) || (i % 7 == 1) || (i % 7 == 4); - depths[i] = depths[i - 1] + (up ? 1 : -1); - } - - check_depths_all_arg_min(std::span(depths)); -} - -TYPED_TEST(DepthRmqContractTest, BoundaryRangesAroundBlocks) { - constexpr std::size_t kBlock = TypeParam::kBlockSize; - std::vector depths(3 * kBlock + 6); - for (std::size_t i = 1; i < depths.size(); ++i) { - depths[i] = - depths[i - 1] + ((i % 5 == 0 || i % 11 == 0 || i % 17 == 0) ? 1 : -1); - } - - const std::vector> ranges = { - {0, 1}, - {kBlock - 2, kBlock}, - {kBlock - 1, kBlock + 1}, - {kBlock, kBlock + 1}, - {kBlock, 2 * kBlock}, - {kBlock + 1, 2 * kBlock}, - {2 * kBlock - 1, 2 * kBlock + 1}, - {0, depths.size()}, - {kBlock - 31, 2 * kBlock + 3}, - {kBlock - 12, 2 * kBlock + 6}, - {2 * kBlock, depths.size()}, - }; - check_depth_case_ranges( - std::span(depths), - std::span>(ranges)); -} - -TYPED_TEST(DepthRmqContractTest, LongSequenceRangesNearEnd) { - std::vector depths(8193); - for (std::size_t i = 1; i < depths.size(); ++i) { - const bool up = (i % 13 == 0) || (i % 17 == 0) || (i % 19 == 0); - depths[i] = depths[i - 1] + (up ? 1 : -1); - } - - const std::vector> ranges = { - {depths.size() - 1, depths.size()}, - {depths.size() - 128, depths.size()}, - {depths.size() - 513, depths.size() - 2}, - {depths.size() - 4097, depths.size() - 6}, - }; - check_depth_case_ranges( - std::span(depths), - std::span>(ranges)); -} - -TYPED_TEST(DepthRmqContractTest, CrossBlockTieKeepsFirstPosition) { - constexpr std::size_t kBlock = TypeParam::kBlockSize; - std::vector depths(3 * kBlock); - for (std::size_t i = 1; i < depths.size(); ++i) { - depths[i] = (i % 2 == 0) ? 0 : 1; - } - - const std::vector bits = pack_depth_deltas(depths); - const typename TypeParam::Rmq rmq = - make_depth_rmq(bits, depths.size()); - const std::size_t left = kBlock - 8; - const std::size_t right = 2 * kBlock + 5; - - EXPECT_EQ(rmq.arg_min(left, right), left); - EXPECT_EQ(rmq.arg_min(left, right), - naive_arg_min(std::span(depths), left, right, - std::less())); -} - -TYPED_TEST(DepthRmqContractTest, DisjointBoundaryRangeMatchesNaive) { - constexpr std::size_t kBlock = TypeParam::kBlockSize; - std::vector depths(3 * kBlock + 7); - for (std::size_t i = 1; i < depths.size(); ++i) { - const bool up = (i % 9 == 0) || (i % 9 == 3) || (i % 11 == 0); - depths[i] = depths[i - 1] + (up ? 1 : -1); - } - - const std::vector> ranges = { - {kBlock - 32, kBlock + 33}, {kBlock - 8, kBlock + 13}, - {kBlock - 1, kBlock + 2}, {2 * kBlock - 10, 2 * kBlock + 5}, - {2 * kBlock - 1, 2 * kBlock + 4}, - }; - check_depth_case_ranges( - std::span(depths), - std::span>(ranges)); -} - -TYPED_TEST(DepthRmqContractTest, RejectsTooSmallBitSpan) { - const std::vector bits; - EXPECT_THROW( - { - const typename TypeParam::Rmq rmq = - TypeParam::make(std::span(bits), 2); - (void)rmq; - }, - std::invalid_argument); -} - -TYPED_TEST(DepthRmqContractTest, DifferentialRandomWalks) { - std::mt19937_64 rng(77); - for (std::size_t size = 1; size <= 257; size += 17) { - std::vector depths(size); - for (std::size_t i = 1; i < depths.size(); ++i) { - depths[i] = depths[i - 1] + ((rng() & 1u) != 0 ? 1 : -1); - } - - check_depths_all_arg_min(std::span(depths)); - } -} - -TEST(RmqSegmentBTreeXLPlusMinusOne, DefaultBoundaryRanges) { - using Rmq = pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq<>; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kHighFanout = Rmq::kHighLevelFanout; - constexpr std::size_t kHighBoundary = kLeaf * kHighFanout; - const std::vector sizes = { - kLeaf - 1, kLeaf, kLeaf + 1, - kHighBoundary - 1, kHighBoundary, kHighBoundary + 1, - }; - - for (const std::size_t size : sizes) { - SCOPED_TRACE(size); - std::vector depths(size); - for (std::size_t i = 1; i < depths.size(); ++i) { - const bool up = (i % 7 == 0) || (i % 11 == 3) || (i % 19 == 5); - depths[i] = depths[i - 1] + (up ? 1 : -1); - } - - const std::vector bits = pack_depth_deltas(depths); - const Rmq rmq(bits, depths.size()); - const auto check_range = [&](std::size_t left, std::size_t right) { - ASSERT_LT(left, right); - ASSERT_LE(right, depths.size()); - EXPECT_EQ(rmq.arg_min(left, right), - naive_arg_min(std::span(depths), left, - right, std::less())) - << "range=[" << left << "," << right << ")"; - }; +TEST(RmqCartesianHybridBTree, BpStorageIsCacheLineAligned) { + using Rmq = pixie::rmq::CartesianHybridBTree; + constexpr std::size_t kLeafWords = 512 / 64; + const auto expect_aligned = [](const Rmq& rmq) { + const std::span words = rmq.bp_words(); + ASSERT_FALSE(words.empty()); + const auto base = reinterpret_cast( + static_cast(words.data())); + EXPECT_EQ(base % pixie::kAlignedStorageLineBytes, 0u); - check_range(0, 1); - check_range(0, size); - check_range(size - 1, size); - for (const std::size_t boundary : {kLeaf, kHighBoundary}) { - if (boundary >= size) { - continue; - } - check_range(boundary - 1, boundary + 1); - check_range(boundary > 97 ? boundary - 97 : 0, - std::min(size, boundary + 101)); - check_range(boundary, size); + for (std::size_t word = 0; word < words.size(); word += kLeafWords) { + EXPECT_EQ((base + word * sizeof(std::uint64_t)) % + pixie::kAlignedStorageLineBytes, + 0u) + << "leaf_start_word=" << word; } - } -} - -TEST(RmqSegmentBTreeXLPlusMinusOne, Select0MatchesNaiveAcrossHighNodes) { - using Rmq = pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq<>; - constexpr std::size_t kLeaf = Rmq::kLeafSize; - constexpr std::size_t kHighFanout = Rmq::kHighLevelFanout; - std::vector depths(kLeaf * kHighFanout + 777); - for (std::size_t i = 1; i < depths.size(); ++i) { - const bool up = (i % 5 == 0) || (i % 17 == 3) || (i % 257 == 11); - depths[i] = depths[i - 1] + (up ? 1 : -1); - } - - const std::vector bits = pack_depth_deltas(depths); - const Rmq rmq(bits, depths.size()); - const std::size_t bit_count = depths.size() - 1; - std::size_t zero_count = 0; - for (std::size_t position = 0; position < bit_count; ++position) { - zero_count += packed_bit(bits, position) ? 0 : 1; - } - - const std::vector ranks = { - 1, 2, 63, 64, 65, zero_count / 4, zero_count / 2, zero_count - 1, - zero_count, - }; - for (const std::size_t rank : ranks) { - ASSERT_GT(rank, 0u); - ASSERT_LE(rank, zero_count); - EXPECT_EQ( - rmq.select0(rank), - naive_select0(std::span(bits), bit_count, rank)) - << "rank=" << rank; - } - EXPECT_EQ(rmq.select0(0), Rmq::npos); - EXPECT_EQ(rmq.select0(zero_count + 1), Rmq::npos); -} - -TEST(RmqSegmentBTreeXLPlusMinusOne, FixedFanoutsExerciseHighBoundary) { - using Rmq = - pixie::rmq::experimental::SegmentBTreeXLPlusMinusOneRmq; - static_assert(Rmq::kMiddleFanout == 192); - static_assert(Rmq::kHighLevelFanout == 256); - constexpr std::size_t kHighBoundary = Rmq::kLeafSize * Rmq::kHighLevelFanout; - std::vector depths(kHighBoundary + 3 * Rmq::kLeafSize + 17); - for (std::size_t i = 1; i < depths.size(); ++i) { - const bool up = (i % 13 == 0) || (i % 17 == 1) || (i % 29 == 7); - depths[i] = depths[i - 1] + (up ? 1 : -1); - } - - const std::vector bits = pack_depth_deltas(depths); - const Rmq rmq(bits, depths.size()); - const std::vector> ranges = { - {0, depths.size()}, - {kHighBoundary - 2, kHighBoundary + 2}, - {kHighBoundary - Rmq::kLeafSize - 3, kHighBoundary + Rmq::kLeafSize + 5}, - {Rmq::kLeafSize, kHighBoundary + 17}, - {kHighBoundary + 1, depths.size()}, - {depths.size() - Rmq::kLeafSize, depths.size()}, - }; - check_depth_ranges( - rmq, std::span(depths), - std::span>(ranges)); -} - -TEST(RmqOneIntervalBTree, FusedBoundaryTiesKeepFirstPosition) { - constexpr std::size_t kBlock = pixie::rmq::OneIntervalBTreeRmq<>::kBlockSize; - std::vector depths(4 * kBlock + 9); - for (std::size_t i = 0; i < depths.size(); ++i) { - depths[i] = (i & 1u) == 0 ? 0 : 1; - } - - const std::vector> ranges = { - {kBlock - 8, 2 * kBlock + 8}, - {kBlock - 1, 3 * kBlock + 4}, - {kBlock + 3, 4 * kBlock + 2}, - {2 * kBlock - 5, 4 * kBlock + 6}, }; - check_one_interval_btree_ranges( - std::span(depths), - std::span>(ranges)); -} - -TEST(RmqOneIntervalBTreeBoundaryRecords, CrossBlockAndTailRanges) { - using Rmq = pixie::rmq::OneIntervalBTreeRmqBoundaryRecords<>; - constexpr std::size_t kBlock = Rmq::kBlockSize; - std::vector depths(2 * kBlock + 17); - for (std::size_t i = 1; i < depths.size(); ++i) { - const bool up = (i % 6 == 0) || (i % 10 == 1) || (i % 17 == 3); - depths[i] = depths[i - 1] + (up ? 1 : -1); - } - - const std::vector bits = pack_depth_deltas(depths); - const Rmq records(bits, depths.size()); - const pixie::rmq::OneIntervalBTreeRmq<> baseline(bits, depths.size()); - const std::vector> ranges = { - {kBlock - 6, kBlock + 1}, {kBlock - 3, 2 * kBlock + 2}, - {kBlock - 7, 2 * kBlock}, {kBlock, 2 * kBlock + 5}, - {kBlock - 5, 2 * kBlock + 6}, {2 * kBlock - 3, depths.size()}, - {0, depths.size()}, - }; - - for (const auto [left, right] : ranges) { - const std::size_t expected = - naive_arg_min(std::span(depths), left, right, - std::less()); - EXPECT_EQ(records.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - EXPECT_EQ(records.arg_min(left, right), baseline.arg_min(left, right)) - << "range=[" << left << "," << right << ")"; - } -} - -TEST(RmqOneIntervalBTree, LowNodeBoundaryRanges) { - constexpr std::size_t kBlock = pixie::rmq::OneIntervalBTreeRmq<>::kBlockSize; - constexpr std::size_t kLowBoundary = - kBlock * pixie::rmq::OneIntervalBTreeRmq<>::kLowFanout; - std::vector depths(kLowBoundary + 4 * kBlock + 1); - for (std::size_t i = 1; i < depths.size(); ++i) { - depths[i] = - depths[i - 1] + ((i % 7 == 0 || i % 19 == 0 || i % 23 == 0) ? 1 : -1); - } - - const std::vector> ranges = { - {kLowBoundary - 2, kLowBoundary + 2}, - {kLowBoundary - kBlock - 1, kLowBoundary + kBlock + 1}, - {1, kLowBoundary + 1}, - {kBlock, kLowBoundary + kBlock}, - {kLowBoundary, depths.size()}, - {0, depths.size()}, - }; - check_one_interval_btree_ranges( - std::span(depths), - std::span>(ranges)); -} - -TEST(RmqOneIntervalBTree, HighNodeBoundaryRanges) { - constexpr std::size_t kBlock = pixie::rmq::OneIntervalBTreeRmq<>::kBlockSize; - constexpr std::size_t kHighBoundary = - kBlock * pixie::rmq::OneIntervalBTreeRmq<>::kLowFanout * - pixie::rmq::OneIntervalBTreeRmq<>::kHighFanout; - std::vector depths(kHighBoundary + 32 * kBlock + 1); - for (std::size_t i = 1; i < depths.size(); ++i) { - depths[i] = - depths[i - 1] + ((i % 13 == 0 || i % 29 == 0 || i % 31 == 0) ? 1 : -1); - } - - const std::vector> ranges = { - {kHighBoundary - 2, kHighBoundary + 2}, - {kHighBoundary - 16 * kBlock - 1, kHighBoundary + 16 * kBlock + 1}, - {kBlock, kHighBoundary + kBlock}, - {kHighBoundary, depths.size()}, - {0, depths.size()}, - }; - check_one_interval_btree_ranges( - std::span(depths), - std::span>(ranges)); -} - -TEST(RmqCartesianTreeSegmentBTreeXL, BoundarySizesAndBpEncoding) { - using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq; - const std::vector sizes = { - 1u, 255u, 256u, 257u, 511u, 512u, 513u, 8191u, 8192u, 8193u, - }; - - for (const std::size_t size : sizes) { - SCOPED_TRACE(size); - std::vector values(size); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 37 + i / 5) % 53); - if (i % 17 == 0 || i % 257 == 3) { - values[i] = -4; - } - } - - const Rmq rmq(values); - expect_valid_bp_encoding(rmq, values.size()); - - const std::vector> ranges = { - {0, 1}, - {0, size}, - {size - 1, size}, - {size / 3, std::min(size, size / 3 + 19)}, - {size / 2, std::min(size, size / 2 + 37)}, - }; - for (const auto [left, right] : ranges) { - ASSERT_LT(left, right); - EXPECT_EQ(rmq.arg_min(left, right), - naive_arg_min(std::span(values), left, right, - std::less())) - << "range=[" << left << "," << right << ")"; - } - } -} -TEST(RmqCartesianTreeSegmentBTreeXL, DuplicateHeavyRandomDifferentialTo8193) { - using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq; - std::mt19937_64 rng(20260610); - std::uniform_int_distribution value_dist(-3, 3); const std::vector sizes = { - 1, 2, 17, 255, 256, 257, 1024, 4096, 8191, 8192, 8193, + 1, 255, 256, 257, 4096, 8193, }; for (const std::size_t size : sizes) { SCOPED_TRACE(size); std::vector values(size); for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = value_dist(rng); - if ((i % 11) < 6) { - values[i] = 0; - } + values[i] = static_cast((i * 31 + i / 3) % 97); } const Rmq rmq(values); - std::uniform_int_distribution width_dist(1, size); - for (std::size_t query = 0; query < 2000; ++query) { - const std::size_t width = width_dist(rng); - std::uniform_int_distribution left_dist(0, size - width); - const std::size_t left = left_dist(rng); - const std::size_t right = left + width; - const std::size_t expected = naive_arg_min(std::span(values), - left, right, std::less()); - EXPECT_EQ(rmq.arg_min(left, right), expected) - << "range=[" << left << "," << right << ")"; - } + Rmq copied(rmq); + Rmq assigned; + assigned = copied; + Rmq moved(std::move(copied)); + expect_aligned(rmq); + expect_aligned(assigned); + expect_aligned(moved); } } -TEST(RmqCartesianTreeSegmentBTreeXL, TopSparseOverlayBoundaryRanges) { - using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq; +TEST(RmqCartesianHybridBTree, TopSparseOverlayBoundaryRanges) { + using Rmq = pixie::rmq::CartesianHybridBTree; constexpr std::size_t kTopBlock = 4096; std::vector values(3 * kTopBlock + 77, 1000); for (std::size_t i = 0; i < values.size(); ++i) { @@ -1594,8 +718,8 @@ TEST(RmqCartesianTreeSegmentBTreeXL, TopSparseOverlayBoundaryRanges) { } } -TEST(RmqCartesianTreeSegmentBTreeXL, TopSparseOverlayCapsBlockCount) { - using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq; +TEST(RmqCartesianHybridBTree, TopSparseOverlayCapsBlockCount) { + using Rmq = pixie::rmq::CartesianHybridBTree; constexpr std::size_t kMinBlock = Rmq::kMinTopSparseBlockSize; constexpr std::size_t kMaxBlocks = Rmq::kMaxTopSparseBlocks; constexpr std::size_t kLargestFixedBlockInput = kMinBlock * kMaxBlocks; @@ -1621,9 +745,8 @@ TEST(RmqCartesianTreeSegmentBTreeXL, TopSparseOverlayCapsBlockCount) { EXPECT_EQ(rmq.top_sparse_block_count(), 4u); } -TEST(RmqCartesianTreeSegmentBTreeXL, TopSparseOverlayComparatorMaximum) { - using Rmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq< - int, std::greater>; +TEST(RmqCartesianHybridBTree, TopSparseOverlayComparatorMaximum) { + using Rmq = pixie::rmq::CartesianHybridBTree>; constexpr std::size_t kTopBlock = 4096; std::vector values(2 * kTopBlock + 33, -1000); for (std::size_t i = 0; i < values.size(); ++i) { @@ -1650,9 +773,8 @@ TEST(RmqCartesianTreeSegmentBTreeXL, TopSparseOverlayComparatorMaximum) { } } -TEST(RmqCartesianTreeSegmentBTreeXL, ComparatorCopyAndMove) { - using MaxRmq = pixie::rmq::experimental::CartesianTreeSegmentBTreeXLRmq< - int, std::greater>; +TEST(RmqCartesianHybridBTree, ComparatorCopyAndMove) { + using MaxRmq = pixie::rmq::CartesianHybridBTree>; const std::vector values = {1, 8, 8, 7, 8, 3, 8, 4, 4, 8}; const MaxRmq original(values); MaxRmq copied(original); @@ -1665,87 +787,3 @@ TEST(RmqCartesianTreeSegmentBTreeXL, ComparatorCopyAndMove) { check_all_ranges(moved, std::span(values), std::greater()); EXPECT_EQ(original.arg_min(0, values.size()), 1u); } - -TEST(RmqCartesianTree, DuplicateHeavyArrays) { - const std::vector> cases = { - {5, 5, 5, 5, 5, 5, 5}, - {4, 1, 4, 1, 4, 1, 4, 1}, - {3, 2, 2, 2, 1, 1, 2, 1, 3}, - {9, 8, 9, 8, 9, 7, 7, 7, 8, 9}, - }; - - for (std::size_t case_index = 0; case_index < cases.size(); ++case_index) { - SCOPED_TRACE(case_index); - const std::vector& values = cases[case_index]; - const pixie::rmq::CartesianTreeRmq cartesian(values); - expect_valid_bp_encoding(cartesian, values.size()); - check_all_ranges(cartesian, std::span(values), std::less()); - } -} - -TEST(RmqCartesianTree, ComparatorTiesKeepFirstMaximum) { - const std::vector values = {1, 8, 8, 7, 8, 3, 8}; - const pixie::rmq::CartesianTreeRmq> cartesian(values); - - expect_valid_bp_encoding(cartesian, values.size()); - check_all_ranges(cartesian, std::span(values), - std::greater()); - EXPECT_EQ(cartesian.arg_min(0, values.size()), 1u); - EXPECT_EQ(cartesian.arg_min(2, values.size()), 2u); -} - -TEST(RmqCartesianTree, CopyAndMoveRebuildInternalSpans) { - const std::vector values = {5, 4, 3, 2, 1, 2, 3}; - const pixie::rmq::CartesianTreeRmq original(values); - pixie::rmq::CartesianTreeRmq copied(original); - pixie::rmq::CartesianTreeRmq assigned; - assigned = copied; - pixie::rmq::CartesianTreeRmq moved(std::move(copied)); - - check_all_ranges(original, std::span(values), std::less()); - check_all_ranges(assigned, std::span(values), std::less()); - check_all_ranges(moved, std::span(values), std::less()); -} - -TEST(RmqCartesianTree, BpEncodingIsBalanced) { - const std::vector values = {4, 1, 3, 2, 5}; - const pixie::rmq::CartesianTreeRmq rmq(values); - expect_valid_bp_encoding(rmq, values.size()); -} - -TEST(RmqCartesianTree, BoundarySizesExerciseBpBlocks) { - for (std::size_t size : {1u, 63u, 64u, 65u, 127u, 128u, 129u, 255u, 256u, - 257u, 511u, 512u, 513u}) { - SCOPED_TRACE(size); - std::vector values(size); - for (std::size_t i = 0; i < values.size(); ++i) { - values[i] = static_cast((i * 37 + i / 3) % 23); - if (i % 11 == 0) { - values[i] = 7; - } - } - - const pixie::rmq::CartesianTreeRmq rmq(values); - expect_valid_bp_encoding(rmq, values.size()); - - const auto check_range = [&](std::size_t left, std::size_t right) { - EXPECT_EQ(rmq.arg_min(left, right), - naive_arg_min(std::span(values), left, right, - std::less())) - << "range=[" << left << "," << right << ")"; - }; - - check_range(0, 1); - check_range(0, size); - check_range(size - 1, size); - check_range(size / 3, std::min(size, size / 3 + 9)); - check_range(size / 2, std::min(size, size / 2 + 17)); - - for (std::size_t boundary : {64u, 128u, 256u, 512u}) { - if (boundary < size) { - check_range(boundary - 1, std::min(size, boundary + 2)); - check_range(boundary / 2, std::min(size, boundary + 1)); - } - } - } -} diff --git a/src/tests/test_rmm.cpp b/src/tests/test_rmm.cpp index 6955634..52fa139 100644 --- a/src/tests/test_rmm.cpp +++ b/src/tests/test_rmm.cpp @@ -1084,6 +1084,22 @@ TEST(RmMBTreeExperimental, RankSelectIgnoresDirtyTrailingStorage) { EXPECT_EQ(rm.select0(2), pixie::experimental::RmMBTree<>::npos); } +TEST(RmMBTreeExperimental, OptionalSelectSupport) { + const std::string bits = "101100"; + auto words = pack_words_lsb_first(bits); + pixie::experimental::RmMBTree<> select0_only( + std::span(words), bits.size(), + pixie::BitVector::SelectSupport::kSelect0, /*one_count=*/3); + + EXPECT_EQ(select0_only.rank1(bits.size()), 3u); + EXPECT_EQ(select0_only.rank0(bits.size()), 3u); + EXPECT_EQ(select0_only.select1(1), pixie::experimental::RmMBTree<>::npos); + EXPECT_EQ(select0_only.select0(1), 1u); + EXPECT_EQ(select0_only.select0(2), 4u); + EXPECT_EQ(select0_only.select0(3), 5u); + EXPECT_EQ(select0_only.select0(4), pixie::experimental::RmMBTree<>::npos); +} + TEST(RmMBTreeExperimental, ParenthesesOnUnmatchedBoundaryBits) { const std::string bits = "1"; auto words = pack_words_lsb_first(bits); diff --git a/src/tests/unittests.cpp b/src/tests/unittests.cpp index 1c435d3..935b972 100644 --- a/src/tests/unittests.cpp +++ b/src/tests/unittests.cpp @@ -4,6 +4,7 @@ #include #include +#include using pixie::BitVector; using pixie::BitVectorInterleaved; @@ -367,6 +368,72 @@ TEST(BitVectorTest, ExactShortSpanRankSelect) { EXPECT_EQ(bv.select0(9), bv.size()); } +TEST(BitVectorTest, OptionalSelectSupport) { + std::vector bits = {0b1100010110010110}; + + BitVector select0_only(bits, 16, BitVector::SelectSupport::kSelect0); + EXPECT_FALSE(select0_only.supports_select1()); + EXPECT_TRUE(select0_only.supports_select0()); + EXPECT_EQ(select0_only.rank(16), 8); + EXPECT_EQ(select0_only.rank0(16), 8); + EXPECT_EQ(select0_only.select(1), select0_only.size()); + EXPECT_EQ(select0_only.select0(1), 0); + EXPECT_EQ(select0_only.select0(8), 13); + + BitVector hinted_both(bits, 16, BitVector::SelectSupport::kBoth, 8); + EXPECT_TRUE(hinted_both.supports_select1()); + EXPECT_TRUE(hinted_both.supports_select0()); + EXPECT_EQ(hinted_both.select(1), 1); + EXPECT_EQ(hinted_both.select(8), 15); + EXPECT_EQ(hinted_both.select0(1), 0); + EXPECT_EQ(hinted_both.select0(8), 13); + EXPECT_THROW((BitVector(bits, 16, BitVector::SelectSupport::kBoth, 17)), + std::invalid_argument); + + BitVector select1_only(bits, 16, BitVector::SelectSupport::kSelect1); + EXPECT_TRUE(select1_only.supports_select1()); + EXPECT_FALSE(select1_only.supports_select0()); + EXPECT_EQ(select1_only.rank(16), 8); + EXPECT_EQ(select1_only.rank0(16), 8); + EXPECT_EQ(select1_only.select(1), 1); + EXPECT_EQ(select1_only.select(8), 15); + EXPECT_EQ(select1_only.select0(1), select1_only.size()); + + BitVector rank_only(bits, 16, BitVector::SelectSupport::kNone); + EXPECT_FALSE(rank_only.supports_select1()); + EXPECT_FALSE(rank_only.supports_select0()); + EXPECT_EQ(rank_only.rank(16), 8); + EXPECT_EQ(rank_only.rank0(16), 8); + EXPECT_EQ(rank_only.select(0), 0); + EXPECT_EQ(rank_only.select0(0), 0); + EXPECT_EQ(rank_only.select(1), rank_only.size()); + EXPECT_EQ(rank_only.select0(1), rank_only.size()); + + std::vector zero_words(512, 0); + BitVector large_select0_only(zero_words, zero_words.size() * 64, + BitVector::SelectSupport::kSelect0); + EXPECT_EQ(large_select0_only.select(1), large_select0_only.size()); + EXPECT_EQ(large_select0_only.select0(1), 0); + EXPECT_EQ(large_select0_only.select0(16384), 16383); + EXPECT_EQ(large_select0_only.select0(16385), 16384); + EXPECT_EQ(large_select0_only.select0(32768), 32767); + + BitVector large_hinted_select0(zero_words, zero_words.size() * 64, + BitVector::SelectSupport::kSelect0, 0); + EXPECT_EQ(large_hinted_select0.select(1), large_hinted_select0.size()); + EXPECT_EQ(large_hinted_select0.select0(1), 0); + EXPECT_EQ(large_hinted_select0.select0(32768), 32767); + + std::vector one_words(512, ~uint64_t{0}); + BitVector large_select1_only(one_words, one_words.size() * 64, + BitVector::SelectSupport::kSelect1); + EXPECT_EQ(large_select1_only.select0(1), large_select1_only.size()); + EXPECT_EQ(large_select1_only.select(1), 0); + EXPECT_EQ(large_select1_only.select(16384), 16383); + EXPECT_EQ(large_select1_only.select(16385), 16384); + EXPECT_EQ(large_select1_only.select(32768), 32767); +} + TEST(BitVectorTest, ShortSpanUsesScalarRankSelectFallbacks) { std::vector one_in_second_word = {0, 1}; BitVector ones(std::span(one_in_second_word), 65); From c134c0df5c30147056a5b0bfff1412b3c98c65d7 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Mon, 15 Jun 2026 14:41:51 +0300 Subject: [PATCH 19/27] rm obsolette bench report --- include/pixie/experimental/rmm_btree.h | 31 -------------------------- 1 file changed, 31 deletions(-) diff --git a/include/pixie/experimental/rmm_btree.h b/include/pixie/experimental/rmm_btree.h index e44a7e4..3009ee8 100644 --- a/include/pixie/experimental/rmm_btree.h +++ b/include/pixie/experimental/rmm_btree.h @@ -1,36 +1,5 @@ #pragma once -/** - * RmMBTree construction-summary experiment, 2026-06-13. - * - * Change under test: full aligned 512-bit block summaries use existing 128-bit - * excess kernels from bits.h instead of scanning every bit. - * - * Command shape: - * taskset -c 0 ./build/release/bench_rmq - * --benchmark_filter='^rmq_build_(cartesian_rmm|cartesian_hybrid_btree|sdsl_sct)/(4194304|67108864)$' - * --benchmark_repetitions=5 - * - * CPU mean, milliseconds. - * - * | N | row | CPU ms | - * | ---: | ----------------------- | ------: | - * | 2^22 | CartesianRmM before | 54.407 | - * | 2^22 | CartesianRmM after | 44.731 | - * | 2^22 | CartesianHybrid control | 47.943 | - * | 2^22 | SdslSct control | 39.911 | - * | ---- | ----------------------- | ------- | - * | 2^26 | CartesianRmM before | 915.712 | - * | 2^26 | CartesianRmM after | 750.316 | - * | 2^26 | CartesianHybrid control | 772.157 | - * | 2^26 | SdslSct control | 642.540 | - * - * Perf profile note: before this change, RmMBTree::summarize_bits()/bit() - * accounted for about 14% of CartesianRmM build samples. After this change, - * those source lines disappear from the visible hotspot list; the replacement - * excess_min_128() work is about 3%. - */ - #include #include #include From f499afec2a37ecfef14703598e16cdf68b2a809b Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Mon, 15 Jun 2026 15:50:49 +0300 Subject: [PATCH 20/27] Update agentic --- agentic/cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agentic/cpp b/agentic/cpp index 4fa94cd..12a959e 160000 --- a/agentic/cpp +++ b/agentic/cpp @@ -1 +1 @@ -Subproject commit 4fa94cdcfbd0ddbcd77492324b45fcb82e4f85bd +Subproject commit 12a959e4883e453c72bcb5f5c4f5dfef0a9ab516 From caed4ffcda5a0b45c1cb1377015090c43868c7ff Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 20 Jun 2026 15:58:29 +0300 Subject: [PATCH 21/27] Finalization --- include/pixie/bits.h | 235 ++++++---- include/pixie/experimental/excess.h | 403 +++++++++++++++--- include/pixie/rmq/cartesian_hybrid_btree.h | 8 +- include/pixie/rmq/cartesian_rmm.h | 12 +- include/pixie/rmq/sdsl_sct.h | 6 +- scripts/coverage_report.sh | 5 + .../excess_positions_benchmarks.cpp | 18 + src/tests/excess_positions_tests.cpp | 63 +++ src/tests/rmq_tests.cpp | 53 ++- 9 files changed, 621 insertions(+), 182 deletions(-) diff --git a/include/pixie/bits.h b/include/pixie/bits.h index f930842..4fb4f57 100644 --- a/include/pixie/bits.h +++ b/include/pixie/bits.h @@ -83,6 +83,16 @@ static inline const __m128i excess_lut_nibble_index_sse = _mm_setr_epi8( 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); +static inline const __m128i excess_lut_low_nibble_index_sse = _mm_setr_epi8( + 0, 2, 4, 6, + 8, 10, 12, 14, + 16, 18, 20, 22, + 24, 26, 28, 30); +static inline const __m128i excess_lut_high_nibble_index_sse = _mm_setr_epi8( + 1, 3, 5, 7, + 9, 11, 13, 15, + 17, 19, 21, 23, + 25, 27, 29, 31); static inline const __m128i excess_lut_nibble_mask_sse = _mm_set1_epi8(0x0F); // clang-format on #endif @@ -96,6 +106,26 @@ static inline __m128i excess_nibbles_64_sse(const uint64_t* s) noexcept { _mm_and_si128(_mm_srli_epi16(word_vec, 4), excess_lut_nibble_mask_sse); return _mm_unpacklo_epi8(lo_nibbles, hi_nibbles); } + +static inline __m128i excess_prefix_sum_16x_i8(__m128i v) noexcept { + __m128i x = v; + __m128i t = _mm_slli_si128(x, 1); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 2); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 4); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 8); + return _mm_add_epi8(x, t); +} + +static inline int excess_horizontal_min_i8(__m128i v) noexcept { + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 8)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 4)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 2)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 1)); + return static_cast(static_cast(_mm_extract_epi8(v, 0))); +} #endif /** @@ -972,6 +1002,20 @@ constexpr int8_t excess_byte_min_prefix_offset_value(uint8_t x) { return static_cast(best_offset); } +constexpr int8_t excess_nibble_min_prefix_offset_value(uint8_t x, int bits) { + int cur = 0; + int best = 0; + int best_offset = 1; + for (int bit = 0; bit < bits; ++bit) { + cur += ((x >> bit) & 1u) != 0 ? 1 : -1; + if (bit == 0 || cur < best) { + best = cur; + best_offset = bit + 1; + } + } + return static_cast(best_offset); +} + template constexpr std::array excess_make_byte_lut(Fn fn) { std::array out{}; @@ -989,6 +1033,17 @@ static inline constexpr std::array excess_byte_min_lut = static inline constexpr std::array excess_byte_min_offset_lut = excess_make_byte_lut( [](uint8_t x) { return excess_byte_min_prefix_offset_value(x); }); +static inline constexpr std::array, 4> + excess_partial_nibble_min_offset_lut = [] { + std::array, 4> out{}; + for (size_t width = 1; width < out.size(); ++width) { + for (size_t nibble = 0; nibble < out[width].size(); ++nibble) { + out[width][nibble] = excess_nibble_min_prefix_offset_value( + static_cast(nibble), static_cast(width)); + } + } + return out; + }(); /** * @brief Compute record-low mask for a single byte relative to a threshold. @@ -1395,106 +1450,103 @@ static inline ExcessResult excess_min_128(const uint64_t* s, } } - const size_t first_full_nibble = bit >> 2; + const size_t first_nibble = bit >> 2; const size_t last_full_nibble = right >> 2; const size_t right_partial_width = bit < right ? (right & 3u) : 0; const size_t end_nibble = last_full_nibble + (right_partial_width == 0 ? 0 : 1); - if (first_full_nibble < end_nibble) { - const __m256i nibbles = excess_nibbles_128_avx2(s); - - __m256i ps = _mm256_shuffle_epi8(excess_lut_delta, nibbles); - ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 1)); - ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 2)); - ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 4)); - ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 8)); - - __m128i ps_lo = _mm256_castsi256_si128(ps); - __m128i ps_hi = _mm256_extracti128_si256(ps, 1); - __m128i carry = - _mm_set1_epi8(static_cast(_mm_extract_epi8(ps_lo, 15))); - ps_hi = _mm_add_epi8(ps_hi, carry); - ps = _mm256_inserti128_si256(_mm256_castsi128_si256(ps_lo), ps_hi, 1); - - __m256i b = _mm256_permute2x128_si256(ps, ps, 0x08); - const __m256i excl_ps = _mm256_alignr_epi8(ps, b, 15); - __m256i local_min = _mm256_shuffle_epi8(excess_lut_min, nibbles); + if (first_nibble < end_nibble) { + const __m128i bytes = _mm_loadu_si128(reinterpret_cast(s)); + const __m128i lo_nibbles = _mm_and_si128(bytes, excess_lut_nibble_mask_sse); + const __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(bytes, 4), excess_lut_nibble_mask_sse); + const __m128i lo_delta = _mm_shuffle_epi8(excess_lut_delta_sse, lo_nibbles); + const __m128i hi_delta = _mm_shuffle_epi8(excess_lut_delta_sse, hi_nibbles); + const __m128i byte_delta = _mm_add_epi8(lo_delta, hi_delta); + const __m128i byte_prefix = excess_prefix_sum_16x_i8(byte_delta); + const __m128i byte_prefix_before = _mm_slli_si128(byte_prefix, 1); + + __m128i lo_local_min = _mm_shuffle_epi8(excess_lut_min_sse, lo_nibbles); + __m128i hi_local_min = _mm_shuffle_epi8(excess_lut_min_sse, hi_nibbles); + + const __m128i byte_index = excess_lut_nibble_index_sse; if (right_partial_width != 0) { - __m256i partial_min = _mm256_shuffle_epi8(excess_lut_pos0, nibbles); + const bool partial_is_high = (last_full_nibble & 1u) != 0; + const size_t partial_byte = last_full_nibble >> 1; + const __m128i partial_source = partial_is_high ? hi_nibbles : lo_nibbles; + __m128i partial_min = + _mm_shuffle_epi8(excess_lut_pos0_sse, partial_source); if (right_partial_width >= 2) { - partial_min = _mm256_min_epi8( - partial_min, _mm256_shuffle_epi8(excess_lut_pos1, nibbles)); + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos1_sse, partial_source)); } if (right_partial_width >= 3) { - partial_min = _mm256_min_epi8( - partial_min, _mm256_shuffle_epi8(excess_lut_pos2, nibbles)); + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos2_sse, partial_source)); + } + const __m128i partial_lane = _mm_cmpeq_epi8( + byte_index, _mm_set1_epi8(static_cast(partial_byte))); + if (partial_is_high) { + hi_local_min = _mm_blendv_epi8(hi_local_min, partial_min, partial_lane); + } else { + lo_local_min = _mm_blendv_epi8(lo_local_min, partial_min, partial_lane); } - local_min = _mm256_blendv_epi8( - local_min, partial_min, - _mm256_cmpeq_epi8( - excess_lut_nibble_index, - _mm256_set1_epi8(static_cast(last_full_nibble)))); } - const __m256i partial_candidates = _mm256_add_epi8(excl_ps, local_min); - const __m256i idx = excess_lut_nibble_index; - const int first_minus_one_value = static_cast(first_full_nibble) - 1; - const __m256i first_minus_one = - _mm256_set1_epi8(static_cast(first_minus_one_value)); - const __m256i last = _mm256_set1_epi8(static_cast(end_nibble)); - const __m256i active = _mm256_and_si256( - _mm256_cmpgt_epi8(idx, first_minus_one), _mm256_cmpgt_epi8(last, idx)); - const __m256i masked_candidates = - _mm256_blendv_epi8(_mm256_set1_epi8(127), partial_candidates, active); - - __m128i min128 = - _mm_min_epi8(_mm256_castsi256_si128(masked_candidates), - _mm256_extracti128_si256(masked_candidates, 1)); - min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 8)); - min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 4)); - min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 2)); - min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 1)); + const __m128i lo_candidates = + _mm_add_epi8(byte_prefix_before, lo_local_min); + const __m128i hi_candidates = + _mm_add_epi8(_mm_add_epi8(byte_prefix_before, lo_delta), hi_local_min); + + __m128i masked_lo = lo_candidates; + __m128i masked_hi = hi_candidates; + if (first_nibble != 0 || end_nibble != 32) { + const __m128i first_minus_one = _mm_set1_epi8( + static_cast(static_cast(first_nibble) - 1)); + const __m128i last = _mm_set1_epi8(static_cast(end_nibble)); + const __m128i lo_active = _mm_and_si128( + _mm_cmpgt_epi8(excess_lut_low_nibble_index_sse, first_minus_one), + _mm_cmpgt_epi8(last, excess_lut_low_nibble_index_sse)); + const __m128i hi_active = _mm_and_si128( + _mm_cmpgt_epi8(excess_lut_high_nibble_index_sse, first_minus_one), + _mm_cmpgt_epi8(last, excess_lut_high_nibble_index_sse)); + masked_lo = _mm_blendv_epi8(_mm_set1_epi8(127), lo_candidates, lo_active); + masked_hi = _mm_blendv_epi8(_mm_set1_epi8(127), hi_candidates, hi_active); + } const int candidate_min = - static_cast(static_cast(_mm_extract_epi8(min128, 0))); + excess_horizontal_min_i8(_mm_min_epi8(masked_lo, masked_hi)); if (candidate_min < best) { - const __m256i equal_min = _mm256_cmpeq_epi8( - masked_candidates, - _mm256_set1_epi8(static_cast(candidate_min))); - const uint32_t equal_mask = - static_cast(_mm256_movemask_epi8(equal_min)); - const uint32_t nibble_index = std::countr_zero(equal_mask); - const uint64_t word = s[nibble_index >> 4]; - const uint8_t nibble = - static_cast((word >> ((nibble_index & 15u) * 4u)) & 0xFu); + const __m128i min_vec = _mm_set1_epi8(static_cast(candidate_min)); + const uint32_t lo_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(masked_lo, min_vec))); + const uint32_t hi_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(masked_hi, min_vec))); + const uint32_t lo_nibble_index = + lo_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(lo_equal_mask)) * 2u; + const uint32_t hi_nibble_index = + hi_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(hi_equal_mask)) * 2u + + 1u; + const uint32_t nibble_index = std::min(lo_nibble_index, hi_nibble_index); + const uint32_t byte_offset = nibble_index >> 1u; + const uint64_t byte_word = s[byte_offset >> 3u]; + const uint8_t byte = static_cast( + (byte_word >> ((byte_offset & 7u) * 8u)) & 0xFFu); + const uint8_t nibble = (nibble_index & 1u) == 0 + ? static_cast(byte & 0xFu) + : static_cast((byte >> 4u) & 0xFu); + const size_t local_offset = + right_partial_width != 0 && nibble_index == last_full_nibble + ? static_cast( + excess_partial_nibble_min_offset_lut[right_partial_width] + [nibble]) + : static_cast(excess_lut_min_offset[nibble]); best = candidate_min; - if (right_partial_width != 0 && nibble_index == last_full_nibble) { - int local = 0; - int local_best = 0; - size_t local_offset = 1; - for (size_t i = 0; i < right_partial_width; ++i) { - local += ((nibble >> i) & 1u) != 0 ? 1 : -1; - if (i == 0 || local < local_best) { - local_best = local; - local_offset = i + 1; - } - } - best_offset = static_cast(nibble_index) * 4u + local_offset; - } else { - best_offset = static_cast(nibble_index) * 4u + - static_cast(excess_lut_min_offset[nibble]); - } - } - - bit = end_nibble * 4; - } - - for (; bit < right; ++bit) { - current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; - const size_t offset = bit + 1; - if (current < best) { - best = current; - best_offset = offset; + best_offset = static_cast(nibble_index) * 4u + local_offset; } } #else @@ -1884,7 +1936,8 @@ static inline void excess_record_lows_128_byte_lut(const uint64_t* s, int best = 0; for (size_t byte_idx = 0; byte_idx < 16; ++byte_idx) { const size_t bit_base = byte_idx * 8; - const uint8_t byte = static_cast((s[bit_base >> 6] >> (bit_base & 63)) & 0xFFu); + const uint8_t byte = + static_cast((s[bit_base >> 6] >> (bit_base & 63)) & 0xFFu); const int byte_min = excess_byte_min_lut[byte]; if (cur + byte_min < best) { for (size_t i = 0; i < 8; ++i) { @@ -1921,7 +1974,8 @@ static inline void excess_record_lows_128_lut(const uint64_t* s, static_cast((s[bit_base >> 6] >> (bit_base & 63)) & 0xFFu); const int gap = cur - best; const int idx = gap > 7 ? 7 : (gap < 0 ? 0 : gap); - const uint8_t mask = excess_byte_record_lows_lut[byte][static_cast(idx)]; + const uint8_t mask = + excess_byte_record_lows_lut[byte][static_cast(idx)]; if (mask != 0) { // Recompute absolute excesses for masked positions to update cur/best. int local = 0; @@ -1945,7 +1999,8 @@ static inline void excess_record_lows_128_lut(const uint64_t* s, } } if (out_mask != 0) { - const uint64_t word = static_cast(out_mask) << (bit_base & 63); + const uint64_t word = static_cast(out_mask) + << (bit_base & 63); out[bit_base >> 6] |= word; } } @@ -1984,8 +2039,8 @@ static inline void excess_record_lows_128_avx2(const uint64_t* s, const __m256i is_zero = _mm256_cmpeq_epi16(m, zero); const __m256i steps = _mm256_blendv_epi8(pos, neg, is_zero); const __m256i pref_rel = excess_prefix_sum_16x_i16(steps); - const __m256i pref_abs = - _mm256_add_epi16(pref_rel, _mm256_set1_epi16(static_cast(cur))); + const __m256i pref_abs = _mm256_add_epi16( + pref_rel, _mm256_set1_epi16(static_cast(cur))); alignas(32) int16_t vals[16]; _mm256_store_si256(reinterpret_cast<__m256i*>(vals), pref_abs); diff --git a/include/pixie/experimental/excess.h b/include/pixie/experimental/excess.h index d5a0e66..fde9f31 100644 --- a/include/pixie/experimental/excess.h +++ b/include/pixie/experimental/excess.h @@ -8,6 +8,7 @@ #include #include +// clang-format off /** * Benchmark note: * @@ -16,56 +17,37 @@ * pixie/bits.h; a benchmark win here should be treated as a candidate to port, * not evidence that callers use this variant already. * - * Diagnostic run, 2026-05-30: - * taskset -c 0 build/benchmarks-diagnostic_local/excess_positions_benchmarks - * --benchmark_filter='BM_ExcessMin128(_|/|$)' - * --benchmark_repetitions=3 - * --benchmark_perf_counters=CYCLES,INSTRUCTIONS,CACHE-MISSES - * --benchmark_counters_tabular=true + * excess_min_128 method comparison, 2026-06-17: + * taskset -c 0 ./build/release/excess_positions_benchmarks + * --benchmark_filter='^(BM_ExcessMin128/(left:0/right:(128|16|32)|left:1/right:17|left:3/right:35|left:5/right:37|left:56/right:72|left:60/right:68|left:63/right:64|left:17/right:17)|BM_ExcessMin128_(ByteLUT|NibbleLUT|DeinterleavedSSE|Lane64SSE|Split64SSE)/(left:0/right:(128|16|32)|left:1/right:17|left:3/right:35|left:5/right:37|left:56/right:72|left:60/right:68|left:63/right:64|left:17/right:17)|BM_ExcessMin128_RandomRange$|BM_ExcessMin128_(ByteLUT|NibbleLUT|DeinterleavedSSE|Lane64SSE|Split64SSE)_RandomRange$)' + * --benchmark_min_time=0.5s + * --benchmark_repetitions=5 + * --benchmark_report_aggregates_only=true + * + * Tables report median CPU time across 5 repetitions. Each numeric cell is + * nanoseconds per `excess_min_128` call. Columns named `A-B` are the inclusive + * prefix-offset arguments `[left, right]`; `Random` is the benchmark's + * reproducible mixed-range workload. * - * Production excess_min_128: - * range ns cycles instructions cache misses - * 0-128 8.1 35.8 117.9 0.001 - * 0-127 11.6 50.9 174.9 0.001 - * 0-16 3.7 16.4 88.5 0.000 - * 0-32 5.6 24.7 124.8 0.000 - * 0-48 9.1 40.5 122.1 0.000 - * 0-64 8.1 36.2 121.4 0.000 - * 0-31 12.1 54.2 177.6 0.000 - * 1-17 14.4 64.6 213.3 0.000 - * 3-35 13.5 60.1 212.4 0.000 - * 5-37 12.9 57.5 215.3 0.000 - * 32-64 7.4 33.1 153.0 0.000 - * 33-65 14.3 63.6 214.6 0.001 - * 64-96 7.8 34.2 148.9 0.000 - * 61-93 16.0 69.4 218.5 0.002 - * 96-128 6.5 28.1 151.9 0.001 - * 56-72 5.2 22.7 116.5 0.000 - * 60-68 8.5 38.1 148.7 0.000 - * 63-64 3.1 13.3 84.0 0.000 - * 17-17 2.1 9.1 47.0 0.000 - * Random 11.3 49.7 197.0 0.001 - * - * New non-aligned/random range timings, ns: - * range Prod Scalar Nibble Byte Hybrid Expand16 Lane64 Split64 Skip - * 1-17 14.4 14.6 10.1 12.5 11.9 30.3 13.3 15.9 16.3 - * 3-35 13.5 27.1 12.5 14.2 16.1 46.1 13.1 12.7 14.1 - * 5-37 12.9 27.0 14.6 15.2 18.5 44.7 13.5 13.0 14.7 - * 33-65 14.3 26.6 13.3 17.8 17.4 42.1 14.5 13.3 13.6 - * 61-93 16.0 26.6 15.4 14.8 17.7 43.8 14.0 13.6 11.4 - * Random 11.3 39.9 18.9 14.7 11.5 68.2 11.9 12.4 12.6 - * - * Random range hardware counters: - * variant ns cycles instructions - * Production 11.3 49.7 197.0 - * ScalarBits 39.9 175.9 721.9 - * NibbleLUT 18.9 83.2 291.2 - * ByteLUT 14.7 63.9 261.2 - * HybridLUT 11.5 50.8 209.8 - * Expand16 68.2 300.8 879.8 - * Lane64SSE 11.9 52.1 224.1 - * Split64SSE 12.4 54.1 236.9 - * ShortSkip 12.6 55.7 252.5 + * excess_min_128 CPU time: + * + * | method | 0-128 | 0-16 | 0-32 | 1-17 | 3-35 | 5-37 | + * | :--------------- | ----: | ----: | ----: | ----: | ----: | ----: | + * | Production | 5.17 | 3.54 | 5.88 | 7.97 | 7.58 | 8.17 | + * | ByteLUT | 17.82 | 2.97 | 5.04 | 11.51 | 11.96 | 10.85 | + * | NibbleLUT | 40.00 | 4.78 | 8.14 | 8.59 | 10.72 | 10.81 | + * | DeinterleavedSSE | 5.44 | 6.08 | 6.24 | 8.28 | 7.84 | 8.90 | + * | Lane64SSE | 6.53 | 7.26 | 7.27 | 10.25 | 10.26 | 10.30 | + * | Split64SSE | 9.67 | 6.22 | 6.30 | 9.63 | 9.52 | 9.48 | + * + * | method | 56-72 | 60-68 | 63-64 | 17-17 | Random | + * | :--------------- | ----: | ----: | ----: | ----: | -----: | + * | Production | 4.01 | 5.89 | 2.12 | 1.55 | 10.07 | + * | ByteLUT | 3.60 | 8.74 | 2.34 | 1.55 | 13.14 | + * | NibbleLUT | 5.27 | 3.52 | 2.33 | 1.55 | 17.48 | + * | DeinterleavedSSE | 6.30 | 6.38 | 2.07 | 1.55 | 9.10 | + * | Lane64SSE | 4.27 | 7.23 | 3.84 | 2.25 | 11.36 | + * | Split64SSE | 9.86 | 9.75 | 3.17 | 1.55 | 11.02 | * * Diagnostic run, 2026-05-30: * taskset -c 0 build/benchmarks-diagnostic_local/excess_positions_benchmarks @@ -74,28 +56,36 @@ * --benchmark_perf_counters=CYCLES,INSTRUCTIONS,CACHE-MISSES * --benchmark_counters_tabular=true * - * excess_positions_512 random target in [-128, 128]: - * variant ns cycles instructions cache misses - * Production 11.4 50.8 188.9 0.001 - * LUTAVX512 12.8 56.8 195.5 0.002 - * BranchingLUT 16.7 73.4 261.4 0.003 - * ExpandAVX512 21.0 93.6 266.6 0.003 - * Expand8 24.6 109.9 449.7 0.002 - * Expand 46.8 207.7 784.8 0.006 - * ByteLUT 49.7 221.4 754.5 0.008 - * Scalar 374.2 1656.0 7716.6 0.041 - * - * excess_positions_512 fixed-target timings, ns: - * variant -64 -8 0 8 64 - * Production 11.6 18.0 18.3 19.1 12.3 - * LUTAVX512 13.4 17.9 19.2 21.3 13.2 - * BranchingLUT 19.1 28.9 28.7 28.3 16.8 - * ExpandAVX512 22.7 36.6 36.3 36.2 22.5 - * Expand8 17.6 52.7 47.5 46.9 17.7 - * Expand 51.0 85.9 86.1 85.5 53.9 - * ByteLUT 34.7 77.4 76.9 79.0 34.3 - * Scalar 367.2 433.5 466.3 428.1 364.6 + * Counter table for `excess_positions_512` with a random target in [-128, 128]. + * `CPU` is nanoseconds per call. `cycles`, `instructions`, and `cache misses` + * are hardware-counter events per call. + * + * | method | CPU (ns) | cycles | instructions | cache misses | + * | :----------- | -------: | -----: | -----------: | -----------: | + * | Production | 11.4 | 50.8 | 188.9 | 0.001 | + * | LUTAVX512 | 12.8 | 56.8 | 195.5 | 0.002 | + * | BranchingLUT | 16.7 | 73.4 | 261.4 | 0.003 | + * | ExpandAVX512 | 21.0 | 93.6 | 266.6 | 0.003 | + * | Expand8 | 24.6 | 109.9 | 449.7 | 0.002 | + * | Expand | 46.8 | 207.7 | 784.8 | 0.006 | + * | ByteLUT | 49.7 | 221.4 | 754.5 | 0.008 | + * | Scalar | 374.2 | 1656.0 | 7716.6 | 0.041 | + * + * CPU time for `excess_positions_512` fixed-target rows. Numeric cells are + * nanoseconds per call. Columns are target excess values. + * + * | method | -64 | -8 | 0 | 8 | 64 | + * | :----------- | ----: | ----: | ----: | ----: | ----: | + * | Production | 11.6 | 18.0 | 18.3 | 19.1 | 12.3 | + * | LUTAVX512 | 13.4 | 17.9 | 19.2 | 21.3 | 13.2 | + * | BranchingLUT | 19.1 | 28.9 | 28.7 | 28.3 | 16.8 | + * | ExpandAVX512 | 22.7 | 36.6 | 36.3 | 36.2 | 22.5 | + * | Expand8 | 17.6 | 52.7 | 47.5 | 46.9 | 17.7 | + * | Expand | 51.0 | 85.9 | 86.1 | 85.5 | 53.9 | + * | ByteLUT | 34.7 | 77.4 | 76.9 | 79.0 | 34.3 | + * | Scalar | 367.2 | 433.5 | 466.3 | 428.1 | 364.6 | */ +// clang-format on namespace pixie::experimental { @@ -150,6 +140,15 @@ static inline constexpr std::array kNibbleMin = make_lut<16>([](uint8_t x) { return min_prefix(x, 4); }); static inline constexpr std::array kNibbleMinOffset = make_lut<16>([](uint8_t x) { return min_prefix_offset(x, 4); }); +static inline constexpr std::array, 4> + kPartialNibbleMinOffset = [] { + std::array, 4> out{}; + for (size_t width = 1; width < out.size(); ++width) { + out[width] = make_lut<16>( + [width](uint8_t x) { return min_prefix_offset(x, width); }); + } + return out; + }(); static inline constexpr std::array kByteDelta = make_lut<256>([](uint8_t x) { return byte_delta(x); }); static inline constexpr std::array kByteMin = @@ -365,6 +364,16 @@ static inline const __m128i excess_lut_nibble_index_128 = _mm_setr_epi8( 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); +static inline const __m128i excess_lut_low_nibble_index_128 = _mm_setr_epi8( + 0, 2, 4, 6, + 8, 10, 12, 14, + 16, 18, 20, 22, + 24, 26, 28, 30); +static inline const __m128i excess_lut_high_nibble_index_128 = _mm_setr_epi8( + 1, 3, 5, 7, + 9, 11, 13, 15, + 17, 19, 21, 23, + 25, 27, 29, 31); static inline const __m128i excess_lut_nibble_mask_128 = _mm_set1_epi8(0x0F); // clang-format on @@ -391,6 +400,14 @@ static inline __m128i excess_prefix_sum_16x_i8(__m128i v) noexcept { return _mm_add_epi8(x, t); } +static inline int horizontal_min_i8(__m128i v) noexcept { + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 8)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 4)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 2)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 1)); + return static_cast(static_cast(_mm_extract_epi8(v, 0))); +} + static inline void scan_full_nibbles_64_sse(uint64_t word, int lane_base_excess, size_t lane_base_offset, @@ -442,6 +459,11 @@ static inline void scan_full_nibbles_64_sse(uint64_t word, } } +static inline size_t partial_nibble_min_offset(uint8_t nibble, + size_t width) noexcept { + return static_cast(kPartialNibbleMinOffset[width][nibble]); +} + static inline ExcessResult excess_min_128_split64_sse_impl( const uint64_t* s, size_t left, @@ -536,6 +558,247 @@ static inline ExcessResult excess_min_128_split64_sse(const uint64_t* s, return detail::excess_min_128_split64_sse_impl(s, left, right); } +/** + * @brief Deinterleaved-nibble SSE excess_min_128 experiment. + * + * @details Workflow: + * + * scalar boundary -> 16 low byte-nibbles -> low candidates + * \-> 16 high byte-nibbles -> high candidates + * -> interleaved first-min selection + * + * The prefix scan is over byte deltas. Low-nibble candidates use the byte + * prefix before each byte; high-nibble candidates add the low-nibble delta. + * This avoids concatenating low/high nibbles or carrying prefix sums across a + * synthetic 32-nibble vector. + */ +static inline ExcessResult excess_min_128_deinterleaved_sse( + const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + size_t bit = left; + for (; bit < right && (bit & 3u) != 0; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + + const size_t first_nibble = bit >> 2; + const size_t last_full_nibble = right >> 2; + const size_t right_partial_width = bit < right ? (right & 3u) : 0; + const size_t end_nibble = + last_full_nibble + (right_partial_width == 0 ? 0 : 1); + + if (first_nibble < end_nibble) { + const __m128i bytes = _mm_loadu_si128(reinterpret_cast(s)); + const __m128i lo_nibbles = _mm_and_si128(bytes, excess_lut_nibble_mask_128); + const __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(bytes, 4), excess_lut_nibble_mask_128); + const __m128i lo_delta = _mm_shuffle_epi8(excess_lut_delta_128, lo_nibbles); + const __m128i hi_delta = _mm_shuffle_epi8(excess_lut_delta_128, hi_nibbles); + const __m128i byte_delta = _mm_add_epi8(lo_delta, hi_delta); + const __m128i byte_prefix = detail::excess_prefix_sum_16x_i8(byte_delta); + const __m128i byte_prefix_before = _mm_slli_si128(byte_prefix, 1); + + __m128i lo_local_min = _mm_shuffle_epi8(excess_lut_min_128, lo_nibbles); + __m128i hi_local_min = _mm_shuffle_epi8(excess_lut_min_128, hi_nibbles); + + const __m128i byte_index = excess_lut_nibble_index_128; + if (right_partial_width != 0) { + const bool partial_is_high = (last_full_nibble & 1u) != 0; + const size_t partial_byte = last_full_nibble >> 1; + const __m128i partial_source = partial_is_high ? hi_nibbles : lo_nibbles; + __m128i partial_min = + _mm_shuffle_epi8(excess_lut_pos0_sse, partial_source); + if (right_partial_width >= 2) { + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos1_sse, partial_source)); + } + if (right_partial_width >= 3) { + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos2_sse, partial_source)); + } + const __m128i partial_lane = _mm_cmpeq_epi8( + byte_index, _mm_set1_epi8(static_cast(partial_byte))); + if (partial_is_high) { + hi_local_min = _mm_blendv_epi8(hi_local_min, partial_min, partial_lane); + } else { + lo_local_min = _mm_blendv_epi8(lo_local_min, partial_min, partial_lane); + } + } + + const __m128i lo_candidates = + _mm_add_epi8(byte_prefix_before, lo_local_min); + const __m128i hi_candidates = + _mm_add_epi8(_mm_add_epi8(byte_prefix_before, lo_delta), hi_local_min); + + __m128i masked_lo = lo_candidates; + __m128i masked_hi = hi_candidates; + if (first_nibble != 0 || end_nibble != 32) { + const __m128i first_minus_one = _mm_set1_epi8( + static_cast(static_cast(first_nibble) - 1)); + const __m128i last = _mm_set1_epi8(static_cast(end_nibble)); + const __m128i lo_active = _mm_and_si128( + _mm_cmpgt_epi8(excess_lut_low_nibble_index_128, first_minus_one), + _mm_cmpgt_epi8(last, excess_lut_low_nibble_index_128)); + const __m128i hi_active = _mm_and_si128( + _mm_cmpgt_epi8(excess_lut_high_nibble_index_128, first_minus_one), + _mm_cmpgt_epi8(last, excess_lut_high_nibble_index_128)); + masked_lo = _mm_blendv_epi8(_mm_set1_epi8(127), lo_candidates, lo_active); + masked_hi = _mm_blendv_epi8(_mm_set1_epi8(127), hi_candidates, hi_active); + } + const int candidate_min = + detail::horizontal_min_i8(_mm_min_epi8(masked_lo, masked_hi)); + + if (candidate_min < best) { + const __m128i min_vec = _mm_set1_epi8(static_cast(candidate_min)); + const uint32_t lo_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(masked_lo, min_vec))); + const uint32_t hi_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(masked_hi, min_vec))); + const uint32_t lo_nibble_index = + lo_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(lo_equal_mask)) * 2u; + const uint32_t hi_nibble_index = + hi_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(hi_equal_mask)) * 2u + + 1u; + const uint32_t nibble_index = std::min(lo_nibble_index, hi_nibble_index); + const uint32_t byte_offset = nibble_index >> 1u; + const uint64_t byte_word = s[byte_offset >> 3u]; + const uint8_t byte = static_cast( + (byte_word >> ((byte_offset & 7u) * 8u)) & 0xFFu); + const uint8_t nibble = (nibble_index & 1u) == 0 + ? static_cast(byte & 0xFu) + : static_cast((byte >> 4u) & 0xFu); + const size_t local_offset = + right_partial_width != 0 && nibble_index == last_full_nibble + ? detail::partial_nibble_min_offset(nibble, right_partial_width) + : static_cast(detail::kNibbleMinOffset[nibble]); + best = candidate_min; + best_offset = static_cast(nibble_index) * 4u + local_offset; + } + } + + return {best, best_offset}; +} + +/** + * @brief Deinterleaved SSE with a full-block fast path. + * + * @details Workflow: + * + * left == 0 && right == 128 -> deinterleaved full block without masks + * otherwise -> deinterleaved SSE + * + * The full-block path skips scalar boundary handling, partial-nibble handling, + * and active-lane masks. It is intended to measure whether the common + * whole-block shape is worth specializing independently from the general + * deinterleaved implementation. + */ +static inline ExcessResult excess_min_128_deinterleaved_full_sse( + const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + if (left != 0 || right != 128) { + return excess_min_128_deinterleaved_sse(s, left, right); + } + + const __m128i bytes = _mm_loadu_si128(reinterpret_cast(s)); + const __m128i lo_nibbles = _mm_and_si128(bytes, excess_lut_nibble_mask_128); + const __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(bytes, 4), excess_lut_nibble_mask_128); + const __m128i lo_delta = _mm_shuffle_epi8(excess_lut_delta_128, lo_nibbles); + const __m128i hi_delta = _mm_shuffle_epi8(excess_lut_delta_128, hi_nibbles); + const __m128i byte_delta = _mm_add_epi8(lo_delta, hi_delta); + const __m128i byte_prefix = detail::excess_prefix_sum_16x_i8(byte_delta); + const __m128i byte_prefix_before = _mm_slli_si128(byte_prefix, 1); + + const __m128i lo_candidates = _mm_add_epi8( + byte_prefix_before, _mm_shuffle_epi8(excess_lut_min_128, lo_nibbles)); + const __m128i hi_candidates = + _mm_add_epi8(_mm_add_epi8(byte_prefix_before, lo_delta), + _mm_shuffle_epi8(excess_lut_min_128, hi_nibbles)); + const int candidate_min = + detail::horizontal_min_i8(_mm_min_epi8(lo_candidates, hi_candidates)); + + int best = 0; + size_t best_offset = 0; + if (candidate_min < best) { + const __m128i min_vec = _mm_set1_epi8(static_cast(candidate_min)); + const uint32_t lo_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(lo_candidates, min_vec))); + const uint32_t hi_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(hi_candidates, min_vec))); + const uint32_t lo_nibble_index = + lo_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(lo_equal_mask)) * 2u; + const uint32_t hi_nibble_index = + hi_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(hi_equal_mask)) * 2u + 1u; + const uint32_t nibble_index = std::min(lo_nibble_index, hi_nibble_index); + const uint32_t byte_offset = nibble_index >> 1u; + const uint64_t byte_word = s[byte_offset >> 3u]; + const uint8_t byte = + static_cast((byte_word >> ((byte_offset & 7u) * 8u)) & 0xFFu); + const uint8_t nibble = (nibble_index & 1u) == 0 + ? static_cast(byte & 0xFu) + : static_cast((byte >> 4u) & 0xFu); + best = candidate_min; + best_offset = static_cast(nibble_index) * 4u + + static_cast(detail::kNibbleMinOffset[nibble]); + } + return {best, best_offset}; +} + +/** + * @brief Deinterleaved SSE with a narrow byte-aligned shortcut. + * + * @details Workflow: + * + * byte-aligned width <= 16 -> byte LUT + * otherwise -> deinterleaved SSE + * + * Production currently keeps a byte-LUT shortcut for byte-aligned widths up to + * 32. This variant narrows that condition to test whether the 32-bit case is + * worth its dispatch cost in mixed workloads. + */ +static inline ExcessResult excess_min_128_deinterleaved_byte16_sse( + const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + const size_t clamped_left = std::min(left, 128); + const size_t clamped_right = std::min(right, 128); + const size_t width = clamped_right - clamped_left; + if (width <= 16 && (clamped_left & 7u) == 0 && (clamped_right & 7u) == 0) { + return excess_min_128_byte_lut(s, left, right); + } + return excess_min_128_deinterleaved_sse(s, left, right); +} + /** * @brief Short-range dispatch experiment. * diff --git a/include/pixie/rmq/cartesian_hybrid_btree.h b/include/pixie/rmq/cartesian_hybrid_btree.h index eeecfdf..5584a1b 100644 --- a/include/pixie/rmq/cartesian_hybrid_btree.h +++ b/include/pixie/rmq/cartesian_hybrid_btree.h @@ -1241,10 +1241,10 @@ class HybridBTreePlusMinusOne { /** * @brief Cartesian-tree value RMQ using HybridBTree-style LCA. * - * @details This class keeps the same public value-RMQ contract as the other - * value RMQ backends. It builds a stable Ferrada-Navarro BP Cartesian-tree - * encoding, uses `BitVector` for close-parenthesis rank/select, and delegates - * the BP-depth minimum query to + * @details This class follows the same public value-RMQ specification as the + * other value RMQ backends. It builds a stable Ferrada-Navarro BP + * Cartesian-tree encoding, uses `BitVector` for close-parenthesis rank/select, + * and delegates the BP-depth minimum query to * `detail::HybridBTreePlusMinusOne`. The BP-depth backend keeps a configurable * low-level leaf size, fixed 192-entry middle nodes with embedded minima, and * fixed 256-entry high nodes. A single coarse value-level sparse table is diff --git a/include/pixie/rmq/cartesian_rmm.h b/include/pixie/rmq/cartesian_rmm.h index eaee562..4502e17 100644 --- a/include/pixie/rmq/cartesian_rmm.h +++ b/include/pixie/rmq/cartesian_rmm.h @@ -226,12 +226,12 @@ class RmMPlusMinusOne { /** * @brief Ferrada-Navarro Cartesian-tree RMQ using rmM support. * - * @details This class keeps the same public value-RMQ contract as the other - * value RMQ backends, but replaces the usual balanced-parentheses rank/select - * and depth-RMQ support with `detail::RmMPlusMinusOne`, which in turn uses the - * experimental range min-max btree over the BP excess sequence. BP - * construction uses a succinct monotone bit-stack, preserving the same stable - * Cartesian-tree shape without an n-entry index stack. + * @details This class follows the same public value-RMQ specification as the + * other value RMQ backends, but replaces the usual balanced-parentheses + * rank/select and depth-RMQ support with `detail::RmMPlusMinusOne`, which in + * turn uses the experimental range min-max btree over the BP excess sequence. + * BP construction uses a succinct monotone bit-stack, preserving the same + * stable Cartesian-tree shape without an n-entry index stack. * * This implementation is included from `pixie/rmq.h` as the rmM-backed * Cartesian-tree reduction. diff --git a/include/pixie/rmq/sdsl_sct.h b/include/pixie/rmq/sdsl_sct.h index d9043d1..329ad75 100644 --- a/include/pixie/rmq/sdsl_sct.h +++ b/include/pixie/rmq/sdsl_sct.h @@ -31,9 +31,9 @@ namespace pixie::rmq { * Johannes Fischer, "Optimal Succinctness for Range Minimum Queries", * LATIN 2010; arXiv:0812.2775. * - * This adapter exposes the Pixie value-RMQ contract: half-open ranges - * `[left, right)`, invalid ranges returning `npos`, and first-minimum tie - * semantics. + * This adapter exposes the Pixie value-RMQ interface specification: half-open + * ranges `[left, right)`, invalid ranges returning `npos`, and first-minimum + * tie semantics. * * The SDSL structure does not retain values after construction, so this adapter * keeps a non-owning span for `range_min()`. The indexed values must outlive diff --git a/scripts/coverage_report.sh b/scripts/coverage_report.sh index 84bec46..4666d08 100755 --- a/scripts/coverage_report.sh +++ b/scripts/coverage_report.sh @@ -7,11 +7,16 @@ BUILD_DIR="${ROOT_DIR}/build/coverage" cmake --preset coverage cmake --build --preset coverage +find "${BUILD_DIR}" -name "*.gcda" -delete +find "${BUILD_DIR}" -name "*.gcov" -delete +rm -f "${BUILD_DIR}/coverage.txt" "${BUILD_DIR}/gcov_files.txt" + "${BUILD_DIR}/unittests" "${BUILD_DIR}/excess_positions_tests" "${BUILD_DIR}/louds_tree_tests" "${BUILD_DIR}/dfuds_tree_tests" "${BUILD_DIR}/test_rmm" +"${BUILD_DIR}/rmq_tests" cd "${BUILD_DIR}" find . -name "*.gcda" > gcov_files.txt diff --git a/src/benchmarks/excess_positions_benchmarks.cpp b/src/benchmarks/excess_positions_benchmarks.cpp index 501536a..6cd5df7 100644 --- a/src/benchmarks/excess_positions_benchmarks.cpp +++ b/src/benchmarks/excess_positions_benchmarks.cpp @@ -17,6 +17,9 @@ using pixie::experimental::excess_positions_512_expand8; using pixie::experimental::excess_positions_512_expand_avx512; using pixie::experimental::excess_positions_512_lut_avx512; #ifdef PIXIE_AVX2_SUPPORT +using pixie::experimental::excess_min_128_deinterleaved_byte16_sse; +using pixie::experimental::excess_min_128_deinterleaved_full_sse; +using pixie::experimental::excess_min_128_deinterleaved_sse; using pixie::experimental::excess_min_128_expand16_avx2; using pixie::experimental::excess_min_128_lane64_sse; using pixie::experimental::excess_min_128_short_skip; @@ -383,6 +386,12 @@ PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_HybridLUT", #ifdef PIXIE_AVX2_SUPPORT PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_Expand16AVX2", excess_min_128_expand16_avx2); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_DeinterleavedSSE", + excess_min_128_deinterleaved_sse); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_DeinterleavedFullSSE", + excess_min_128_deinterleaved_full_sse); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_DeinterleavedByte16SSE", + excess_min_128_deinterleaved_byte16_sse); PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_Lane64SSE", excess_min_128_lane64_sse); PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_Split64SSE", @@ -451,6 +460,15 @@ PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_HybridLUT_RandomRange", PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT( "BM_ExcessMin128_Expand16AVX2_RandomRange", excess_min_128_expand16_avx2); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT( + "BM_ExcessMin128_DeinterleavedSSE_RandomRange", + excess_min_128_deinterleaved_sse); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT( + "BM_ExcessMin128_DeinterleavedFullSSE_RandomRange", + excess_min_128_deinterleaved_full_sse); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT( + "BM_ExcessMin128_DeinterleavedByte16SSE_RandomRange", + excess_min_128_deinterleaved_byte16_sse); PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_Lane64SSE_RandomRange", excess_min_128_lane64_sse); PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_Split64SSE_RandomRange", diff --git a/src/tests/excess_positions_tests.cpp b/src/tests/excess_positions_tests.cpp index 507c248..6321478 100644 --- a/src/tests/excess_positions_tests.cpp +++ b/src/tests/excess_positions_tests.cpp @@ -19,6 +19,9 @@ using pixie::experimental::excess_positions_512_expand8; using pixie::experimental::excess_positions_512_expand_avx512; using pixie::experimental::excess_positions_512_lut_avx512; #ifdef PIXIE_AVX2_SUPPORT +using pixie::experimental::excess_min_128_deinterleaved_byte16_sse; +using pixie::experimental::excess_min_128_deinterleaved_full_sse; +using pixie::experimental::excess_min_128_deinterleaved_sse; using pixie::experimental::excess_min_128_expand16_avx2; using pixie::experimental::excess_min_128_lane64_sse; using pixie::experimental::excess_min_128_short_skip; @@ -639,6 +642,15 @@ TEST(ExcessPositions128Experimental, MinVariantsMatchNaive) { #ifdef PIXIE_AVX2_SUPPORT check_min_matches_naive(excess_min_128_expand16_avx2, "expand16_avx2", s.data(), left, right, case_id); + check_min_matches_naive(excess_min_128_deinterleaved_sse, + "deinterleaved_sse", s.data(), left, right, + case_id); + check_min_matches_naive(excess_min_128_deinterleaved_full_sse, + "deinterleaved_full_sse", s.data(), left, right, + case_id); + check_min_matches_naive(excess_min_128_deinterleaved_byte16_sse, + "deinterleaved_byte16_sse", s.data(), left, right, + case_id); check_min_matches_naive(excess_min_128_lane64_sse, "lane64_sse", s.data(), left, right, case_id); check_min_matches_naive(excess_min_128_split64_sse, "split64_sse", @@ -671,6 +683,14 @@ TEST(ExcessPositions128Experimental, MinVariantsMatchNaiveRandom) { #ifdef PIXIE_AVX2_SUPPORT check_min_matches_naive(excess_min_128_expand16_avx2, "expand16_avx2", s.data(), left, right, t); + check_min_matches_naive(excess_min_128_deinterleaved_sse, + "deinterleaved_sse", s.data(), left, right, t); + check_min_matches_naive(excess_min_128_deinterleaved_full_sse, + "deinterleaved_full_sse", s.data(), left, right, + t); + check_min_matches_naive(excess_min_128_deinterleaved_byte16_sse, + "deinterleaved_byte16_sse", s.data(), left, right, + t); check_min_matches_naive(excess_min_128_lane64_sse, "lane64_sse", s.data(), left, right, t); check_min_matches_naive(excess_min_128_split64_sse, "split64_sse", @@ -682,6 +702,49 @@ TEST(ExcessPositions128Experimental, MinVariantsMatchNaiveRandom) { } } +#ifdef PIXIE_AVX2_SUPPORT +TEST(ExcessPositions128Experimental, DeinterleavedSseBoundaryAndTieCases) { + const std::array, 5> cases = {{ + {0x0F0F0F0F0F0F0F0Full, 0xF0F0F0F0F0F0F0F0ull}, + {0x3333333333333333ull, 0xCCCCCCCCCCCCCCCCull}, + {0x6666666666666666ull, 0x9999999999999999ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull}, + {0x1111111111111111ull, 0xEEEEEEEEEEEEEEEEull}, + }}; + const std::array, 13> ranges = {{ + {0, 5}, + {4, 7}, + {4, 11}, + {8, 15}, + {12, 19}, + {1, 17}, + {3, 35}, + {5, 37}, + {33, 65}, + {61, 93}, + {124, 127}, + {125, 128}, + {0, 127}, + }}; + + int case_id = 0; + for (const auto& s : cases) { + for (const auto [left, right] : ranges) { + check_min_matches_naive(excess_min_128_deinterleaved_sse, + "deinterleaved_sse", s.data(), left, right, + case_id); + check_min_matches_naive(excess_min_128_deinterleaved_full_sse, + "deinterleaved_full_sse", s.data(), left, right, + case_id); + check_min_matches_naive(excess_min_128_deinterleaved_byte16_sse, + "deinterleaved_byte16_sse", s.data(), left, right, + case_id); + ++case_id; + } + } +} +#endif + TEST(ExcessPositions128, ForwardAndBackwardSearchMatchNaive) { std::mt19937_64 rng(42); const std::array offsets = {0, 1, 63, 64, 65, 126, 127, 128}; diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index 67ff50e..b48c4d2 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -166,16 +166,16 @@ struct HybridBTreeCase { } // namespace template -class ValueRmqContractTest : public ::testing::Test {}; +class ValueRmqSpecificationTest : public ::testing::Test {}; using ValueRmqCases = ::testing::Types; -TYPED_TEST_SUITE(ValueRmqContractTest, ValueRmqCases); +TYPED_TEST_SUITE(ValueRmqSpecificationTest, ValueRmqCases); -TYPED_TEST(ValueRmqContractTest, ExhaustiveSmallArray) { +TYPED_TEST(ValueRmqSpecificationTest, ExhaustiveSmallArray) { const std::vector values = {4, 1, 3, 1, 5, 0, 0, 2}; const typename TypeParam::Rmq rmq = TypeParam::make(std::span(values)); @@ -237,7 +237,7 @@ TEST(RmqSuccinctIncreasingStack, PopsAcrossEmptyBlocks) { EXPECT_TRUE(stack.empty()); } -TYPED_TEST(ValueRmqContractTest, FirstMinimumTieBreaking) { +TYPED_TEST(ValueRmqSpecificationTest, FirstMinimumTieBreaking) { const std::vector values = {7, 2, 2, 3, 2}; const typename TypeParam::Rmq rmq = TypeParam::make(std::span(values)); @@ -246,7 +246,7 @@ TYPED_TEST(ValueRmqContractTest, FirstMinimumTieBreaking) { EXPECT_EQ(rmq.arg_min(2, 5), 2u); } -TYPED_TEST(ValueRmqContractTest, InvalidAndEmptyRanges) { +TYPED_TEST(ValueRmqSpecificationTest, InvalidAndEmptyRanges) { using Rmq = typename TypeParam::Rmq; const std::vector values = {3, 1, 2}; @@ -265,18 +265,23 @@ TYPED_TEST(ValueRmqContractTest, InvalidAndEmptyRanges) { EXPECT_TRUE(empty_rmq.empty()); EXPECT_EQ(default_rmq.arg_min(0, 0), Rmq::npos); EXPECT_EQ(empty_rmq.arg_min(0, 0), Rmq::npos); + EXPECT_EQ(default_rmq.range_min(0, 0), 0); + EXPECT_EQ(empty_rmq.range_min(0, 0), 0); } -TYPED_TEST(ValueRmqContractTest, ComparatorCanSelectMaximum) { +TYPED_TEST(ValueRmqSpecificationTest, ComparatorCanSelectMaximum) { const std::vector values = {1, 8, 3, 8, 4}; const typename TypeParam::MaxRmq rmq = TypeParam::make_max(std::span(values)); check_all_ranges(rmq, std::span(values), std::greater()); EXPECT_EQ(rmq.arg_min(0, 5), 1u); + EXPECT_EQ(rmq.range_min(2, 2), 0); + EXPECT_EQ(rmq.range_min(4, 3), 0); + EXPECT_EQ(rmq.range_min(0, values.size() + 1), 0); } -TYPED_TEST(ValueRmqContractTest, MonotoneArrays) { +TYPED_TEST(ValueRmqSpecificationTest, MonotoneArrays) { const std::vector increasing = {1, 2, 3, 4, 5, 6}; const std::vector decreasing = {6, 5, 4, 3, 2, 1}; const typename TypeParam::Rmq increasing_rmq = @@ -290,7 +295,7 @@ TYPED_TEST(ValueRmqContractTest, MonotoneArrays) { std::less()); } -TYPED_TEST(ValueRmqContractTest, DifferentialRandom) { +TYPED_TEST(ValueRmqSpecificationTest, DifferentialRandom) { std::mt19937_64 rng(42); std::uniform_int_distribution value_dist(-50, 50); for (std::size_t size = 1; size <= 257; size += 17) { @@ -304,8 +309,38 @@ TYPED_TEST(ValueRmqContractTest, DifferentialRandom) { } } +TEST(RmqSparseTable, OverlappingBlockCandidateDirectionsAndTies) { + { + const std::vector values = {5, 4, 3, 2, 1, 0, 9}; + const pixie::rmq::SparseTable rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 5u); + EXPECT_EQ(rmq.range_min(0, 6), 0); + } + + { + const std::vector values = {0, 5, 6, 7, 8, 9, 1}; + const pixie::rmq::SparseTable rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 0u); + EXPECT_EQ(rmq.range_min(0, 6), 0); + } + + { + const std::vector values = {1, 0, 9, 9, 0, 2}; + const pixie::rmq::SparseTable rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 1u); + EXPECT_EQ(rmq.range_min(0, 6), 0); + } +} + +TEST(RmqSegmentTree, NonPowerOfTwoTailRanges) { + const std::vector values = {6, 5, 4, 3, 2}; + const pixie::rmq::SegmentTree rmq(values); + check_all_ranges(rmq, std::span(values), std::less()); +} + #ifdef SDSL_SUPPORT -TEST(RmqSdslSct, MatchesPixieMinContractForSignedValuesAndDuplicates) { +TEST(RmqSdslSct, + MatchesPixieValueRmqSpecificationForSignedValuesAndDuplicates) { const std::vector values = {4, -3, 7, -3, 0, -8, -8, 2, 2}; const pixie::rmq::SdslSct rmq{std::span(values)}; From f9abbb538e8c9af0e3ae67256dbeac8addf31489 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 20 Jun 2026 16:16:53 +0300 Subject: [PATCH 22/27] Submodule --- .gitmodules | 2 +- CMakePresets.json | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 6f6274b..00371e1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "agentic/cpp"] path = agentic/cpp - url = git@github.com:Malkovsky/ai_for_cpp.git + url = https://github.com/Malkovsky/ai_for_cpp.git diff --git a/CMakePresets.json b/CMakePresets.json index becff7a..c8e31cf 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -112,6 +112,7 @@ "targets": [ "unittests", "benchmark_tests", + "rmq_tests", "test_rmm", "louds_tree_tests", "dfuds_tree_tests", @@ -125,6 +126,7 @@ "targets": [ "unittests", "benchmark_tests", + "rmq_tests", "test_rmm", "louds_tree_tests", "dfuds_tree_tests", @@ -189,6 +191,7 @@ "targets": [ "unittests", "benchmark_tests", + "rmq_tests", "test_rmm", "louds_tree_tests", "dfuds_tree_tests", @@ -202,6 +205,7 @@ "targets": [ "unittests", "benchmark_tests", + "rmq_tests", "test_rmm", "louds_tree_tests", "dfuds_tree_tests", From e3b49fc3a3a23add12619e1b20e630974d5a4047 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 20 Jun 2026 16:51:13 +0300 Subject: [PATCH 23/27] fix --- src/tests/excess_record_lows_tests.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tests/excess_record_lows_tests.cpp b/src/tests/excess_record_lows_tests.cpp index 6a79668..a39d85f 100644 --- a/src/tests/excess_record_lows_tests.cpp +++ b/src/tests/excess_record_lows_tests.cpp @@ -163,8 +163,7 @@ TEST(ExcessRecordLows128, Random) { } #ifdef PIXIE_AVX2_SUPPORT -static void check_nibble_lut_matches_naive(const uint64_t* s, - int case_id = 0) { +static void check_nibble_lut_matches_naive(const uint64_t* s, int case_id = 0) { alignas(16) uint64_t out[2]; alignas(16) uint64_t ref[2]; excess_record_lows_128_nibble_lut(s, out); From a7d9e31319da02a2a3f3e6ebf7eb3345ffd3a402 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 20 Jun 2026 18:09:29 +0300 Subject: [PATCH 24/27] Coverage --- include/pixie/experimental/rmm_btree.h | 23 +++++++ include/pixie/rmq/cartesian_rmm.h | 23 +++++++ include/pixie/rmq/sparse_table.h | 14 ++--- src/tests/rmq_tests.cpp | 84 ++++++++++++++++++++++++-- 4 files changed, 130 insertions(+), 14 deletions(-) diff --git a/include/pixie/experimental/rmm_btree.h b/include/pixie/experimental/rmm_btree.h index 3009ee8..981637b 100644 --- a/include/pixie/experimental/rmm_btree.h +++ b/include/pixie/experimental/rmm_btree.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -331,6 +332,28 @@ class RmMBTree : public RmMBase> { return range_extreme_query_val(range_begin, range_end, false); } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this rmM object, rank/select support, and summary buffers. + * The external bit words indexed by this object are not owned and are + * excluded. + */ + std::size_t memory_usage_bytes() const { + std::size_t bytes = sizeof(*this); + bytes += pixie::optional_nested_owned_memory_bytes(rank_index_); + bytes += pixie::vector_capacity_bytes(level_counts_); + bytes += pixie::vector_capacity_bytes(low_levels_); + bytes += pixie::vector_capacity_bytes(high_levels_); + for (const std::vector& level : low_levels_) { + bytes += pixie::vector_capacity_bytes(level); + } + for (const std::vector& level : high_levels_) { + bytes += pixie::vector_capacity_bytes(level); + } + return bytes; + } + std::size_t mincount_impl(std::size_t range_begin, std::size_t range_end) const { if (range_begin > range_end || range_end >= bit_count_) { diff --git a/include/pixie/rmq/cartesian_rmm.h b/include/pixie/rmq/cartesian_rmm.h index 4502e17..e5900a3 100644 --- a/include/pixie/rmq/cartesian_rmm.h +++ b/include/pixie/rmq/cartesian_rmm.h @@ -68,6 +68,7 @@ */ #include +#include #include #include @@ -213,6 +214,16 @@ class RmMPlusMinusOne { return rmm_.rank0(end_position); } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this ±1 RMQ adapter and the nested rmM support. The + * external packed BP words are not owned and are excluded. + */ + std::size_t memory_usage_bytes() const { + return sizeof(*this) + pixie::nested_owned_memory_bytes(rmm_); + } + private: using RmM = pixie::experimental::RmMBTree; @@ -353,6 +364,18 @@ class CartesianRmM */ std::span bp_words() const { return bp_bits_; } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this value-RMQ object, packed Cartesian BP words, and + * nested BP rmM support. The external input values are not owned and are + * excluded. + */ + std::size_t memory_usage_bytes_impl() const { + return sizeof(*this) + pixie::vector_capacity_bytes(bp_bits_) + + pixie::nested_owned_memory_bytes(bp_support_); + } + private: using BpSupport = detail::RmMPlusMinusOne; diff --git a/include/pixie/rmq/sparse_table.h b/include/pixie/rmq/sparse_table.h index fdb10d4..e6b9566 100644 --- a/include/pixie/rmq/sparse_table.h +++ b/include/pixie/rmq/sparse_table.h @@ -118,20 +118,14 @@ class SparseTable /** * @brief Choose the better of two candidate positions. * - * @details `npos` is treated as missing. If both values compare equal, the - * smaller position wins to preserve first-minimum semantics. + * @details Both candidates are valid table entries. If both values compare + * equal, the smaller position wins to preserve first-minimum semantics. * - * @param left First candidate position, or `npos`. - * @param right Second candidate position, or `npos`. + * @param left First candidate position. + * @param right Second candidate position. * @return Position of the selected candidate. */ std::size_t better(std::size_t left, std::size_t right) const { - if (left == npos) { - return right; - } - if (right == npos) { - return left; - } if (compare_(values_[right], values_[left])) { return right; } diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index b48c4d2..783a5f7 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -269,16 +269,50 @@ TYPED_TEST(ValueRmqSpecificationTest, InvalidAndEmptyRanges) { EXPECT_EQ(empty_rmq.range_min(0, 0), 0); } +TYPED_TEST(ValueRmqSpecificationTest, MemoryUsageCountsOwnedIndexStorage) { + using Rmq = typename TypeParam::Rmq; + using MaxRmq = typename TypeParam::MaxRmq; + + const Rmq default_rmq; + EXPECT_GE(default_rmq.memory_usage_bytes(), sizeof(Rmq)); + + std::vector values(1537); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 19 + i / 5) % 127); + } + + const Rmq rmq = TypeParam::make(std::span(values)); + const MaxRmq max_rmq = TypeParam::make_max(std::span(values)); + EXPECT_GE(rmq.memory_usage_bytes(), sizeof(Rmq)); + EXPECT_GE(max_rmq.memory_usage_bytes(), sizeof(MaxRmq)); + EXPECT_GT(rmq.memory_usage_bytes(), default_rmq.memory_usage_bytes()); +} + TYPED_TEST(ValueRmqSpecificationTest, ComparatorCanSelectMaximum) { + using MaxRmq = typename TypeParam::MaxRmq; + const std::vector values = {1, 8, 3, 8, 4}; - const typename TypeParam::MaxRmq rmq = - TypeParam::make_max(std::span(values)); + const MaxRmq rmq = TypeParam::make_max(std::span(values)); check_all_ranges(rmq, std::span(values), std::greater()); EXPECT_EQ(rmq.arg_min(0, 5), 1u); + EXPECT_EQ(rmq.arg_min(2, 2), MaxRmq::npos); + EXPECT_EQ(rmq.arg_min(4, 3), MaxRmq::npos); + EXPECT_EQ(rmq.arg_min(0, values.size() + 1), MaxRmq::npos); EXPECT_EQ(rmq.range_min(2, 2), 0); EXPECT_EQ(rmq.range_min(4, 3), 0); EXPECT_EQ(rmq.range_min(0, values.size() + 1), 0); + + const std::vector empty_values; + const MaxRmq default_rmq; + const MaxRmq empty_rmq = + TypeParam::make_max(std::span(empty_values)); + EXPECT_TRUE(default_rmq.empty()); + EXPECT_TRUE(empty_rmq.empty()); + EXPECT_EQ(default_rmq.arg_min(0, 0), MaxRmq::npos); + EXPECT_EQ(empty_rmq.arg_min(0, 0), MaxRmq::npos); + EXPECT_EQ(default_rmq.range_min(0, 0), 0); + EXPECT_EQ(empty_rmq.range_min(0, 0), 0); } TYPED_TEST(ValueRmqSpecificationTest, MonotoneArrays) { @@ -330,6 +364,27 @@ TEST(RmqSparseTable, OverlappingBlockCandidateDirectionsAndTies) { EXPECT_EQ(rmq.arg_min(0, 6), 1u); EXPECT_EQ(rmq.range_min(0, 6), 0); } + + { + const std::vector values = {1, 2, 3, 4, 9, 10, 0}; + const pixie::rmq::SparseTable> rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 5u); + EXPECT_EQ(rmq.range_min(0, 6), 10); + } + + { + const std::vector values = {10, 9, 8, 7, 6, 5, 20}; + const pixie::rmq::SparseTable> rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 0u); + EXPECT_EQ(rmq.range_min(0, 6), 10); + } + + { + const std::vector values = {10, 1, 2, 3, 10, 4}; + const pixie::rmq::SparseTable> rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 0u); + EXPECT_EQ(rmq.range_min(0, 6), 10); + } } TEST(RmqSegmentTree, NonPowerOfTwoTailRanges) { @@ -422,6 +477,12 @@ TEST(RmqHybridBTree, LeafSelectorEnumVariants) { pixie::rmq::HybridBTree, std::size_t, 252, 256, pixie::rmq::HybridBTreeLeafSelector::BP>; + EXPECT_EQ(MaskRmq::top_sparse_block_size_for(0), + MaskRmq::kMinTopSparseBlockSize); + EXPECT_EQ(MaskRmq::top_sparse_block_count_for(0), 0u); + EXPECT_EQ(BpRmq::top_sparse_block_size_for(0), BpRmq::kMinTopSparseBlockSize); + EXPECT_EQ(BpRmq::top_sparse_block_count_for(0), 0u); + std::vector values(4099, 10000); for (std::size_t i = 0; i < values.size(); ++i) { values[i] = static_cast((i * 37 + i / 7) % 257); @@ -435,9 +496,18 @@ TEST(RmqHybridBTree, LeafSelectorEnumVariants) { const MaskRmq mask_rmq{std::span(values)}; const BpRmq bp_rmq{std::span(values)}; + const MaskRmq empty_mask_rmq; + const BpRmq empty_bp_rmq; + EXPECT_EQ(empty_mask_rmq.arg_min(0, 0), MaskRmq::npos); + EXPECT_EQ(empty_bp_rmq.arg_min(0, 0), BpRmq::npos); + EXPECT_EQ(empty_mask_rmq.range_min(0, 0), 0); + EXPECT_EQ(empty_bp_rmq.range_min(0, 0), 0); + const std::vector> ranges = { - {0, 1}, {0, 252}, {1, 251}, {248, 253}, {251, 753}, - {700, 2100}, {2048, 2049}, {0, values.size()}, {3000, 4099}, + {0, 1}, {0, 13}, {0, 252}, {1, 251}, + {20, 40}, {100, 200}, {200, 248}, {248, 253}, + {251, 753}, {700, 2100}, {2048, 2049}, {0, values.size()}, + {3000, 4099}, }; for (const auto [left, right] : ranges) { @@ -447,6 +517,10 @@ TEST(RmqHybridBTree, LeafSelectorEnumVariants) { << "mask range=[" << left << "," << right << ")"; EXPECT_EQ(bp_rmq.arg_min(left, right), expected) << "bp range=[" << left << "," << right << ")"; + EXPECT_EQ(mask_rmq.range_min(left, right), values[expected]) + << "mask range=[" << left << "," << right << ")"; + EXPECT_EQ(bp_rmq.range_min(left, right), values[expected]) + << "bp range=[" << left << "," << right << ")"; } } @@ -545,6 +619,8 @@ TEST(RmqHybridBTree, TopSparseOverlayCapsBlockCount) { TEST(RmqHybridBTree, TopSparseOverlayComparatorMaximum) { using Rmq = pixie::rmq::HybridBTree>; constexpr std::size_t kTopBlock = Rmq::kMinTopSparseBlockSize; + EXPECT_EQ(Rmq::top_sparse_block_size_for(0), kTopBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(0), 0u); std::vector values(2 * kTopBlock + 33, -1000); for (std::size_t i = 0; i < values.size(); ++i) { From d26a5cacecb716b87f2fd3fd1d63d903f39ef248 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 20 Jun 2026 18:58:07 +0300 Subject: [PATCH 25/27] coverage --- src/tests/rmq_tests.cpp | 148 ++++++++++++++++++++++++++++++++++++++++ src/tests/test_rmm.cpp | 56 +++++++++++++++ 2 files changed, 204 insertions(+) diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index 783a5f7..b5a5006 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -469,6 +469,57 @@ TEST(RmqHybridBTree, BoundaryAndFallbackRanges) { } } +TEST(RmqHybridBTree, MaskLeafSelectorPrefixSuffixAndInteriorFallback) { + using Rmq = pixie::rmq::HybridBTree; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + + { + std::vector values(kLeaf, 1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast(i % 17); + } + values[0] = -10; + values[kLeaf - 1] = -100; + + const Rmq rmq{std::span(values)}; + const std::size_t right = kLeaf / 2; + EXPECT_EQ(rmq.arg_min(0, right), naive_arg_min(std::span(values), + 0, right, std::less())); + } + + { + std::vector values(kLeaf, 1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 5) % 23); + } + values[0] = -100; + values[kLeaf - 1] = -10; + + const Rmq rmq{std::span(values)}; + const std::size_t left = kLeaf / 2; + EXPECT_EQ(rmq.arg_min(left, kLeaf), + naive_arg_min(std::span(values), left, kLeaf, + std::less())); + } + + { + std::vector values(kLeaf, 1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 7) % 31); + } + values[0] = -100; + values[kLeaf / 2] = -50; + values[kLeaf - 1] = -80; + + const Rmq rmq{std::span(values)}; + const std::size_t left = kLeaf / 2 - 20; + const std::size_t right = kLeaf / 2 + 21; + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())); + } +} + TEST(RmqHybridBTree, LeafSelectorEnumVariants) { using MaskRmq = pixie::rmq::HybridBTree< int, std::less, std::size_t, 248, 256, @@ -524,6 +575,62 @@ TEST(RmqHybridBTree, LeafSelectorEnumVariants) { } } +TEST(RmqHybridBTree, BorderCorrectionForLeafSelectorVariants) { + using DefaultRmq = pixie::rmq::HybridBTree; + using SmallMaskRmq = pixie::rmq::HybridBTree< + int, std::less, std::size_t, 248, 256, + pixie::rmq::HybridBTreeLeafSelector::PrefixSuffix>; + using BpRmq = + pixie::rmq::HybridBTree, std::size_t, 252, 256, + pixie::rmq::HybridBTreeLeafSelector::BP>; + + const auto check = []() { + constexpr std::size_t kLeaf = Rmq::kLeafSize; + + { + std::vector values(3 * kLeaf, 100000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 13 + i / 5) % 97); + } + values[0] = -100000; + values[17] = -100; + values[kLeaf + 23] = -1000; + values[2 * kLeaf + 31] = -500; + + const Rmq rmq{std::span(values)}; + const std::size_t left = 1; + const std::size_t right = values.size() - 1; + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())) + << "left-border correction leaf=" << kLeaf; + } + + { + std::vector values(3 * kLeaf, 100000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 11 + i / 7) % 89); + } + values[11] = -500; + values[kLeaf + 29] = -1000; + values[2 * kLeaf + 41] = -100; + values[values.size() - 1] = -100000; + + const Rmq rmq{std::span(values)}; + const std::size_t left = 1; + const std::size_t right = values.size() - 1; + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())) + << "right-border correction leaf=" << kLeaf; + } + }; + + check.template operator()(); + check.template operator()(); + check.template operator()(); +} + TEST(RmqHybridBTree, MiddleFanoutBoundaryRanges) { using Rmq = pixie::rmq::HybridBTree; constexpr std::size_t kLeaf = Rmq::kLeafSize; @@ -715,6 +822,47 @@ TEST(RmqCartesianHybridBTree, BoundarySizesAndBpEncoding) { } } +TEST(RmqCartesianHybridBTree, BorderMinimaOutsideQueryFallBackToMiddle) { + using Rmq = pixie::rmq::CartesianHybridBTree; + constexpr std::size_t kLeaf = 512; + + { + std::vector values(3 * kLeaf, 100000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 17 + i / 9) % 101); + } + values[0] = -100000; + values[13] = -100; + values[kLeaf + 37] = -1000; + values[2 * kLeaf + 53] = -500; + + const Rmq rmq{std::span(values)}; + const std::size_t left = 1; + const std::size_t right = values.size() - 1; + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())); + } + + { + std::vector values(3 * kLeaf, 100000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 19 + i / 11) % 107); + } + values[17] = -500; + values[kLeaf + 41] = -1000; + values[2 * kLeaf + 59] = -100; + values[values.size() - 1] = -100000; + + const Rmq rmq{std::span(values)}; + const std::size_t left = 1; + const std::size_t right = values.size() - 1; + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())); + } +} + TEST(RmqCartesianHybridBTree, DuplicateHeavyRandomDifferentialTo8193) { using Rmq = pixie::rmq::CartesianHybridBTree; std::mt19937_64 rng(20260610); diff --git a/src/tests/test_rmm.cpp b/src/tests/test_rmm.cpp index 52fa139..4e129ac 100644 --- a/src/tests/test_rmm.cpp +++ b/src/tests/test_rmm.cpp @@ -1152,6 +1152,62 @@ TEST(RmMBTreeExperimental, InvalidArgumentsGuards) { EXPECT_EQ(rm.enclose(bits.size()), pixie::experimental::RmMBTree<>::npos); } +TEST(RmMBTreeExperimental, PartialBlockSearchesAndTailMinSelect) { + std::string bits; + bits.reserve(385); + for (size_t i = 0; i < 385; ++i) { + bits.push_back((i % 7 == 0 || i % 11 == 3 || i % 29 == 5) ? '1' : '0'); + } + + auto words = pack_words_lsb_first(bits); + pixie::experimental::RmMBTree<> rm(std::span(words), + bits.size()); + NaiveRmM nv(bits); + + const std::array positions = { + 0, 1, 63, 64, 127, 128, 255, 320, bits.size() - 1, + }; + for (size_t position : positions) { + for (int delta : {-120, -17, -2, -1, 0, 1, 2, 17, 120}) { + SCOPED_TRACE(::testing::Message() + << "position=" << position << " delta=" << delta); + EXPECT_EQ(rm.fwdsearch(position, delta), nv.fwdsearch(position, delta)); + EXPECT_EQ(rm.bwdsearch(position + 1, delta), + nv.bwdsearch(position + 1, delta)); + } + } + + const std::array, 4> ranges = { + std::pair{0, bits.size() - 1}, + std::pair{3, 97}, + std::pair{64, 255}, + std::pair{257, bits.size() - 1}, + }; + for (const auto& [left, right] : ranges) { + SCOPED_TRACE(::testing::Message() + << "range=[" << left << "," << right << "]"); + EXPECT_EQ(rm.range_min_query_pos(left, right), + nv.range_min_query_pos(left, right)); + EXPECT_EQ(rm.range_min_query_val(left, right), + nv.range_min_query_val(left, right)); + EXPECT_EQ(rm.range_max_query_pos(left, right), + nv.range_max_query_pos(left, right)); + EXPECT_EQ(rm.range_max_query_val(left, right), + nv.range_max_query_val(left, right)); + EXPECT_EQ(rm.mincount(left, right), nv.mincount(left, right)); + + const size_t count = nv.mincount(left, right); + for (size_t rank : {size_t{1}, count, count + 1}) { + if (rank == 0) { + continue; + } + EXPECT_EQ(rm.minselect(left, right, rank), + nv.minselect(left, right, rank)) + << "rank=" << rank; + } + } +} + TEST(RmMBTreeExperimental, FwdBwdSearchAcrossHighLevels) { std::mt19937_64 rng(kSeed); const size_t n = 512 * 32 * 8 + 4096; From 4251f34e0dba6bac3ff69c6b4425088b8a12b589 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 20 Jun 2026 20:03:02 +0300 Subject: [PATCH 26/27] coverage --- AGENTS.md | 53 +++++++- agentic/local/cpp/skills/cmake/EXAMPLES.md | 73 +++++++++++ src/tests/rmq_tests.cpp | 141 +++++++++++++++++++++ 3 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 agentic/local/cpp/skills/cmake/EXAMPLES.md diff --git a/AGENTS.md b/AGENTS.md index 997d599..6c9acef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ When a task matches a skill, read: ### Project Layout Conventions - **`include/`**: Header-only library API (all implementations here, no `.cpp` files) -- **`include/misc/`**: Naive reference implementations for differential testing +- **`include/reference_implementations/`**: Naive reference implementations for differential testing - **`src/*_tests.cpp`**: Unit tests (Google Test) - **`src/*_benchmarks.cpp`**: Performance benchmarks (Google Benchmark) - **`src/docs/`**: Doxygen configuration @@ -118,9 +118,10 @@ cmake --build build/release -j ### Running Tests ```bash -cmake-build/release/unittests # BitVector tests -cmake-build/release/test_rmm # RmM Tree tests -cmake-build/release/louds_tree_tests # LOUDS Tree tests +./build/release/unittests # BitVector tests +./build/release/test_rmm # RmM Tree tests +./build/release/rmq_tests # RMQ tests +./build/release/louds_tree_tests # LOUDS Tree tests ``` ### Test Configuration via Environment Variables @@ -133,9 +134,51 @@ For `test_rmm.cpp`: ### Testing Patterns -- **Differential testing**: Compare against naive reference implementations in `include/misc/` +- **Differential testing**: Compare against naive reference implementations in `include/reference_implementations/` - **Randomized testing**: Random bit vectors and balanced parentheses sequences - **Exhaustive short inputs**: Test all patterns for small sizes +- **Shape-forcing tests**: For RMQ/RmM trees, deterministic tests that force + a specific traversal path are often more valuable than adding more random + cases. Target border correction, same-leaf paths, prefix/suffix leaf + selectors, top sparse-table hit/miss paths, partial last blocks, and + first-min tie cases. + +### Coverage and Codecov + +Use the repository script for local coverage: + +```bash +./scripts/coverage_report.sh +``` + +This script configures the `coverage` preset, builds coverage targets, deletes +old `.gcda`/`.gcov` files, runs the test binaries, and writes +`build/coverage/coverage.txt`. + +Coverage interpretation notes: + +- The report is generated with `gcov -pb`, so Codecov receives branch + probabilities in addition to line hits. Codecov "partial" lines are lines + that executed but did not cover all branch paths, for example short-circuit + conditions, ternaries, loop exits, or exception edges. +- Do not compare Codecov's branch-aware percentage directly with a plain + line-only local summary. Codecov effectively penalizes partially covered + branch lines. +- After recompiling an instrumented target, stale `.gcda` files can produce + `overwriting an existing profile data with a different checksum`. Run + `./scripts/coverage_report.sh` or delete the coverage `.gcda` files before + trusting a local coverage run. +- Header-only templates can appear in multiple gcov translation-unit reports. + For headers such as `experimental/rmm_btree.h`, inspect the generated + `.gcov` file or Codecov aggregate rather than trusting the first `File ...` + block in `coverage.txt`. +- For RMQ/RmM coverage work, prioritize public behavior paths over artificial + internal coverage. Useful tests cover HybridBTree leaf selector variants, + internal border correction, top sparse-overlay hit/miss behavior, + CartesianHybridBTree BP-depth query paths, and RmMBTree forward/backward + search/minselect edge cases. Low-value gaps include allocator failure paths, + `length_error` paths that require impossible index sizes, and private + defensive `npos` branches that cannot be reached through a valid public API. ## Code Style Guidelines diff --git a/agentic/local/cpp/skills/cmake/EXAMPLES.md b/agentic/local/cpp/skills/cmake/EXAMPLES.md new file mode 100644 index 0000000..95cae08 --- /dev/null +++ b/agentic/local/cpp/skills/cmake/EXAMPLES.md @@ -0,0 +1,73 @@ +# Pixie CMake Examples + +Use these notes with `agentic/cpp/skills/cmake/SKILL.md` for Pixie's local +presets and coverage workflow. + +## Coverage + +Prefer the repository script when checking coverage: + +```bash +./scripts/coverage_report.sh +``` + +The script uses the `coverage` preset, where Pixie's coverage option is +`PIXIE_COVERAGE=ON`, not the generic `ENABLE_COVERAGE=ON` spelling used in +some other CMake projects. It also clears old `.gcda` and `.gcov` files before +running tests, which avoids gcov checksum warnings after recompilation. + +The generated report is: + +```text +build/coverage/coverage.txt +``` + +Useful focused targets while iterating on coverage: + +```bash +cmake --build build/coverage --target rmq_tests -j +./build/coverage/rmq_tests + +cmake --build build/coverage --target test_rmm -j +./build/coverage/test_rmm +``` + +Run the full script before reporting final coverage numbers, because individual +test-binary runs can leave stale profile counters when source checksums change. + +## Codecov Interpretation + +Pixie uploads `gcov -pb` output to Codecov. The `-b` flag includes branch +probabilities, so Codecov reports "partial" coverage for executed lines whose +branch paths were not all covered. A local line-only summary can therefore look +better than the Codecov project or patch percentage. + +When a PR coverage check looks unexpectedly low: + +1. Check the Codecov file table for missing and partial counts. +2. Check `scripts/coverage_report.sh` and `.github/workflows/coverage.yml` to + confirm the uploaded file and flags. +3. For header-only templates, remember that the same header may appear in more + than one translation-unit gcov report. Inspect the generated `.gcov` file or + Codecov aggregate before drawing conclusions from the first `coverage.txt` + block. + +## RMQ/RmM Coverage Strategy + +Random differential tests are necessary but not sufficient for Pixie's RMQ and +RmM structures. The best coverage improvements usually come from deterministic +input shapes that force a specific traversal branch: + +- HybridBTree leaf selector paths: prefix answer, suffix answer, embedded leaf + minimum answer, and unsupported interior range fallback. +- HybridBTree and CartesianHybridBTree internal query paths where the local + selector chooses a border child whose subtree minimum is outside the queried + range, forcing border recursion plus middle-child comparison. +- Top sparse-overlay paths where the padded sparse candidate is inside the + query and where it misses and delegates borders to the tree. +- RmMBTree partial last-block scans, forward/backward search no-hit cases, + backward boundary-only returns, and `minselect` ranks across node boundaries. + +Do not chase every uncovered branch. Low-value coverage includes allocator +failure branches, impossible index-size `length_error` paths, and private +defensive `npos` checks that cannot be reached through valid public API calls. diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp index b5a5006..6e12d55 100644 --- a/src/tests/rmq_tests.cpp +++ b/src/tests/rmq_tests.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -73,6 +74,43 @@ bool packed_bit(std::span words, std::size_t position) { return ((words[position >> 6] >> (position & 63)) & 1u) != 0; } +std::vector make_packed_delta_bits(std::size_t bit_count) { + std::vector words((bit_count + 63) / 64); + for (std::size_t position = 0; position < bit_count; ++position) { + const bool bit = ((position * 11 + position / 3) % 17) < 8; + if (bit) { + words[position >> 6] |= std::uint64_t{1} << (position & 63); + } + } + return words; +} + +std::vector depths_from_delta_bits( + std::span words, + std::size_t depth_count) { + std::vector depths(depth_count); + for (std::size_t position = 1; position < depth_count; ++position) { + depths[position] = + depths[position - 1] + (packed_bit(words, position - 1) ? 1 : -1); + } + return depths; +} + +std::size_t naive_depth_arg_min(std::span depths, + std::size_t left, + std::size_t right) { + if (left >= right || right > depths.size()) { + return std::numeric_limits::max(); + } + std::size_t best = left; + for (std::size_t position = left + 1; position < right; ++position) { + if (depths[position] < depths[best]) { + best = position; + } + } + return best; +} + template void expect_valid_bp_encoding(const Rmq& rmq, std::size_t value_count) { const std::size_t bit_count = rmq.bp_bit_count(); @@ -547,12 +585,19 @@ TEST(RmqHybridBTree, LeafSelectorEnumVariants) { const MaskRmq mask_rmq{std::span(values)}; const BpRmq bp_rmq{std::span(values)}; + const std::vector empty_values; const MaskRmq empty_mask_rmq; const BpRmq empty_bp_rmq; + const MaskRmq empty_mask_span_rmq{std::span(empty_values)}; + const BpRmq empty_bp_span_rmq{std::span(empty_values)}; EXPECT_EQ(empty_mask_rmq.arg_min(0, 0), MaskRmq::npos); EXPECT_EQ(empty_bp_rmq.arg_min(0, 0), BpRmq::npos); + EXPECT_EQ(empty_mask_span_rmq.arg_min(0, 0), MaskRmq::npos); + EXPECT_EQ(empty_bp_span_rmq.arg_min(0, 0), BpRmq::npos); EXPECT_EQ(empty_mask_rmq.range_min(0, 0), 0); EXPECT_EQ(empty_bp_rmq.range_min(0, 0), 0); + EXPECT_EQ(empty_mask_span_rmq.range_min(0, 0), 0); + EXPECT_EQ(empty_bp_span_rmq.range_min(0, 0), 0); const std::vector> ranges = { {0, 1}, {0, 13}, {0, 252}, {1, 251}, @@ -573,6 +618,44 @@ TEST(RmqHybridBTree, LeafSelectorEnumVariants) { EXPECT_EQ(bp_rmq.range_min(left, right), values[expected]) << "bp range=[" << left << "," << right << ")"; } + + const auto check_small_tree_paths = []() { + constexpr std::size_t kLeaf = Rmq::kLeafSize; + std::vector one_leaf_values(kLeaf / 2 + 3, 1000); + for (std::size_t i = 0; i < one_leaf_values.size(); ++i) { + one_leaf_values[i] += static_cast((i * 17 + i / 5) % 71); + } + one_leaf_values[one_leaf_values.size() / 2] = -20; + const Rmq one_leaf_rmq{std::span(one_leaf_values)}; + EXPECT_EQ(one_leaf_rmq.arg_min(0, one_leaf_values.size()), + naive_arg_min(std::span(one_leaf_values), 0, + one_leaf_values.size(), std::less())) + << "one leaf path leaf=" << kLeaf; + + std::vector small_values(3 * kLeaf + 5, 10000); + for (std::size_t i = 0; i < small_values.size(); ++i) { + small_values[i] += static_cast((i * 41 + i / 9) % 263); + } + small_values[7] = -100; + small_values[kLeaf + 3] = -500; + small_values[2 * kLeaf + 11] = -300; + + const Rmq rmq{std::span(small_values)}; + const std::vector> small_ranges = { + {0, small_values.size()}, {0, kLeaf}, + {kLeaf, 2 * kLeaf}, {2 * kLeaf, 3 * kLeaf}, + {1, small_values.size() - 1}, {kLeaf + 1, kLeaf + 2}, + }; + for (const auto [left, right] : small_ranges) { + const std::size_t expected = naive_arg_min( + std::span(small_values), left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "small tree range=[" << left << "," << right << ") leaf=" << kLeaf; + } + }; + + check_small_tree_paths.template operator()(); + check_small_tree_paths.template operator()(); } TEST(RmqHybridBTree, BorderCorrectionForLeafSelectorVariants) { @@ -753,6 +836,28 @@ TEST(RmqHybridBTree, TopSparseOverlayComparatorMaximum) { } } +TEST(RmqHybridBTree, InternalRootFullRangeShortcut) { + using Rmq = pixie::rmq::HybridBTree; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + + std::vector values(7 * kLeaf + 19, 10000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 29 + i / 13) % 257); + } + values[17] = -100; + values[3 * kLeaf + 11] = -700; + values[values.size() - 3] = -200; + + const Rmq rmq{std::span(values)}; + EXPECT_EQ(rmq.arg_min(0, values.size()), 3 * kLeaf + 11); + EXPECT_EQ(rmq.range_min(0, values.size()), -700); + EXPECT_EQ(rmq.arg_min(0, kLeaf), naive_arg_min(std::span(values), + 0, kLeaf, std::less())); + EXPECT_EQ(rmq.arg_min(kLeaf - 3, 2 * kLeaf + 5), + naive_arg_min(std::span(values), kLeaf - 3, + 2 * kLeaf + 5, std::less())); +} + TEST(RmqHybridBTree, DuplicateHeavyRandomTo8193) { using Rmq = pixie::rmq::HybridBTree; std::mt19937_64 rng(901496); @@ -896,6 +1001,42 @@ TEST(RmqCartesianHybridBTree, DuplicateHeavyRandomDifferentialTo8193) { } } +TEST(RmqCartesianHybridBTree, DepthBackendDirectPaths) { + using HighSparseBackend = + pixie::rmq::detail::HybridBTreePlusMinusOne; + constexpr std::size_t kDepthCount = 3 * 512 + 17; + + const std::vector words = + make_packed_delta_bits(kDepthCount - 1); + const std::vector depths = depths_from_delta_bits( + std::span(words), kDepthCount); + const HighSparseBackend high_sparse(std::span(words), + kDepthCount); + + EXPECT_EQ(high_sparse.arg_min(kDepthCount, kDepthCount), + HighSparseBackend::npos); + EXPECT_EQ(high_sparse.arg_min(37, 38), 37u); + EXPECT_GT(high_sparse.memory_usage_bytes(), sizeof(HighSparseBackend)); + + const std::vector> ranges = { + {0, kDepthCount}, + {0, 512}, + {511, 1027}, + {9, 1400}, + {700, kDepthCount - 3}, + {kDepthCount - 65, kDepthCount}, + }; + for (const auto [left, right] : ranges) { + const std::size_t expected = + naive_depth_arg_min(std::span(depths), left, right); + EXPECT_EQ(high_sparse.arg_min(left, right), expected) + << "high sparse depth range=[" << left << "," << right << ")"; + } + + EXPECT_THROW(HighSparseBackend(std::span(), 65), + std::invalid_argument); +} + TEST(RmqCartesianHybridBTree, BpStorageIsCacheLineAligned) { using Rmq = pixie::rmq::CartesianHybridBTree; constexpr std::size_t kLeafWords = 512 / 64; From 1917f200f9b543ad913d1e4d81cd62b710265f86 Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Sat, 20 Jun 2026 21:05:47 +0300 Subject: [PATCH 27/27] coverage --- src/tests/test_rmm.cpp | 75 +++++++++++++++++++++++++++++++++++++++++ src/tests/unittests.cpp | 36 ++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/src/tests/test_rmm.cpp b/src/tests/test_rmm.cpp index 4e129ac..dc0d7d8 100644 --- a/src/tests/test_rmm.cpp +++ b/src/tests/test_rmm.cpp @@ -1090,6 +1090,12 @@ TEST(RmMBTreeExperimental, OptionalSelectSupport) { pixie::experimental::RmMBTree<> select0_only( std::span(words), bits.size(), pixie::BitVector::SelectSupport::kSelect0, /*one_count=*/3); + pixie::experimental::RmMBTree<> select1_only( + std::span(words), bits.size(), + pixie::BitVector::SelectSupport::kSelect1, /*one_count=*/3); + pixie::experimental::RmMBTree<> rank_only( + std::span(words), bits.size(), + pixie::BitVector::SelectSupport::kNone, /*one_count=*/3); EXPECT_EQ(select0_only.rank1(bits.size()), 3u); EXPECT_EQ(select0_only.rank0(bits.size()), 3u); @@ -1098,6 +1104,75 @@ TEST(RmMBTreeExperimental, OptionalSelectSupport) { EXPECT_EQ(select0_only.select0(2), 4u); EXPECT_EQ(select0_only.select0(3), 5u); EXPECT_EQ(select0_only.select0(4), pixie::experimental::RmMBTree<>::npos); + + EXPECT_EQ(select1_only.rank1(bits.size()), 3u); + EXPECT_EQ(select1_only.rank0(bits.size()), 3u); + EXPECT_EQ(select1_only.select1(1), 0u); + EXPECT_EQ(select1_only.select1(2), 2u); + EXPECT_EQ(select1_only.select1(3), 3u); + EXPECT_EQ(select1_only.select1(4), pixie::experimental::RmMBTree<>::npos); + EXPECT_EQ(select1_only.select0(1), pixie::experimental::RmMBTree<>::npos); + + EXPECT_EQ(rank_only.rank1(bits.size()), 3u); + EXPECT_EQ(rank_only.rank0(bits.size()), 3u); + EXPECT_EQ(rank_only.select1(1), pixie::experimental::RmMBTree<>::npos); + EXPECT_EQ(rank_only.select0(1), pixie::experimental::RmMBTree<>::npos); +} + +TEST(RmMBTreeExperimental, RangeMinResultAndMemoryUsage) { + using Tree = pixie::experimental::RmMBTree<>; + + Tree empty; + EXPECT_GE(empty.memory_usage_bytes(), sizeof(Tree)); + EXPECT_EQ(empty.size(), 0u); + EXPECT_EQ(empty.rank1(7), 0u); + EXPECT_EQ(empty.rank0(7), 0u); + EXPECT_EQ(empty.select1(1), Tree::npos); + EXPECT_EQ(empty.select0(1), Tree::npos); + EXPECT_EQ(empty.excess(7), 0); + EXPECT_EQ(empty.range_min_query_pos(0, 0), Tree::npos); + EXPECT_EQ(empty.range_min_query_val(0, 0), 0); + EXPECT_EQ(empty.range_max_query_pos(0, 0), Tree::npos); + EXPECT_EQ(empty.range_max_query_val(0, 0), 0); + EXPECT_EQ(empty.mincount(0, 0), 0u); + EXPECT_EQ(empty.minselect(0, 0, 1), Tree::npos); + + std::string bits; + bits.reserve(512 * 40 + 37); + for (size_t i = 0; i < 512 * 40 + 37; ++i) { + bits.push_back(((i * 19 + i / 13) % 9) < 4 ? '1' : '0'); + } + + auto words = pack_words_lsb_first(bits); + Tree rm(std::span(words), bits.size()); + NaiveRmM nv(bits); + + EXPECT_GT(rm.memory_usage_bytes(), empty.memory_usage_bytes()); + + const std::array, 6> ranges = { + std::pair{0, bits.size() - 1}, + std::pair{3, 97}, + std::pair{511, 512 * 3 + 7}, + std::pair{512, 512 * 32 + 11}, + std::pair{512 * 31 - 5, 512 * 40 + 7}, + std::pair{bits.size() - 19, bits.size() - 1}, + }; + + for (const auto& [left, right] : ranges) { + SCOPED_TRACE(::testing::Message() + << "range=[" << left << "," << right << "]"); + const auto result = rm.range_min_query_result(left, right); + EXPECT_EQ(result.position, nv.range_min_query_pos(left, right)); + EXPECT_EQ(result.value, nv.range_min_query_val(left, right)); + } + + const auto reversed = rm.range_min_query_result(3, 2); + EXPECT_EQ(reversed.position, Tree::npos); + EXPECT_EQ(reversed.value, 0); + + const auto out_of_bounds = rm.range_min_query_result(0, bits.size()); + EXPECT_EQ(out_of_bounds.position, Tree::npos); + EXPECT_EQ(out_of_bounds.value, 0); } TEST(RmMBTreeExperimental, ParenthesesOnUnmatchedBoundaryBits) { diff --git a/src/tests/unittests.cpp b/src/tests/unittests.cpp index 935b972..54b0dd4 100644 --- a/src/tests/unittests.cpp +++ b/src/tests/unittests.cpp @@ -434,6 +434,14 @@ TEST(BitVectorTest, OptionalSelectSupport) { EXPECT_EQ(large_select1_only.select(32768), 32767); } +TEST(BitVectorTest, ThrowsWhenOneCountHintUnderestimatesSamples) { + std::vector one_words(512, ~uint64_t{0}); + + EXPECT_THROW((BitVector(one_words, one_words.size() * 64, + BitVector::SelectSupport::kSelect1, 0)), + std::invalid_argument); +} + TEST(BitVectorTest, ShortSpanUsesScalarRankSelectFallbacks) { std::vector one_in_second_word = {0, 1}; BitVector ones(std::span(one_in_second_word), 65); @@ -451,6 +459,16 @@ TEST(BitVectorTest, ShortSpanUsesScalarRankSelectFallbacks) { EXPECT_EQ(zeros.select0(1), 64u); } +TEST(BitVectorTest, ShortSpanScalarRankUsesPartialWordTail) { + std::vector bits = {~uint64_t{0}, 0, 0b101}; + BitVector bv(std::span(bits), 130); + + EXPECT_EQ(bv.rank(128), 64u); + EXPECT_EQ(bv.rank(129), 65u); + EXPECT_EQ(bv.rank(130), 65u); + EXPECT_EQ(bv.rank0(129), 64u); +} + TEST(BitVectorTest, IgnoresDirtyPaddingAndTrailingWords) { std::vector bits = {~uint64_t{0}, ~uint64_t{0}}; BitVector bv(bits, 3); @@ -466,6 +484,24 @@ TEST(BitVectorTest, IgnoresDirtyPaddingAndTrailingWords) { EXPECT_EQ(bv.select0(1), bv.size()); } +TEST(BitVectorTest, SelectZeroSkewedDistributionFallback) { + constexpr size_t kWordsPerBasicBlock = 8; + constexpr size_t kBasicBlocksPerSuperBlock = 128; + constexpr size_t kZeroBasicBlocks = 40; + + std::vector bits(kWordsPerBasicBlock * kBasicBlocksPerSuperBlock, + ~uint64_t{0}); + std::fill(bits.begin(), bits.begin() + kZeroBasicBlocks * kWordsPerBasicBlock, + 0); + + BitVector bv(std::span(bits), bits.size() * 64); + EXPECT_EQ(bv.rank0(bv.size()), kZeroBasicBlocks * 512u); + EXPECT_EQ(bv.select0(1), 0u); + EXPECT_EQ(bv.select0(10000), 9999u); + EXPECT_EQ(bv.select0(kZeroBasicBlocks * 512u), kZeroBasicBlocks * 512u - 1); + EXPECT_EQ(bv.select0(kZeroBasicBlocks * 512u + 1), bv.size()); +} + TEST(BitVectorTest, MainRankZeroTest) { std::mt19937_64 rng(42); std::vector bits(65536 * 32);