diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..00371e1 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "agentic/cpp"] + path = agentic/cpp + url = https://github.com/Malkovsky/ai_for_cpp.git diff --git a/AGENTS.md b/AGENTS.md index 4078e17..6c9acef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ When a task matches a skill, read: ### Project Layout Conventions - **`include/`**: Header-only library API (all implementations here, no `.cpp` files) -- **`include/misc/`**: Naive reference implementations for differential testing +- **`include/reference_implementations/`**: Naive reference implementations for differential testing - **`src/*_tests.cpp`**: Unit tests (Google Test) - **`src/*_benchmarks.cpp`**: Performance benchmarks (Google Benchmark) - **`src/docs/`**: Doxygen configuration @@ -36,6 +36,11 @@ When a task matches a skill, read: 3. **SIMD conditional compilation**: Uses `#ifdef PIXIE_AVX512_SUPPORT` / `PIXIE_AVX2_SUPPORT` with scalar fallbacks. 4. **Target domain**: Optimized for data sizes up to 2^64 bits. 5. **Platform**: Linux/Unix is the primary target platform. +6. **Interval convention**: Public APIs, tests, and benchmarks use C++-style + half-open ranges `[left, right)`: `left` is included and `right` is excluded. + Empty ranges (`left == right`) are invalid unless an API explicitly documents + otherwise. Low-level primitives may still use their own documented interval + conventions. ### Why Header-Only? @@ -113,9 +118,10 @@ cmake --build build/release -j ### Running Tests ```bash -cmake-build/release/unittests # BitVector tests -cmake-build/release/test_rmm # RmM Tree tests -cmake-build/release/louds_tree_tests # LOUDS Tree tests +./build/release/unittests # BitVector tests +./build/release/test_rmm # RmM Tree tests +./build/release/rmq_tests # RMQ tests +./build/release/louds_tree_tests # LOUDS Tree tests ``` ### Test Configuration via Environment Variables @@ -128,9 +134,51 @@ For `test_rmm.cpp`: ### Testing Patterns -- **Differential testing**: Compare against naive reference implementations in `include/misc/` +- **Differential testing**: Compare against naive reference implementations in `include/reference_implementations/` - **Randomized testing**: Random bit vectors and balanced parentheses sequences - **Exhaustive short inputs**: Test all patterns for small sizes +- **Shape-forcing tests**: For RMQ/RmM trees, deterministic tests that force + a specific traversal path are often more valuable than adding more random + cases. Target border correction, same-leaf paths, prefix/suffix leaf + selectors, top sparse-table hit/miss paths, partial last blocks, and + first-min tie cases. + +### Coverage and Codecov + +Use the repository script for local coverage: + +```bash +./scripts/coverage_report.sh +``` + +This script configures the `coverage` preset, builds coverage targets, deletes +old `.gcda`/`.gcov` files, runs the test binaries, and writes +`build/coverage/coverage.txt`. + +Coverage interpretation notes: + +- The report is generated with `gcov -pb`, so Codecov receives branch + probabilities in addition to line hits. Codecov "partial" lines are lines + that executed but did not cover all branch paths, for example short-circuit + conditions, ternaries, loop exits, or exception edges. +- Do not compare Codecov's branch-aware percentage directly with a plain + line-only local summary. Codecov effectively penalizes partially covered + branch lines. +- After recompiling an instrumented target, stale `.gcda` files can produce + `overwriting an existing profile data with a different checksum`. Run + `./scripts/coverage_report.sh` or delete the coverage `.gcda` files before + trusting a local coverage run. +- Header-only templates can appear in multiple gcov translation-unit reports. + For headers such as `experimental/rmm_btree.h`, inspect the generated + `.gcov` file or Codecov aggregate rather than trusting the first `File ...` + block in `coverage.txt`. +- For RMQ/RmM coverage work, prioritize public behavior paths over artificial + internal coverage. Useful tests cover HybridBTree leaf selector variants, + internal border correction, top sparse-overlay hit/miss behavior, + CartesianHybridBTree BP-depth query paths, and RmMBTree forward/backward + search/minselect edge cases. Low-value gaps include allocator failure paths, + `length_error` paths that require impossible index sizes, and private + defensive `npos` branches that cannot be reached through a valid public API. ## Code Style Guidelines diff --git a/CMakeLists.txt b/CMakeLists.txt index 87c59ad..d932a40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,6 +183,28 @@ if(PIXIE_TESTS) gtest gtest_main ${PIXIE_DIAGNOSTICS_LIBS}) + + add_executable(excess_record_lows_tests + src/tests/excess_record_lows_tests.cpp) + target_include_directories(excess_record_lows_tests + PUBLIC include) + target_link_libraries(excess_record_lows_tests + gtest + gtest_main + ${PIXIE_DIAGNOSTICS_LIBS}) + + add_executable(rmq_tests + src/tests/rmq_tests.cpp) + target_include_directories(rmq_tests + PUBLIC include) + target_link_libraries(rmq_tests + gtest + gtest_main + ${PIXIE_DIAGNOSTICS_LIBS}) + if(PIXIE_THIRD_PARTY_BACKENDS) + target_include_directories(rmq_tests + PRIVATE ${sdsl_lite_SOURCE_DIR}/include) + endif() endif() # --------------------------------------------------------------------------- @@ -218,6 +240,20 @@ if(PIXIE_BENCHMARKS) benchmark ${PIXIE_DIAGNOSTICS_LIBS}) + add_executable(bench_rmq + src/benchmarks/bench_rmq.cpp) + target_include_directories(bench_rmq + PUBLIC include) + target_link_libraries(bench_rmq + benchmark + ${PIXIE_DIAGNOSTICS_LIBS}) + if(PIXIE_THIRD_PARTY_BACKENDS) + target_include_directories(bench_rmq + PRIVATE ${sdsl_lite_SOURCE_DIR}/include) + target_compile_definitions(bench_rmq + PRIVATE PIXIE_THIRD_PARTY_BENCHMARKS) + endif() + if(PIXIE_THIRD_PARTY_BACKENDS) add_executable(bench_rmm_sdsl src/benchmarks/bench_rmm_sdsl.cpp) diff --git a/CMakePresets.json b/CMakePresets.json index 5da7763..c8e31cf 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -112,6 +112,7 @@ "targets": [ "unittests", "benchmark_tests", + "rmq_tests", "test_rmm", "louds_tree_tests", "dfuds_tree_tests", @@ -125,6 +126,7 @@ "targets": [ "unittests", "benchmark_tests", + "rmq_tests", "test_rmm", "louds_tree_tests", "dfuds_tree_tests", @@ -138,6 +140,7 @@ "targets": [ "benchmarks", "bench_rmm", + "bench_rmq", "louds_tree_benchmarks", "dfuds_tree_benchmarks", "alignment_comparison", @@ -152,6 +155,7 @@ "benchmarks", "bench_rmm", "bench_rmm_sdsl", + "bench_rmq", "louds_tree_benchmarks", "dfuds_tree_benchmarks", "alignment_comparison", @@ -165,6 +169,7 @@ "targets": [ "benchmarks", "bench_rmm", + "bench_rmq", "louds_tree_benchmarks", "dfuds_tree_benchmarks", "alignment_comparison", @@ -186,6 +191,7 @@ "targets": [ "unittests", "benchmark_tests", + "rmq_tests", "test_rmm", "louds_tree_tests", "dfuds_tree_tests", @@ -199,6 +205,7 @@ "targets": [ "unittests", "benchmark_tests", + "rmq_tests", "test_rmm", "louds_tree_tests", "dfuds_tree_tests", diff --git a/README.md b/README.md index 9537bfe..56ad227 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,122 @@ python3 scripts/excess_benchmark_table.py excess_positions.json -o src/docs/exce Generated benchmark documentation can be written to `src/docs/benchmark_results.md`; the documentation pipeline does not run benchmarks. +### Adding an RMQ Benchmark + +Value RMQ implementations are benchmarked through the common CRTP interface in +`pixie::rmq::RmqBase`. To add a comparable backend, implement a non-owning index +that can be constructed from `std::span` and provides: + +* `size_impl()` +* `arg_min_impl(left, right)` for half-open ranges `[left, right)` +* `value_at_impl(position)` + +The public `size()`, `empty()`, `arg_min()`, and `range_min()` methods are then +provided by `RmqBase`. Ties should return the smaller original position. + +Minimal example: + +```cpp +#include + +#include +#include +#include + +namespace pixie::rmq { + +template > +class LinearRmq : public RmqBase, T> { + public: + using Self = LinearRmq; + static constexpr std::size_t npos = RmqBase::npos; + + explicit LinearRmq(std::span values, Compare compare = Compare()) + : values_(values), compare_(compare) {} + + std::size_t size_impl() const { return values_.size(); } + + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size()) { + return npos; + } + std::size_t best = left; + for (std::size_t i = left + 1; i < right; ++i) { + if (compare_(values_[i], values_[best])) { + best = i; + } + } + return best; + } + + T value_at_impl(std::size_t position) const { return values_[position]; } + + private: + std::span values_; + Compare compare_; +}; + +} // namespace pixie::rmq +``` + +Then add rows to `src/benchmarks/bench_rmq.cpp` in `register_benchmarks()`. +Use `run_value_rmq_build` for construction cost and `run_queries` for query +cost: + +```cpp +benchmark::RegisterBenchmark( + "rmq_build_linear", + &run_value_rmq_build>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond); + +benchmark::RegisterBenchmark( + "rmq_linear", + &run_queries>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond); +``` + +The RMQ benchmark harness rotates through several value arrays so results are +less dependent on the global-minimum position. The `width` argument is the +maximum query width, not an exact width. + +To compare the new backend with `HybridBTree`, run both benchmark families +with a Google Benchmark filter. For example, after registering the new backend +as `rmq_linear` and `rmq_build_linear`: + +```sh +./build/release/bench_rmq \ + --benchmark_filter='^(rmq_linear|rmq_hybrid_btree)/(4194304|16777216)/(64|4096|262144|4194304|16777216)$' + +./build/release/bench_rmq \ + --benchmark_filter='^(rmq_build_linear|rmq_build_hybrid_btree)/(262144|4194304|16777216)$' +``` + +The first command compares query time for `2^22` and `2^24` input sizes across +the common RMQ widths. The second command compares construction time for the +same implementations. + +For hardware counters, use the diagnostic preset, which builds Google Benchmark +with libpfm support: + +```sh +cmake --preset benchmarks-diagnostic +cmake --build --preset benchmarks-diagnostic -j + +./build/release-with-deb/bench_rmq \ + --benchmark_filter='rmq_cartesian_hybrid_btree/67108864/4096' \ + --benchmark_perf_counters=CYCLES,INSTRUCTIONS,CACHE-MISSES \ + --benchmark_counters_tabular=true +``` + +Counter names are platform/libpfm dependent. Google Benchmark pauses timing and +perf counters during `state.PauseTiming()`, so RMQ dataset-variant rebuilds are +excluded from query counter rows. + ### RmM Tree ```sh diff --git a/agentic/cpp b/agentic/cpp new file mode 160000 index 0000000..12a959e --- /dev/null +++ b/agentic/cpp @@ -0,0 +1 @@ +Subproject commit 12a959e4883e453c72bcb5f5c4f5dfef0a9ab516 diff --git a/agentic/cpp/README.md b/agentic/cpp/README.md deleted file mode 100644 index 68a2262..0000000 --- a/agentic/cpp/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Shared C++ Agent Skills - -This subtree contains reusable C++ agent skills and related commands. - -Keep this tree generic: - -- Do not add project-specific benchmark names, CMake options, or paths. -- Keep reusable scripts beside the skills that use them. -- Put project-specific examples in the consuming repository under - `agentic/local/cpp/skills//EXAMPLES.md`. - -When using a skill in a project, read: - -1. `agentic/cpp/skills//SKILL.md` -2. `agentic/local/cpp/skills//EXAMPLES.md`, if present diff --git a/agentic/cpp/commands/README.md b/agentic/cpp/commands/README.md deleted file mode 100644 index 2a24b20..0000000 --- a/agentic/cpp/commands/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Shared C++ Agent Commands - -Reusable command definitions for C++ projects belong here. - -Keep project-specific commands in the consuming repository under -`agentic/local/cpp/commands`. diff --git a/agentic/cpp/commands/benchmarks-affected.md b/agentic/cpp/commands/benchmarks-affected.md deleted file mode 100644 index 158cb9e..0000000 --- a/agentic/cpp/commands/benchmarks-affected.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -description: Scan current branch and report impacted benchmark targets/functions. ---- - -# Benchmarks Affected - -Identify which benchmark binaries and benchmark functions are affected by changes on the current branch. - -Use the `benchmarks-affected` skill as the single source of truth for workflow details and guardrails. -Do not duplicate or override the skill instructions in this command. - -## Inputs - -- Optional `--baseline ` (default: `main`) -- Optional `--compile-commands ` -- Optional `--no-include-working-tree` -- Optional `--format ` (default: `text`) - -## Workflow - -1. Execute the `benchmarks-affected` skill workflow. -2. Pass through command inputs to the analyzer invocation defined by the skill. -3. Report results with these sections: - - Changed files - - Affected benchmark targets - - Affected benchmark functions - - Suggested `--benchmark_filter` regex - - Any warnings/failures - -## Output rules - -1. If `affected_benchmarks` is non-empty, prioritize those names. -2. If `affected_benchmarks` is empty but benchmark targets are affected, mark result as partial and include target-level impact. -3. Do not run full benchmark suites in this command; this command is for impact discovery only. diff --git a/agentic/cpp/commands/perf-review.md b/agentic/cpp/commands/perf-review.md deleted file mode 100644 index 8ef1865..0000000 --- a/agentic/cpp/commands/perf-review.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -description: Benchmark-driven PR performance review versus target branch ---- - -# Perf Review Workflow - -You are performing a performance review for the current PR branch. - -Non-negotiable requirements: -1. Benchmark timing plus profiling data is the highest-priority judgment tool. -2. Compare source branch versus target branch and report relevant benchmark metric changes. -3. Provide analysis and a final verdict: does the PR improve performance or not. - -## Inputs - -- Optional argument `--target `: target branch override. -- Optional argument `--filter `: benchmark filter regex. -- Optional argument `--no-counters`: disable hardware-counter collection. - -If arguments are omitted: -- Default target branch to PR base branch from `gh pr view --json baseRefName` when available. -- Fall back target branch to `main`. - -Filter handling: -- If `--filter` is provided, pass it through. -- Else use the filter produced by `benchmarks-affected` through `benchmarks-compare-revisions`. -- If no filter can be derived, run conservative full-binary compare for impacted binaries. - -## Step 1 - Resolve branches and hashes - -1. Resolve contender from current checkout (`HEAD`) and compute short hash. -2. Resolve baseline branch using precedence: `--target` -> PR base from `gh pr view --json baseRefName` -> `main`. -3. Resolve baseline short hash. -4. Print branch/hash mapping before benchmark execution. - -## Step 2 - Run timing and hardware-counter comparison via skill (single source of truth) - -Use `benchmarks-compare-revisions` as the single source of truth for revision builds, benchmark scope, compare.py flow, retry policy, and guardrails. - -Pass-through inputs: -- Baseline ref/hash from Step 1. -- Contender ref/hash from Step 1. -- Optional `--filter` override. -- Counter mode: default on (`COLLECT_COUNTERS=1`) on Linux, disabled when `--no-counters` is provided. - -Consume outputs from `benchmarks-compare-revisions`: -- Baseline and contender benchmark JSON artifacts. -- compare.py output per binary. -- Effective filter used. -- Scope metadata from `benchmarks-affected` (`affected_benchmark_targets`, `affected_benchmarks`) when available. -- `counters_available` status and, when unavailable, explicit reason. -- Baseline and contender counter JSON artifacts (when available). -- Derived counter metrics per benchmark (IPC, cache miss rate, branch mispredict rate). -- Counter anomaly list and ready-to-embed counter summary table. - -Execution guardrails: -- Run benchmarks sequentially. -- No background jobs (`nohup`, `&`). -- Use Release timing builds only. -- If timing comparison fails, return blocked verdict with exact failure points. - -## Step 3 - Consume delegated hardware-counter outputs - -Hardware-counter collection is delegated to `benchmarks-compare-revisions`. - -Pass-through inputs: -- `COLLECT_COUNTERS=1` by default on Linux (unless `--no-counters` is provided). -- Same baseline/contender refs and effective filter used in Step 2. - -Consume outputs: -- Counter preflight result. -- Counter JSON artifacts for both revisions. -- Derived metrics (IPC, cache miss rate, branch mispredict rate). -- Anomaly list and counter summary table for report embedding. - -If counters are unavailable (`counters_available=false`), continue with timing-only review and explicitly mark profiling as unavailable in the report. - -## Step 4 - Analyze timing and counter data - -Timing classification per benchmark entry: -- Improvement: time delta < -5% -- Regression: time delta > +5% -- Neutral: between -5% and +5% - -Aggregate per binary: -- Number of improvements/regressions/neutral -- Net average percentage change -- Largest regression and largest improvement - -Counter correlation: -- Use skill-provided hardware counter summary and anomaly list to explain major timing changes. -- Do not recompute derived counter metrics in this command. - -Judgment priority: -- Base verdict primarily on benchmark timing comparison. -- Use counter data as explanatory evidence and confidence signal. - -Noise-control expectations: -- Include at least one control benchmark family expected to be unaffected by the code change. -- Treat isolated swings without pattern as noise unless reproduced across related sizes/fill ratios. - -## Step 5 - Produce final markdown report - -Return a structured markdown report with this shape: - -```markdown -## Performance Review: vs - -### Configuration -- Baseline: () -- Contender: () -- Platform: -- Benchmarks run: -- Filter: -- Hardware counters: available / unavailable - -### Timing Summary -| Binary | Improvements | Regressions | Neutral | Net Change | -|---|---:|---:|---:|---:| -| ... | N | N | N | +/-X% | - -### Detailed Timing Results - - -### Hardware Counter Profile (if available) -| Benchmark | IPC (base->new) | Cache Miss Rate (base->new) | Branch Mispredict (base->new) | -|---|---:|---:|---:| -| ... | X.XX -> Y.YY | A.A% -> B.B% | C.C% -> D.D% | - -### Key Findings -- -- - -### Verdict -**[IMPROVES PERFORMANCE | REGRESSES PERFORMANCE | NO SIGNIFICANT CHANGE]** - -<1-2 sentence justification grounded in benchmark metrics, with profiling context if available> -``` - -Verdict rules: -- `IMPROVES PERFORMANCE`: improvements outnumber regressions, no severe regression (>10%), and net average change is favorable. -- `REGRESSES PERFORMANCE`: any severe regression (>10%) or regressions dominate with net unfavorable average. -- `NO SIGNIFICANT CHANGE`: mostly neutral changes or mixed results that approximately cancel out. - -## Failure Handling - -- If required builds fail or timing comparison cannot run, output a blocked review with exact failure points and no misleading verdict. -- If only profiling fails (`counters_available=false` from delegated skill output), continue with timing-based verdict and explicitly list profiling limitation. -- If JSON output is invalid/truncated, discard it and rerun that benchmark command once with tighter filter and explicit output redirection. diff --git a/agentic/cpp/commands/ping.md b/agentic/cpp/commands/ping.md deleted file mode 100644 index b5edaf7..0000000 --- a/agentic/cpp/commands/ping.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -description: Test command that replies with pong ---- - -Respond with exactly `pong`. -Do not add any other words. -Do not add quotes or punctuation. diff --git a/agentic/cpp/skills/benchmarks-affected/SKILL.md b/agentic/cpp/skills/benchmarks-affected/SKILL.md deleted file mode 100644 index 2072dcb..0000000 --- a/agentic/cpp/skills/benchmarks-affected/SKILL.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -name: benchmarks-affected -description: Analyze current branch versus a baseline and extract affected benchmark targets and benchmark functions using compile_commands and clang AST. ---- - -# Benchmarks Affected Skill - -Use this skill to identify exactly which benchmark binaries and benchmark functions are affected by code changes on the current branch. - -It implements a two-stage workflow: - -1. `compile_commands.json` analysis to determine affected compile targets. -2. Clang AST analysis to determine affected benchmark functions. - -## Goal - -Given `HEAD` and a baseline branch (default `main`), produce: - -- Changed files. -- Affected targets (with emphasis on benchmark targets). -- Exact benchmark functions impacted by the changes. -- A ready-to-use Google Benchmark filter regex. - -## Prerequisites - -1. Build tree with benchmarks enabled and compile database exported. Use the -repository's normal benchmark-enabling CMake options: - -```bash -BUILD_SUFFIX=local -cmake -B build/benchmarks-all_${BUILD_SUFFIX} \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -cmake --build build/benchmarks-all_${BUILD_SUFFIX} --config Release -j -``` - -2. `clang++` must be available on `PATH` (used for AST dump). - -For repository-specific invocations, check -`agentic/local/cpp/skills/benchmarks-affected/EXAMPLES.md` when present. - -## Run - -```bash -python3 agentic/cpp/skills/benchmarks-affected/analyze_benchmarks_affected.py \ - --baseline main \ - --compile-commands build/benchmarks-all_local/compile_commands.json \ - --format json -``` - -If `--compile-commands` is omitted, the script auto-selects the most recently modified `build/**/compile_commands.json`. -Working tree changes are included by default. Use `--no-include-working-tree` to restrict analysis to `...HEAD` only. - -## Output - -The analyzer reports: - -- `affected_targets`: impacted CMake targets inferred from compile dependency analysis. -- `affected_benchmark_targets`: subset of benchmark binaries impacted. -- `affected_benchmarks`: precise benchmark function names from AST-level call analysis. -- `suggested_filter_regex`: regex to pass as `--benchmark_filter`. - -## How to Use Findings - -1. Build only impacted benchmark binaries where feasible. -2. Run benchmark binaries with the suggested filter: - -```bash -FILTER='^(BM_Foo|BM_Bar)(/|$)' -BENCH_CPU=${BENCH_CPU:-0} -taskset -c "${BENCH_CPU}" build/benchmarks-all_local/benchmarks --benchmark_filter="${FILTER}" -``` - -3. If impact mapping is broad/uncertain, run full binary for selected benchmark target(s). - -## Guardrails - -1. Keep baseline comparison at merge-base style diff: `...HEAD`. -2. Use Release binaries for timing runs. -3. If AST parse fails for a TU, still trust compile target impact and mark benchmark-function scope as partial. -4. If benchmark infra (`CMakeLists.txt`, benchmark source layout) changed, fall back to conservative benchmark selection. diff --git a/agentic/cpp/skills/benchmarks-affected/analyze_benchmarks_affected.py b/agentic/cpp/skills/benchmarks-affected/analyze_benchmarks_affected.py deleted file mode 100755 index 4dcae85..0000000 --- a/agentic/cpp/skills/benchmarks-affected/analyze_benchmarks_affected.py +++ /dev/null @@ -1,1132 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import argparse -import concurrent.futures -import json -import os -import re -import shlex -import shutil -import subprocess -import sys -from collections import defaultdict -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - - -def is_project_source(source: Path, repo_root: Path) -> bool: - """Exclude third-party deps and generated build files.""" - try: - rel = source.relative_to(repo_root) - except ValueError: - return False - rel_text = rel.as_posix() - if rel_text.startswith("build/") or "_deps/" in rel_text: - return False - return True - - -KNOWN_BENCHMARK_TARGETS = {"benchmarks"} - -HEADER_EXTENSIONS = { - ".h", - ".hh", - ".hpp", - ".hxx", - ".inc", - ".ipp", - ".tcc", -} - -BUILD_INFRA_FILES = { - "CMakeLists.txt", - "CMakePresets.json", -} - -DIFF_HUNK_RE = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") - -CPP_FUNCTION_START_RE = re.compile( - r"^\s*" - r"(?:template\s*<[^>]*>\s*)?" - r"(?:(?:inline|constexpr|consteval|constinit|static|friend|virtual|explicit)\s+)*" - r"[A-Za-z_~][\w:<>,\s\*&\[\]]*\s+" - r"([~A-Za-z_][A-Za-z0-9_]*)\s*" - r"\([^;{}]*\)\s*" - r"(?:const\s*)?" - r"(?:noexcept\s*)?" - r"(?:->\s*[^\{]+)?\{" -) - - -@dataclass -class CompileCommandEntry: - directory: Path - source: Path - arguments: list[str] - output: Path | None - target: str | None - dependencies: set[Path] = field(default_factory=set) - dep_error: str | None = None - - -@dataclass -class AstImpactResult: - benchmark_names: set[str] = field(default_factory=set) - affected_names: set[str] = field(default_factory=set) - ast_error: str | None = None - - -def run_command( - args: list[str], - cwd: Path, - check: bool = True, - timeout: float | None = 60.0, -) -> subprocess.CompletedProcess[str]: - return subprocess.run( - args, - cwd=str(cwd), - text=True, - capture_output=True, - check=check, - timeout=timeout, - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=( - "Analyze benchmark impact between baseline and HEAD via " - "compile_commands dependency mapping and clang AST analysis." - ) - ) - parser.add_argument( - "--baseline", - default="main", - help="Baseline ref used as ...HEAD (default: main).", - ) - parser.add_argument( - "--head", - default="HEAD", - help="Contender ref (default: HEAD).", - ) - parser.add_argument( - "--compile-commands", - default=None, - help=( - "Path to compile_commands.json. If omitted, auto-discovers most " - "recent build/**/compile_commands.json." - ), - ) - parser.add_argument( - "--clangxx", - default=None, - help="clang++ executable for AST dump (auto-detected if omitted).", - ) - parser.add_argument( - "--format", - choices=["text", "json"], - default="text", - help="Output format (default: text).", - ) - parser.add_argument( - "--include-working-tree", - dest="include_working_tree", - action="store_true", - default=True, - help=( - "Include local unstaged/staged changes in changed-files set, " - "in addition to ... (default: enabled)." - ), - ) - parser.add_argument( - "--no-include-working-tree", - dest="include_working_tree", - action="store_false", - help="Disable working-tree inclusion and only analyze ....", - ) - return parser.parse_args() - - -def git_repo_root() -> Path: - proc = run_command(["git", "rev-parse", "--show-toplevel"], cwd=Path.cwd()) - return Path(proc.stdout.strip()).resolve() - - -def resolve_compile_commands(repo_root: Path, explicit_path: str | None) -> Path: - if explicit_path: - compile_path = Path(explicit_path) - if not compile_path.is_absolute(): - compile_path = (repo_root / compile_path).resolve() - if not compile_path.exists(): - raise FileNotFoundError(f"compile_commands.json not found: {compile_path}") - return compile_path - - candidates = sorted( - repo_root.glob("build/**/compile_commands.json"), - key=lambda path: path.stat().st_mtime, - reverse=True, - ) - if not candidates: - raise FileNotFoundError( - "No compile_commands.json found under build/**. " - "Configure with -DCMAKE_EXPORT_COMPILE_COMMANDS=ON first." - ) - return candidates[0].resolve() - - -def load_compile_commands( - compile_commands_path: Path, - repo_root: Path, -) -> list[CompileCommandEntry]: - entries: list[CompileCommandEntry] = [] - data = json.loads(compile_commands_path.read_text(encoding="utf-8")) - for raw_entry in data: - directory = Path(raw_entry["directory"]).resolve() - - raw_source = Path(raw_entry["file"]) - if raw_source.is_absolute(): - source = raw_source.resolve() - else: - source = (directory / raw_source).resolve() - - if not is_project_source(source, repo_root): - continue - - if "arguments" in raw_entry: - arguments = [str(arg) for arg in raw_entry["arguments"]] - else: - arguments = shlex.split(raw_entry["command"]) - - output = infer_output_path(arguments, directory) - target = infer_cmake_target_from_output(output) - - entries.append( - CompileCommandEntry( - directory=directory, - source=source, - arguments=arguments, - output=output, - target=target, - ) - ) - return entries - - -def infer_output_path(arguments: list[str], directory: Path) -> Path | None: - output_token: str | None = None - for idx, arg in enumerate(arguments): - if arg == "-o" and idx + 1 < len(arguments): - output_token = arguments[idx + 1] - elif arg.startswith("-o") and len(arg) > 2: - output_token = arg[2:] - elif arg.startswith("/Fo") and len(arg) > 3: - output_token = arg[3:] - - if output_token is None: - return None - - out_path = Path(output_token) - if out_path.is_absolute(): - return out_path.resolve() - return (directory / out_path).resolve() - - -def infer_cmake_target_from_output(output: Path | None) -> str | None: - if output is None: - return None - parts = output.parts - for index, part in enumerate(parts): - if part == "CMakeFiles" and index + 1 < len(parts): - target_part = parts[index + 1] - if target_part.endswith(".dir"): - return target_part[: -len(".dir")] - return target_part - return None - - -def git_changed_files(repo_root: Path, baseline: str, head: str) -> set[Path]: - diff_range = f"{baseline}...{head}" - proc = run_command(["git", "diff", "--name-only", diff_range], cwd=repo_root) - changed_files: set[Path] = set() - for line in proc.stdout.splitlines(): - line = line.strip() - if not line: - continue - changed_files.add((repo_root / line).resolve()) - return changed_files - - -def git_working_tree_changed_files(repo_root: Path) -> set[Path]: - changed_files: set[Path] = set() - commands = [ - ["git", "diff", "--name-only"], - ["git", "diff", "--name-only", "--cached"], - ] - for cmd in commands: - proc = run_command(cmd, cwd=repo_root) - for line in proc.stdout.splitlines(): - line = line.strip() - if not line: - continue - changed_files.add((repo_root / line).resolve()) - return changed_files - - -def parse_changed_lines_from_diff_text( - diff_text: str, - repo_root: Path, -) -> dict[Path, set[int]]: - changed_lines: dict[Path, set[int]] = defaultdict(set) - - current_file: Path | None = None - in_hunk = False - new_line = 0 - - for raw_line in diff_text.splitlines(): - if raw_line.startswith("+++ "): - file_token = raw_line[4:].strip() - if file_token == "/dev/null": - current_file = None - in_hunk = False - continue - if file_token.startswith("b/"): - file_token = file_token[2:] - current_file = (repo_root / file_token).resolve() - in_hunk = False - continue - - hunk_match = DIFF_HUNK_RE.match(raw_line) - if hunk_match: - in_hunk = current_file is not None - new_line = int(hunk_match.group(1)) - continue - - if not in_hunk or current_file is None: - continue - - if raw_line.startswith("+") and not raw_line.startswith("+++"): - changed_lines[current_file].add(new_line) - new_line += 1 - continue - - if raw_line.startswith("-") and not raw_line.startswith("---"): - continue - - if raw_line.startswith(" "): - new_line += 1 - continue - - return changed_lines - - -def git_changed_line_map( - repo_root: Path, - baseline: str, - head: str, - include_working_tree: bool, -) -> dict[Path, set[int]]: - changed_lines: dict[Path, set[int]] = defaultdict(set) - - proc = run_command( - ["git", "diff", "--unified=0", f"{baseline}...{head}"], - cwd=repo_root, - ) - baseline_map = parse_changed_lines_from_diff_text(proc.stdout, repo_root) - for path, lines in baseline_map.items(): - changed_lines[path].update(lines) - - if include_working_tree: - for cmd in ( - ["git", "diff", "--unified=0"], - ["git", "diff", "--cached", "--unified=0"], - ): - wt_proc = run_command(cmd, cwd=repo_root) - wt_map = parse_changed_lines_from_diff_text(wt_proc.stdout, repo_root) - for path, lines in wt_map.items(): - changed_lines[path].update(lines) - - return changed_lines - - -def extract_changed_symbol_names_from_file( - file_path: Path, - changed_lines: set[int], -) -> set[str]: - if not changed_lines or not file_path.exists(): - return set() - - lines = file_path.read_text(encoding="utf-8", errors="replace").splitlines() - symbols: set[str] = set() - - line_index = 1 - max_line = len(lines) - while line_index <= max_line: - line = lines[line_index - 1] - match = CPP_FUNCTION_START_RE.match(line) - if not match: - line_index += 1 - continue - - symbol_name = match.group(1) - start_line = line_index - brace_depth = line.count("{") - line.count("}") - end_line = start_line - - while brace_depth > 0 and end_line < max_line: - end_line += 1 - body_line = lines[end_line - 1] - brace_depth += body_line.count("{") - body_line.count("}") - - if any(start_line <= line_no <= end_line for line_no in changed_lines): - symbols.add(symbol_name) - - line_index = end_line + 1 - - return symbols - - -def collect_changed_symbol_names( - changed_line_map: dict[Path, set[int]], -) -> set[str]: - symbol_names: set[str] = set() - for file_path, changed_lines in changed_line_map.items(): - symbol_names.update( - extract_changed_symbol_names_from_file(file_path, changed_lines) - ) - return symbol_names - - -def clean_command_for_dependency_scan(arguments: list[str]) -> list[str]: - cleaned: list[str] = [] - skip_next = False - flags_with_value = { - "-o", - "-MF", - "-MT", - "-MQ", - "-MJ", - "-Xclang", - } - standalone_drop = { - "-c", - "-MD", - "-MMD", - "-MP", - "-MM", - "-M", - "-E", - "-S", - } - - index = 0 - while index < len(arguments): - arg = arguments[index] - if skip_next: - skip_next = False - index += 1 - continue - - if arg in flags_with_value: - skip_next = True - index += 1 - continue - if arg in standalone_drop: - index += 1 - continue - if arg.startswith("-o") and len(arg) > 2: - index += 1 - continue - if arg.startswith("-MF") and len(arg) > 3: - index += 1 - continue - if arg.startswith("-MT") and len(arg) > 3: - index += 1 - continue - if arg.startswith("-MQ") and len(arg) > 3: - index += 1 - continue - if arg.startswith("-MJ") and len(arg) > 3: - index += 1 - continue - - cleaned.append(arg) - index += 1 - - return cleaned - - -def parse_makefile_dependencies(stdout_text: str) -> list[str]: - flattened = stdout_text.replace("\\\n", " ").replace("\n", " ") - if ":" not in flattened: - return [] - dep_payload = flattened.split(":", 1)[1].strip() - if not dep_payload: - return [] - return shlex.split(dep_payload) - - -def compute_tu_dependencies(entry: CompileCommandEntry) -> None: - dep_cmd = clean_command_for_dependency_scan(entry.arguments) - if not dep_cmd: - entry.dep_error = "Empty compile command after sanitization" - entry.dependencies = {entry.source} - return - - dep_cmd.extend(["-MM", "-MF", "-", "-MT", "__benchmark_affected_tu__"]) - source_arg = str(entry.source) - if source_arg not in dep_cmd: - dep_cmd.append(source_arg) - - try: - proc = run_command(dep_cmd, cwd=entry.directory, check=False) - except FileNotFoundError as exc: - entry.dep_error = str(exc) - entry.dependencies = {entry.source} - return - - dependencies: set[Path] = {entry.source} - if proc.returncode != 0: - stderr = proc.stderr.strip() - entry.dep_error = ( - stderr if stderr else f"Dependency scan failed ({proc.returncode})" - ) - entry.dependencies = dependencies - return - - for dep in parse_makefile_dependencies(proc.stdout): - dep_path = Path(dep) - resolved = ( - dep_path.resolve() - if dep_path.is_absolute() - else (entry.directory / dep_path).resolve() - ) - dependencies.add(resolved) - - entry.dependencies = dependencies - - -def is_build_infra_change(repo_root: Path, changed: set[Path]) -> bool: - for path in changed: - if path.name in BUILD_INFRA_FILES: - return True - try: - rel = path.relative_to(repo_root) - except ValueError: - continue - rel_text = rel.as_posix() - if rel_text.startswith("cmake/"): - return True - return False - - -def identify_benchmark_targets( - entries: list[CompileCommandEntry], repo_root: Path -) -> set[str]: - benchmark_targets: set[str] = set() - targets_present = {entry.target for entry in entries if entry.target} - for entry in entries: - if entry.target is None: - continue - try: - rel = entry.source.relative_to(repo_root) - rel_text = rel.as_posix() - except ValueError: - rel_text = entry.source.as_posix() - - if rel_text.startswith("src/benchmarks/"): - benchmark_targets.add(entry.target) - - benchmark_targets.update(targets_present.intersection(KNOWN_BENCHMARK_TARGETS)) - return benchmark_targets - - -def is_benchmark_source(source: Path, repo_root: Path) -> bool: - try: - rel_text = source.relative_to(repo_root).as_posix() - except ValueError: - return False - return rel_text.startswith("src/benchmarks/") - - -def dedupe_entries_by_target_source( - entries: list[CompileCommandEntry], -) -> list[CompileCommandEntry]: - deduped: list[CompileCommandEntry] = [] - seen: set[tuple[str | None, Path]] = set() - for entry in entries: - key = (entry.target, entry.source) - if key in seen: - continue - seen.add(key) - deduped.append(entry) - return deduped - - -def discover_clangxx(explicit: str | None) -> str: - if explicit: - return explicit - - candidates = [ - "clang++", - "clang++-19", - "clang++-18", - "clang++-17", - "clang++-16", - ] - for candidate in candidates: - resolved = shutil.which(candidate) - if resolved: - return resolved - raise FileNotFoundError( - "clang++ was not found on PATH. Provide --clangxx to select a clang compiler." - ) - - -def clean_command_for_ast(arguments: list[str], clangxx: str) -> list[str]: - cleaned = clean_command_for_dependency_scan(arguments) - if not cleaned: - return [] - cleaned[0] = clangxx - cleaned.extend(["-Xclang", "-ast-dump=json", "-fsyntax-only"]) - return cleaned - - -def normalize_path_candidate(path_text: str | None, working_dir: Path) -> Path | None: - if not path_text: - return None - path = Path(path_text) - if path.is_absolute(): - return path.resolve() - return (working_dir / path).resolve() - - -def file_from_loc(loc: dict[str, Any] | None, working_dir: Path) -> Path | None: - if not isinstance(loc, dict): - return None - if "file" in loc: - return normalize_path_candidate(str(loc["file"]), working_dir) - for nested_key in ("spellingLoc", "expansionLoc", "includedFrom"): - nested_loc = loc.get(nested_key) - if isinstance(nested_loc, dict): - resolved = file_from_loc(nested_loc, working_dir) - if resolved is not None: - return resolved - return None - - -def iter_ast_nodes(node: Any): - if isinstance(node, dict): - yield node - inner = node.get("inner", []) - if isinstance(inner, list): - for child in inner: - yield from iter_ast_nodes(child) - elif isinstance(node, list): - for item in node: - yield from iter_ast_nodes(item) - - -def referenced_decl_file(node: dict[str, Any], working_dir: Path) -> Path | None: - referenced = node.get("referencedDecl") - if not isinstance(referenced, dict): - return None - return file_from_loc(referenced.get("loc"), working_dir) - - -def node_references_changed_symbol( - node: dict[str, Any], - changed_symbol_names: set[str], -) -> bool: - if not changed_symbol_names: - return False - - for subnode in iter_ast_nodes(node): - if not isinstance(subnode, dict): - continue - - kind = subnode.get("kind") - if kind == "MemberExpr": - member_name = subnode.get("name") - if isinstance(member_name, str) and member_name in changed_symbol_names: - return True - - if kind == "DeclRefExpr": - ref_decl = subnode.get("referencedDecl") - if not isinstance(ref_decl, dict): - continue - ref_name = ref_decl.get("name") - if isinstance(ref_name, str) and ref_name in changed_symbol_names: - return True - - return False - - -def call_expr_callee_name(call_expr: dict[str, Any]) -> str | None: - for node in iter_ast_nodes(call_expr): - if not isinstance(node, dict): - continue - if node.get("kind") != "DeclRefExpr": - continue - referenced = node.get("referencedDecl") - if isinstance(referenced, dict) and isinstance(referenced.get("name"), str): - return referenced["name"] - return None - - -def string_literals_in_node(node: dict[str, Any]) -> list[str]: - values: list[str] = [] - for cur in iter_ast_nodes(node): - if not isinstance(cur, dict): - continue - if cur.get("kind") != "StringLiteral": - continue - value = cur.get("value") - if isinstance(value, str): - if len(value) >= 2 and value[0] == '"' and value[-1] == '"': - value = value[1:-1] - values.append(value) - return values - - -def benchmark_names_from_source(source: Path) -> set[str]: - names: set[str] = set() - if not source.exists(): - return names - text = source.read_text(encoding="utf-8", errors="replace") - for match in re.finditer(r"BENCHMARK\(\s*([A-Za-z_][A-Za-z0-9_]*)\s*\)", text): - names.add(match.group(1)) - for match in re.finditer(r"register_op\(\s*\"([^\"]+)\"", text): - names.add(match.group(1)) - return names - - -def ast_analyze_entry( - entry: CompileCommandEntry, - changed_files: set[Path], - changed_symbol_names: set[str], - clangxx: str, -) -> AstImpactResult: - result = AstImpactResult() - - ast_cmd = clean_command_for_ast(entry.arguments, clangxx) - if not ast_cmd: - result.ast_error = "Failed to build AST command" - return result - - try: - proc = run_command(ast_cmd, cwd=entry.directory, check=False) - except FileNotFoundError as exc: - result.ast_error = str(exc) - return result - - if proc.returncode != 0: - stderr = proc.stderr.strip() - result.ast_error = ( - stderr if stderr else f"AST command failed ({proc.returncode})" - ) - return result - - try: - ast_root = json.loads(proc.stdout) - except json.JSONDecodeError as exc: - result.ast_error = f"Invalid AST JSON: {exc}" - return result - - function_callees: dict[str, set[str]] = defaultdict(set) - direct_impacted_functions: set[str] = set() - dynamic_benchmarks_by_function: dict[str, set[str]] = defaultdict(set) - - for node in iter_ast_nodes(ast_root): - if not isinstance(node, dict): - continue - - if node.get("kind") not in {"FunctionDecl", "CXXMethodDecl"}: - continue - - function_name = node.get("name") - if not isinstance(function_name, str) or not function_name: - continue - - if function_name.startswith("BM_"): - result.benchmark_names.add(function_name) - - function_callees.setdefault(function_name, set()) - - function_loc = file_from_loc(node.get("loc"), entry.directory) - is_directly_impacted = function_loc in changed_files - if not is_directly_impacted: - is_directly_impacted = node_references_changed_symbol( - node, changed_symbol_names - ) - - for subnode in iter_ast_nodes(node): - if not isinstance(subnode, dict): - continue - - sub_kind = subnode.get("kind") - if sub_kind in {"CallExpr", "CXXMemberCallExpr", "CXXOperatorCallExpr"}: - callee = call_expr_callee_name(subnode) - if callee: - function_callees[function_name].add(callee) - - if callee == "register_op": - literal_values = string_literals_in_node(subnode) - if literal_values: - dynamic_benchmarks_by_function[function_name].add( - literal_values[0] - ) - - if not is_directly_impacted: - ref_file = referenced_decl_file(subnode, entry.directory) - if ref_file in changed_files: - is_directly_impacted = True - - if is_directly_impacted: - direct_impacted_functions.add(function_name) - - # Reverse call-graph propagation: if a function is directly impacted, - # every caller in this TU is impacted as well (fixed-point DFS/BFS). - callers_of: dict[str, set[str]] = defaultdict(set) - for caller, callees in function_callees.items(): - for callee in callees: - callers_of[callee].add(caller) - - impacted_functions = set(direct_impacted_functions) - stack = list(direct_impacted_functions) - while stack: - callee_name = stack.pop() - for caller_name in callers_of.get(callee_name, set()): - if caller_name in impacted_functions: - continue - impacted_functions.add(caller_name) - stack.append(caller_name) - - for function_name in impacted_functions: - if function_name.startswith("BM_"): - result.affected_names.add(function_name) - - for function_name, names in dynamic_benchmarks_by_function.items(): - result.benchmark_names.update(names) - if function_name in impacted_functions: - result.affected_names.update(names) - - return result - - -def regex_for_benchmarks(names: set[str]) -> str | None: - if not names: - return None - ordered = sorted(names) - body = "|".join(re.escape(name) for name in ordered) - return rf"^({body})(/|$)" - - -def relpath_or_abs(path: Path, root: Path) -> str: - try: - return path.relative_to(root).as_posix() - except ValueError: - return path.as_posix() - - -def main() -> int: - cli = parse_args() - - try: - repo_root = git_repo_root() - changed_files = git_changed_files(repo_root, cli.baseline, cli.head) - if cli.include_working_tree: - changed_files.update(git_working_tree_changed_files(repo_root)) - changed_line_map = git_changed_line_map( - repo_root, - cli.baseline, - cli.head, - cli.include_working_tree, - ) - changed_symbol_names = collect_changed_symbol_names(changed_line_map) - compile_commands_path = resolve_compile_commands( - repo_root, cli.compile_commands - ) - entries = load_compile_commands(compile_commands_path, repo_root) - except FileNotFoundError as exc: - print(f"error: {exc}", file=sys.stderr) - return 2 - except subprocess.CalledProcessError as exc: - stderr = (exc.stderr or "").strip() - if stderr: - print(f"error: {stderr}", file=sys.stderr) - else: - print(f"error: command failed: {' '.join(exc.cmd)}", file=sys.stderr) - return 2 - - target_to_entries: dict[str, list[CompileCommandEntry]] = defaultdict(list) - source_to_entries: dict[Path, list[CompileCommandEntry]] = defaultdict(list) - for entry in entries: - source_to_entries[entry.source].append(entry) - if entry.target: - target_to_entries[entry.target].append(entry) - - benchmark_targets = identify_benchmark_targets(entries, repo_root) - all_targets = {entry.target for entry in entries if entry.target} - benchmark_entries = dedupe_entries_by_target_source( - [entry for entry in entries if entry.target in benchmark_targets] - ) - - infra_change = is_build_infra_change(repo_root, changed_files) - relevant_changed_files = { - path - for path in changed_files - if is_project_source(path, repo_root) - or path.name in BUILD_INFRA_FILES - or relpath_or_abs(path, repo_root).startswith("cmake/") - } - has_header_changes = any( - path.suffix.lower() in HEADER_EXTENSIONS for path in relevant_changed_files - ) - benchmark_source_extensions = {".c", ".cc", ".cpp", ".cxx"} - only_benchmark_source_changes = bool(relevant_changed_files) and all( - is_benchmark_source(path, repo_root) - and path.suffix.lower() in benchmark_source_extensions - for path in relevant_changed_files - ) - - directly_affected_targets: set[str] = set() - for changed_path in changed_files: - for entry in source_to_entries.get(changed_path, []): - if entry.target: - directly_affected_targets.add(entry.target) - - dependency_scan_entries: list[CompileCommandEntry] = [] - if not infra_change and not only_benchmark_source_changes: - if has_header_changes: - dependency_scan_entries = dedupe_entries_by_target_source(entries) - else: - dependency_scan_entries = benchmark_entries - - if dependency_scan_entries: - with concurrent.futures.ThreadPoolExecutor( - max_workers=min(8, (os.cpu_count() or 4)) - ) as pool: - list(pool.map(compute_tu_dependencies, dependency_scan_entries)) - - affected_targets: set[str] = set(directly_affected_targets) - for entry in dependency_scan_entries: - has_changed_dependency = any(dep in changed_files for dep in entry.dependencies) - if has_changed_dependency and entry.target: - affected_targets.add(entry.target) - - if infra_change: - affected_targets.update(all_targets) - - dependency_impacted_benchmark_targets = affected_targets.intersection( - benchmark_targets - ) - impacted_benchmark_entries = [ - entry - for entry in benchmark_entries - if entry.target in dependency_impacted_benchmark_targets - ] - - ast_errors: dict[str, str] = {} - benchmark_target_to_names: dict[str, set[str]] = defaultdict(set) - benchmark_target_to_affected: dict[str, set[str]] = defaultdict(set) - warnings: list[str] = [] - ast_fallback_used = False - ast_entries_scanned = 0 - - if impacted_benchmark_entries: - try: - clangxx = discover_clangxx(cli.clangxx) - except FileNotFoundError as exc: - clangxx = "" - warnings.append(str(exc)) - - if not clangxx: - ast_fallback_used = True - for entry in impacted_benchmark_entries: - target_name = entry.target or "" - fallback_names = benchmark_names_from_source(entry.source) - benchmark_target_to_names[target_name].update(fallback_names) - benchmark_target_to_affected[target_name].update(fallback_names) - else: - max_ast_workers = min(2, (os.cpu_count() or 2)) - with concurrent.futures.ThreadPoolExecutor( - max_workers=max_ast_workers - ) as pool: - futures = { - pool.submit( - ast_analyze_entry, - entry, - changed_files, - changed_symbol_names, - clangxx, - ): entry - for entry in impacted_benchmark_entries - } - ast_entries_scanned = len(futures) - for future in concurrent.futures.as_completed(futures): - entry = futures[future] - target_name = entry.target or "" - source_path = entry.source - source_is_changed = source_path in changed_files - - try: - ast_result = future.result(timeout=120) - except Exception as exc: - ast_result = AstImpactResult( - ast_error=f"AST worker failed: {exc}" - ) - - if ast_result.ast_error: - ast_errors[relpath_or_abs(source_path, repo_root)] = ( - ast_result.ast_error - ) - - benchmark_names = ast_result.benchmark_names - if not benchmark_names: - benchmark_names = benchmark_names_from_source(source_path) - benchmark_target_to_names[target_name].update(benchmark_names) - - if ast_result.affected_names: - benchmark_target_to_affected[target_name].update( - ast_result.affected_names - ) - elif source_is_changed or ast_result.ast_error: - benchmark_target_to_affected[target_name].update( - benchmark_names - ) - if benchmark_names: - ast_fallback_used = True - - if infra_change and benchmark_targets: - for target_name in sorted(benchmark_targets): - for entry in target_to_entries.get(target_name, []): - names = benchmark_names_from_source(entry.source) - benchmark_target_to_names[target_name].update(names) - benchmark_target_to_affected[target_name].update(names) - - if infra_change: - affected_benchmark_targets = sorted(benchmark_targets) - else: - affected_benchmark_targets = sorted( - target for target, names in benchmark_target_to_affected.items() if names - ) - - all_affected_benchmarks: set[str] = set() - for names in benchmark_target_to_affected.values(): - all_affected_benchmarks.update(names) - - dep_scan_failures = { - relpath_or_abs(entry.source, repo_root): entry.dep_error - for entry in dependency_scan_entries - if entry.dep_error - } - - scope_mode = "normal" - if infra_change: - scope_mode = "infra_fallback" - elif ast_fallback_used: - scope_mode = "ast_fallback" - - report: dict[str, Any] = { - "baseline": cli.baseline, - "head": cli.head, - "include_working_tree": cli.include_working_tree, - "changed_symbols": sorted(changed_symbol_names), - "compile_commands": relpath_or_abs(compile_commands_path, repo_root), - "changed_files": sorted( - relpath_or_abs(path, repo_root) for path in changed_files - ), - "affected_targets": sorted(affected_targets), - "affected_benchmark_targets": affected_benchmark_targets, - "affected_benchmarks": { - target: sorted(names) - for target, names in sorted(benchmark_target_to_affected.items()) - if names - }, - "suggested_filter_regex": regex_for_benchmarks(all_affected_benchmarks), - "dependency_entries_scanned": len(dependency_scan_entries), - "benchmark_entries_scanned": len(benchmark_entries), - "ast_entries_scanned": ast_entries_scanned, - "scope_mode": scope_mode, - "dependency_scan_failures": dep_scan_failures, - "ast_failures": ast_errors, - "warnings": warnings, - } - - if cli.format == "json": - json.dump(report, sys.stdout, indent=2) - sys.stdout.write("\n") - return 0 - - print(f"Baseline: {cli.baseline}") - print(f"Head: {cli.head}") - print(f"Compile commands: {report['compile_commands']}") - print(f"Scope mode: {report['scope_mode']}") - print( - "Scan counts: " - f"dependency={report['dependency_entries_scanned']}, " - f"benchmark={report['benchmark_entries_scanned']}, " - f"ast={report['ast_entries_scanned']}" - ) - print("") - - print(f"Changed files ({len(report['changed_files'])}):") - for item in report["changed_files"]: - print(f"- {item}") - if not report["changed_files"]: - print("- none") - - print("") - print(f"Affected targets ({len(report['affected_targets'])}):") - for item in report["affected_targets"]: - print(f"- {item}") - if not report["affected_targets"]: - print("- none") - - print("") - print(f"Affected benchmark targets ({len(report['affected_benchmark_targets'])}):") - for item in report["affected_benchmark_targets"]: - print(f"- {item}") - if not report["affected_benchmark_targets"]: - print("- none") - - print("") - print("Affected benchmark functions:") - if report["affected_benchmarks"]: - for target, names in report["affected_benchmarks"].items(): - print(f"- {target}:") - for name in names: - print(f" - {name}") - else: - print("- none") - - print("") - print("Suggested --benchmark_filter regex:") - print(report["suggested_filter_regex"] or "none") - - if dep_scan_failures: - print("") - print("Dependency scan failures:") - for source, error in dep_scan_failures.items(): - print(f"- {source}: {error}") - - if ast_errors: - print("") - print("AST failures:") - for source, error in ast_errors.items(): - print(f"- {source}: {error}") - - if warnings: - print("") - print("Warnings:") - for warning in warnings: - print(f"- {warning}") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/agentic/cpp/skills/benchmarks-compare-revisions/SKILL.md b/agentic/cpp/skills/benchmarks-compare-revisions/SKILL.md deleted file mode 100644 index caa9542..0000000 --- a/agentic/cpp/skills/benchmarks-compare-revisions/SKILL.md +++ /dev/null @@ -1,225 +0,0 @@ ---- -name: benchmarks-compare-revisions -description: Compare benchmark performance between two git revisions via Google Benchmark compare.py, with optional hardware-counter comparison from diagnostic libpfm builds. ---- - -# Benchmarks Compare Revisions Skill - -Use this skill to compare performance between two git revisions. - -This workflow now depends on: - -1. `../benchmarks-affected/SKILL.md` to determine affected benchmark targets/functions and produce a benchmark filter. -2. `../benchmarks/SKILL.md` for build/run operational details. - -## Goal - -Build two separate benchmark binaries using short commit hashes as build suffixes, compare timing results with Google Benchmark compare.py, and optionally compare hardware counters across the same revisions. - -## Step 0 — Choose revisions, hashes, and options - -Pick a baseline and a contender revision. Use short commit hashes to suffix build directories so builds do not collide. - -Optional behavior flags: - -- `COLLECT_COUNTERS=1` to enable hardware-counter collection and analysis in addition to timing comparison. -- `COLLECT_COUNTERS=0` to run timing-only comparison. - -Counter collection is Linux-only and requires: - -- diagnostic builds with `BENCHMARK_ENABLE_LIBPFM=ON` -- perf permissions on the host (for access to performance counters) - -Example: -```bash -BASELINE=abc1234 -CONTENDER=def5678 -``` - -## Step 1 — Compute affected benchmark scope first - -Run `benchmarks-affected` from the contender checkout to derive the compare scope. - -Do not duplicate `benchmarks-affected` internals here (compile database selection, AST analysis, or fallback heuristics). Follow that skill directly and consume only its outputs. - -Inputs to pass through: - -- `--baseline ${BASELINE}` -- optional compile-commands path if auto-detection is not desired -- optional output format (`json` recommended for parsing) - -Consume these outputs from `benchmarks-affected`: - -- `suggested_filter_regex` -> set `FILTER` -- `affected_benchmark_targets` -> optionally constrain which benchmark binary/binaries to run -- `affected_benchmarks` -> function-level scope for validation/reporting - -If `FILTER` is empty, fall back to full benchmark binary compare (conservative mode). - -## Step 2 — Build both revisions - -Use the existing benchmarks skill build steps, but set the build suffix to include the short hash for each revision. - -Always build Release timing binaries. - -If `COLLECT_COUNTERS=1`, also build diagnostic binaries (RelWithDebInfo + libpfm) for both revisions. - -```bash -# Baseline -BUILD_SUFFIX=bench_${BASELINE} -git checkout ${BASELINE} -# Follow ../benchmarks/SKILL.md timing build instructions with this suffix -# If COLLECT_COUNTERS=1, also follow the diagnostic build instructions with this suffix - -# Contender -BUILD_SUFFIX=bench_${CONTENDER} -git checkout ${CONTENDER} -# Follow ../benchmarks/SKILL.md timing build instructions with this suffix -# If COLLECT_COUNTERS=1, also follow the diagnostic build instructions with this suffix -``` - -Expected build trees: - -- Timing: `build/benchmarks-all_bench_` -- Counters (optional): `build/benchmarks-diagnostic_bench_` - -## Step 3 — Compare using compare.py - -Use Google Benchmark compare tooling with a JSON-first flow to avoid long-running binary-vs-binary retries. - -Locate compare.py from the Google Benchmark dependency (installed under the build tree): -```bash -COMPARE_PY=build/benchmarks-all_bench_${BASELINE}/_deps/googlebenchmark-src/tools/compare.py -``` - -Verify Python deps once (compare.py imports numpy/scipy): -```bash -python3 -c "import numpy, scipy" -``` - -Generate baseline/contender JSON sequentially with explicit file outputs: -```bash -BENCH_CPU=${BENCH_CPU:-0} -BENCH_RUN="taskset -c ${BENCH_CPU}" -BASE_JSON=/tmp/bench_${BASELINE}.json -CONT_JSON=/tmp/bench_${CONTENDER}.json - -${BENCH_RUN} build/benchmarks-all_bench_${BASELINE}/benchmarks \ - --benchmark_report_aggregates_only=true \ - --benchmark_display_aggregates_only=true \ - --benchmark_format=json \ - --benchmark_out=${BASE_JSON} > /tmp/bench_${BASELINE}.log 2>&1 - -${BENCH_RUN} build/benchmarks-all_bench_${CONTENDER}/benchmarks \ - --benchmark_report_aggregates_only=true \ - --benchmark_display_aggregates_only=true \ - --benchmark_format=json \ - --benchmark_out=${CONT_JSON} > /tmp/bench_${CONTENDER}.log 2>&1 -``` - -Validate JSON before comparing: -```bash -python3 -m json.tool ${BASE_JSON} > /dev/null -python3 -m json.tool ${CONT_JSON} > /dev/null -``` - -Run the comparison: -```bash -python3 ${COMPARE_PY} -a benchmarks ${BASE_JSON} ${CONT_JSON} -``` - -Use the affected filter from Step 1 when generating JSON files: -```bash -if [ -n "${FILTER}" ]; then - FILTER_ARG="--benchmark_filter=${FILTER}" -else - FILTER_ARG="" -fi - -${BENCH_RUN} build/benchmarks-all_bench_${BASELINE}/benchmarks ${FILTER_ARG} --benchmark_report_aggregates_only=true --benchmark_display_aggregates_only=true ... -${BENCH_RUN} build/benchmarks-all_bench_${CONTENDER}/benchmarks ${FILTER_ARG} --benchmark_report_aggregates_only=true --benchmark_display_aggregates_only=true ... -``` - -## Step 3b — Compare hardware counters (optional, Linux only) - -Run this step only when `COLLECT_COUNTERS=1`. - -1. Preflight first with one tiny counter-enabled benchmark run from a diagnostic binary. If output includes warnings such as `Failed to get a file descriptor for performance counter`, mark counters unavailable and skip counter collection. -2. Run baseline and contender diagnostic binaries sequentially with explicit JSON outputs and the same filter scope: - -```bash -BASE_COUNTERS_JSON=/tmp/bench_counters_${BASELINE}.json -CONT_COUNTERS_JSON=/tmp/bench_counters_${CONTENDER}.json - -${BENCH_RUN} build/benchmarks-diagnostic_bench_${BASELINE}/benchmarks \ - ${FILTER_ARG} \ - --benchmark_counters_tabular=true \ - --benchmark_format=json \ - --benchmark_out=${BASE_COUNTERS_JSON} > /tmp/bench_counters_${BASELINE}.log 2>&1 - -${BENCH_RUN} build/benchmarks-diagnostic_bench_${CONTENDER}/benchmarks \ - ${FILTER_ARG} \ - --benchmark_counters_tabular=true \ - --benchmark_format=json \ - --benchmark_out=${CONT_COUNTERS_JSON} > /tmp/bench_counters_${CONTENDER}.log 2>&1 -``` - -3. Validate JSON files before consuming: - -```bash -python3 -m json.tool ${BASE_COUNTERS_JSON} > /dev/null -python3 -m json.tool ${CONT_COUNTERS_JSON} > /dev/null -``` - -4. Collect and compare these counter families when present: - -- `instructions`, `cycles` -- `cache-misses`, `cache-references` -- `branch-misses`, `branches` -- `L1-dcache-load-misses` - -5. Compute derived metrics when denominators are non-zero: - -- IPC = `instructions / cycles` -- Cache miss rate = `cache-misses / cache-references` -- Branch mispredict rate = `branch-misses / branches` - -6. Pair baseline and contender rows by benchmark name, compute deltas, and flag anomalies where timing direction conflicts with key counter direction. - -7. Emit a canonical summary table for downstream consumers: - -```markdown -| Benchmark | IPC (base -> new) | Cache Miss Rate (base -> new) | Branch Mispredict (base -> new) | Anomaly? | -|---|---:|---:|---:|---| -``` - -## Retry and Timeout Policy - -1. Run benchmarks sequentially; do not background with `nohup`/`&`. -2. If a run times out, narrow filter and retry once. -3. Maximum retries per benchmark group: 1. -4. If still failing, emit blocked/partial findings instead of repeated attempts. - -Apply this policy to both timing and counter runs. - -## Step 4 — Record findings - -Capture and return: - -- compare.py output (terminal transcript or redirected file) -- effective filter used -- timing JSON artifacts for baseline and contender -- `counters_available` (`true`/`false`) -- if `counters_available=false`, a reason string (unsupported OS, missing libpfm, perf permission denied, preflight failure) -- if counters are available: counter JSON artifacts, derived metrics table, and anomaly list - -## Best Practices / Guardrails - -1. **Release only**: never compare Debug binaries. -2. **Short hash suffixes**: keep build dirs isolated per revision (example: `bench_`). -3. **Same host, same conditions**: do not compare across different machines or power profiles. -4. **Filter from analysis**: use `benchmarks-affected` output instead of hand-crafted filters whenever possible. -5. **Pin process and frequency**: use `taskset -c ${BENCH_CPU:-0}` for all benchmark executions and follow benchmark skill guidance on CPU governor. -6. **Counter collection is optional and Linux-only**: when unavailable, return timing-only outputs with `counters_available=false`. -7. **Always preflight counters**: do not run full counter collection if preflight fails. -8. **Keep build types separated**: timing uses `benchmarks-all_*` Release builds; counters use `benchmarks-diagnostic_*` RelWithDebInfo builds; never Debug. diff --git a/agentic/cpp/skills/benchmarks/SKILL.md b/agentic/cpp/skills/benchmarks/SKILL.md deleted file mode 100644 index 92af231..0000000 --- a/agentic/cpp/skills/benchmarks/SKILL.md +++ /dev/null @@ -1,209 +0,0 @@ ---- -name: benchmarks -description: Run Google Benchmark binaries, including filtering, hardware counters, and perf profiling. ---- - -# Benchmarks Skill - -You now have expertise in running and interpreting Google Benchmark suites. -Follow these workflows: - -## Build Directory Convention - -Use a short commit hash suffix for committed revisions: - -```bash -BUILD_SUFFIX=$(git rev-parse --short HEAD) -``` - -If the worktree has uncommitted changes, append a descriptive suffix so results -cannot be confused with a clean HEAD build: - -```bash -BUILD_SUFFIX=$(git rev-parse --short HEAD)-dirty -``` - -If not a git repository, use - -```bash -BUILD_SUFFIX=agent -``` - -## CRITICAL: Never Run Benchmarks from a Debug Build - -> **Always pass `--config Release` (or `--config RelWithDebInfo`) to `cmake --build`.** -> Multi-config generators (MSVC, Xcode) default to `Debug` if no `--config` is given. -> Google Benchmark will print `***WARNING*** Library was built as DEBUG` and timings will -> be 3-10x slower and meaningless. Always verify the binary path contains `Release/` or -> `RelWithDebInfo/`, never `Debug/`. - -## Step 1 — Build - -If benchmarks affected by the changes are easily tractable build only related targets. - -**Pure timing (benchmarks, Release):** -```bash -cmake -B build/benchmarks_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Release -cmake --build build/benchmarks_${BUILD_SUFFIX} --config Release -j -``` - -**Hardware counters / verbose report (benchmarks-diagnostic, RelWithDebInfo, Linux only):** -```bash -cmake -B build/benchmarks-diagnostic_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBENCHMARK_ENABLE_LIBPFM=ON -cmake --build build/benchmarks-diagnostic_${BUILD_SUFFIX} --config RelWithDebInfo -j -``` - -For repository-specific benchmark examples, check -`agentic/local/cpp/skills/benchmarks/EXAMPLES.md` when present. - -## Step 2 — Run - -Prefer running benchmarks with filtering passing the benchmarks that should be affected. - -Unless the user explicitly asks otherwise, pin benchmark execution to one CPU with -`taskset` to reduce scheduler noise. Use CPU 0 by default, or override with -`BENCH_CPU=` when a better isolated/performance core is known: - -```bash -BENCH_CPU=${BENCH_CPU:-0} -BENCH_RUN="taskset -c ${BENCH_CPU}" -``` - -If `taskset` is unavailable or fails on the host, report that benchmark results -are unpinned and more noisy. - -Execution guardrails: -- Run benchmark commands sequentially in CI. -- Avoid background jobs (`nohup`, `&`) for benchmark collection. -- Always write machine-readable results with `--benchmark_out` when data is later parsed. - -### Available benchmark binaries - -Discover benchmark binary names from the repository's build system. Common -locations include `build/**/` for single-config generators and -`build/**/Release/` for multi-config generators. Repository-specific -binary lists belong in the repo-local benchmark examples overlay. - -Binary paths vary by generator type: - -| Generator | Path pattern | -|-----------|-------------| -| MSVC / Xcode (multi-config) | `build/_/Release/` | -| Ninja / Make (single-config) | `build/_/` | - -### Run all benchmarks in a binary - -```bash -# Multi-config (MSVC/Xcode) -${BENCH_RUN} build/benchmarks_${BUILD_SUFFIX}/Release/benchmarks - -# Single-config (Ninja/Make) -${BENCH_RUN} build/benchmarks_${BUILD_SUFFIX}/benchmarks -``` - -### Filter benchmarks with a regex (FILTER parameter) - -```bash -FILTER="BM_Foo" # change to match benchmark names in the target binary - -# Multi-config -${BENCH_RUN} build/benchmarks_${BUILD_SUFFIX}/Release/benchmarks --benchmark_filter="${FILTER}" - -# Single-config -${BENCH_RUN} build/benchmarks_${BUILD_SUFFIX}/benchmarks --benchmark_filter="${FILTER}" -``` - -Examples: -```bash -# Only one benchmark family -... --benchmark_filter="BM_Foo" - -# Only one layout/parameter family -... --benchmark_filter="BM_Foo.*Variant" - -# List all available benchmark names without running -... --benchmark_list_tests=true -``` - -### Run with hardware counters (benchmarks-diagnostic build, Linux only) - -The `--benchmark_perf_counters` flag requests hardware counter collection via libpfm. Counter names are platform-specific but common ones include `CYCLES`, `INSTRUCTIONS`, `CACHE-MISSES`, `CACHE-REFERENCES`, `BRANCH-MISSES`, `BRANCH-INSTRUCTIONS`. - -```bash -${BENCH_RUN} build/benchmarks-diagnostic_${BUILD_SUFFIX}/RelWithDebInfo/benchmarks \ - --benchmark_filter="${FILTER}" \ - --benchmark_perf_counters=CYCLES,INSTRUCTIONS,CACHE-MISSES \ - --benchmark_counters_tabular=true -``` - -### Save results to file - -```bash -${BENCH_RUN} build/benchmarks_${BUILD_SUFFIX}/Release/benchmarks \ - --benchmark_filter="${FILTER}" \ - --benchmark_report_aggregates_only=true \ - --benchmark_display_aggregates_only=true \ - --benchmark_format=json \ - --benchmark_out=results.json -``` - -Validate output before consuming: -```bash -python3 -m json.tool results.json > /dev/null -``` - -## Step 3 — Profile with perf (Linux only) - -Use when hardware counters alone are not enough and you need a full call-graph profile for post-processing. - -**Record:** -```bash -perf record -g -F 999 \ - -- ${BENCH_RUN} build/benchmarks-diagnostic_${BUILD_SUFFIX}/benchmarks \ - --benchmark_filter="${FILTER}" \ - --benchmark_min_time=5s -``` - -**Quick report (terminal):** -```bash -perf report --stdio -``` - -**Flame graph (requires FlameGraph scripts):** -```bash -perf script | stackcollapse-perf.pl | flamegraph.pl > flamegraph.html -``` - -**Export for external tools (Hotspot, Firefox Profiler):** -```bash -perf script -F +pid > perf.data.txt -# or open with `hotspot perf.data` -``` - -## Useful Benchmark Flags - -| Flag | Purpose | -|------|---------| -| `--benchmark_filter=` | Run only matching benchmarks | -| `--benchmark_list_tests=true` | List names without running | -| `--benchmark_repetitions=` | Repeat each benchmark n times | -| `--benchmark_min_time=` | Minimum run time per benchmark | -| `--benchmark_format=json` | Machine-readable output | -| `--benchmark_out=` | Save output to file | -| `--benchmark_perf_counters=CYCLES,INSTRUCTIONS,...` | Collect hardware perf counters (requires libpfm build) | -| `--benchmark_counters_tabular=true` | Align user/perf counter columns into a table | -| `--benchmark_time_unit=ms` | Change display unit (ns/us/ms/s) | - -## Best Practices - -1. **Never run from a Debug binary**: always use `--config Release` at build time; check path contains `Release/` -2. **Use benchmarks for clean timing**: Release optimizations, no debug info, no libpfm overhead -3. **Use benchmarks-diagnostic for hardware counters**: RelWithDebInfo + libpfm; Linux only -4. **Use perf for deep profiling**: when counters point to a hotspot but don't explain it -5. **Pin benchmark process** with `taskset -c ${BENCH_CPU:-0}` unless unavailable -6. **Pin CPU frequency** before timing runs: `sudo cpupower frequency-set -g performance` -7. **Filter to reduce noise**: narrow the filter regex to the benchmark under investigation -8. **Save JSON output** when comparing before/after changes: use `--benchmark_out` and diff the files -9. **Fail fast on environment issues**: precheck Python deps used by compare tooling (`numpy`, `scipy`) -10. **Use explicit retry limits**: on timeout, narrow scope and retry once; avoid repeated full-suite attempts -11. **Preflight perf counters**: run a tiny counter-enabled benchmark first; if counters unavailable, skip counter workflow diff --git a/agentic/cpp/skills/cmake/SKILL.md b/agentic/cpp/skills/cmake/SKILL.md deleted file mode 100644 index a659e1a..0000000 --- a/agentic/cpp/skills/cmake/SKILL.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -name: cmake -description: Compile and build CMake projects, including configuring build types, options, and running test binaries. ---- - -# CMake Build Skill - -You now have expertise in building and configuring CMake projects. Follow these workflows: - -## Build Directory Convention - -Use a short commit hash suffix for committed revisions: - -```bash -BUILD_SUFFIX=$(git rev-parse --short HEAD) -``` - -If the worktree has uncommitted changes, append a descriptive suffix so generated -artifacts cannot be confused with a clean HEAD build: - -```bash -BUILD_SUFFIX=$(git rev-parse --short HEAD)-dirty -``` - -If not a git repository, use - -```bash -BUILD_SUFFIX=agent -``` - -Build directories follow the pattern `build/_`. - -## Using Presets (Preferred When Available) - -> **Important**: `cmake --preset` sets cache variables and generator but its `binaryDir` cannot be -> overridden from the command line. To use a preset's settings with a custom build dir, pass the -> relevant `-D` flags explicitly together with `-B`. Use `--preset` only to discover what flags a -> preset applies. - -**List available presets:** -```bash -cmake --list-presets -``` - -**Replicate a preset's settings with a custom suffix build dir:** - -Release: -```bash -cmake -B build/release_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Release -cmake --build build/release_${BUILD_SUFFIX} -j -``` - -Debug: -```bash -cmake -B build/debug_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Debug -cmake --build build/debug_${BUILD_SUFFIX} -j -``` - -AddressSanitizer: -```bash -cmake -B build/asan_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Debug -DENABLE_ADDRESS_SANITIZER=ON -cmake --build build/asan_${BUILD_SUFFIX} -j -``` - -Coverage: -```bash -cmake -B build/coverage_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Debug -DENABLE_COVERAGE=ON -cmake --build build/coverage_${BUILD_SUFFIX} -j -``` - -Benchmarks: -```bash -cmake -B build/benchmarks_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Release -DBUILD_BENCHMARKS=ON -cmake --build build/benchmarks_${BUILD_SUFFIX} -j -``` - -## Additional Feature Options - -Feature flags are project-specific. Inspect `CMakeLists.txt`, -`CMakePresets.json`, or `cmake -LAH ` before toggling options. For -repository-specific examples, check -`agentic/local/cpp/skills/cmake/EXAMPLES.md` when present. - -**Example feature toggle:** -```bash -cmake -B build/release_${BUILD_SUFFIX} -DCMAKE_BUILD_TYPE=Release -DENABLE_FEATURE=ON -cmake --build build/release_${BUILD_SUFFIX} -j -``` - -## Best Practices - -1. **Use out-of-source builds**: Keep build artifacts in `build/_` directories -2. **Presets fix binaryDir**: `--preset` cannot be combined with `-B` to change the build dir; replicate `-D` flags manually with `-B` instead -3. **Reconfigure when options change**: Rerun the `cmake -B ...` step when toggling options -4. **Clean build directory when needed**: Delete the entire build folder for a fresh configuration -5. **Match build type to task**: Release for performance work, Debug/ASan for correctness diff --git a/agentic/cpp/skills/paper-search/SKILL.md b/agentic/cpp/skills/paper-search/SKILL.md deleted file mode 100644 index 27f108c..0000000 --- a/agentic/cpp/skills/paper-search/SKILL.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -name: paper-search -description: "Search for academic papers across Semantic Scholar, arXiv, and CrossRef APIs. Returns unified results with title, authors, year, abstract, DOI, venue, and citation counts. Integrates with Zotero MCP tools for adding found papers to a Zotero library and generating BibTeX entries. Use when the user asks to find papers, search for related work, look up a DOI, or discover references on a topic." ---- - -# Paper Search - -Search external academic APIs for papers. Provides a unified interface across Semantic Scholar, arXiv, and CrossRef with optional Zotero integration. - -## Workflow - -### 1. Search for Papers - -Run the search script from the skill's `scripts/` directory: - -```bash -python3 scripts/search_papers.py --query "topic" --source semantic_scholar --limit 10 --format compact -``` - -Available sources: -- `semantic_scholar` — Default. Best for comprehensive search with citation counts. -- `arxiv` — Best for preprints and recent unpublished work. -- `crossref` — Best for published works and DOI-based metadata. -- `all` — Query all three sources (slower, results combined). - -Output formats: -- `json` — Full JSON output (default). Good for programmatic use. -- `compact` — Human-readable summary with title, authors, year, venue, citations, and truncated abstract. - -### 2. DOI Lookup - -Look up a specific paper by DOI: - -```bash -python3 scripts/search_papers.py --doi "10.1145/1234567.1234568" --format compact -``` - -### 3. Download PDFs - -Download open-access PDFs directly from search results: - -```bash -python3 scripts/search_papers.py --query "wavelet tree" --source arxiv --limit 3 --download ~/papers -``` - -- arXiv papers always have PDFs available. -- Semantic Scholar provides `openAccessPdf` URLs when available. -- CrossRef may provide PDF links via publisher APIs. - -The `--download` flag adds a `downloaded_path` field to each result in JSON output. - -### 4. Add to Zotero - -**Option A: Via DOI/URL (metadata only)** - -After finding relevant papers, add them to Zotero using the Zotero MCP tools: - -- `zotero_add_by_doi` — Preferred when DOI is available. Fetches full metadata from CrossRef. -- `zotero_add_by_url` — Use for arXiv papers or when only a URL is available. - -**Option B: Via downloaded PDF (metadata + attachment)** - -Download the PDF first, then add to Zotero with the PDF file: - -```bash -# Step 1: Download PDFs and get paths in JSON -python3 scripts/search_papers.py --doi "10.1007/978-3-540-73420-8_13" --download ~/papers --format json - -# Step 2: Use zotero_zotero_add_from_file with the downloaded_path -``` - -The agent should call `zotero_zotero_add_from_file` with the `downloaded_path` from the JSON output. This attaches the PDF to the Zotero item and attempts DOI-based metadata extraction. - -**Option C: Download + Zotero in one step** - -Use `--zotero` to download PDFs with paths formatted for easy Zotero import: - -```bash -python3 scripts/search_papers.py -q "succinct data structures" -s arxiv -n 3 --zotero --download ~/papers -``` - -After adding papers, update the semantic search database: - -``` -zotero_update_search_database -``` - -### 5. Generate BibTeX - -For papers already in Zotero, use `zotero_get_item_metadata` with `format: "bibtex"` to get BibTeX entries. Alternatively, use `zotero_fetch` for full metadata. - -For papers NOT in Zotero, BibTeX can be constructed from the search results' JSON fields (`authors`, `year`, `title`, `venue`, `doi`). - -## Guidance - -- Start with `semantic_scholar` for general queries — it has the broadest coverage and citation data. -- Use `arxiv` when looking for very recent work or preprints in CS/ML/physics. -- Use `crossref` for DOI lookups or when Semantic Scholar returns no results. -- When using `--source all`, results may contain duplicates (same paper from different sources). Deduplicate by DOI or title similarity. -- Citation counts are approximate and may differ across sources. -- arXiv results return the arXiv ID (e.g., `2301.12345`) which can be used with `zotero_add_by_url` via `https://arxiv.org/abs/2301.12345`. - -## API Quirks - -- **arXiv `atom:id` is NOT a DOI** — it contains an arXiv URL like `http://arxiv.org/abs/2301.12345`. Store the extracted ID in `arxiv_id` only; set `doi` to `None` for arXiv results. Writing the arXiv URL into `doi` produces invalid DOI metadata downstream (e.g., Zotero import). -- **CrossRef `select` must include `link`** — the `link` field is needed for `pdf_url` extraction. If omitted from `select`, the API won't return link metadata and `pdf_url` will silently be empty for all CrossRef results. diff --git a/agentic/cpp/skills/paper-search/references/api_reference.md b/agentic/cpp/skills/paper-search/references/api_reference.md deleted file mode 100644 index dcb5aa5..0000000 --- a/agentic/cpp/skills/paper-search/references/api_reference.md +++ /dev/null @@ -1,32 +0,0 @@ -# External Paper Search APIs - -## Semantic Scholar - -- **Base URL**: `https://api.semanticscholar.org/graph/v1/` -- **Rate limit**: 1 req/sec without API key, 10 req/sec with key -- **No auth required** for basic usage -- **Fields**: title, authors, year, abstract, externalIds (DOI, ArXiv), venue, citationCount -- **Best for**: Comprehensive academic search with citation counts - -## arXiv - -- **Base URL**: `http://export.arxiv.org/api/query` -- **Rate limit**: Be nice, ~3 sec between requests -- **No auth required** -- **Returns**: XML (Atom feed) -- **Best for**: Preprints, recent work not yet published - -## CrossRef - -- **Base URL**: `https://api.crossref.org/` -- **Rate limit**: 50 req/sec with polite pool (include `mailto` header) -- **No auth required** -- **Best for**: DOI lookup, published works, metadata enrichment - -## Zotero Integration - -After finding papers via external search, use Zotero MCP tools: - -1. `zotero_add_by_doi` — Add paper by DOI (fetches metadata from CrossRef) -2. `zotero_add_by_url` — Add paper by URL (arXiv, DOI URLs) -3. `zotero_update_search_database` — Update semantic search index after adding diff --git a/agentic/cpp/skills/paper-search/scripts/search_papers.py b/agentic/cpp/skills/paper-search/scripts/search_papers.py deleted file mode 100755 index c65351d..0000000 --- a/agentic/cpp/skills/paper-search/scripts/search_papers.py +++ /dev/null @@ -1,333 +0,0 @@ -#!/usr/bin/env python3 -"""Search external APIs for academic papers. - -Sources: Semantic Scholar, arXiv, CrossRef. -Outputs unified JSON to stdout. - -Usage: - python3 search_papers.py --query "wavelet tree succinct" --source semantic_scholar --limit 10 - python3 search_papers.py --query "succinct data structures" --source arxiv --limit 5 - python3 search_papers.py --doi "10.1145/123" --source crossref - python3 search_papers.py --query "rank select" --source all --limit 5 - python3 search_papers.py --query "wavelet tree" --source arxiv --limit 1 --download ~/papers - python3 search_papers.py --doi "10.1007/978-3-540-73420-8_13" --download ~/papers --zotero -""" - -import argparse -import json -import os -import re -import sys -import time -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path -from typing import Any - - -def _get(url: str, headers: dict[str, str] | None = None, timeout: int = 30, - retries: int = 2) -> dict: - for attempt in range(retries + 1): - req = urllib.request.Request(url, headers=headers or {}) - try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - return json.loads(resp.read().decode()) - except urllib.error.HTTPError as e: - if e.code == 429 and attempt < retries: - wait = 2 ** attempt - print(f"Rate limited, retrying in {wait}s...", file=sys.stderr) - time.sleep(wait) - continue - print(f"HTTP {e.code}: {e.reason} for {url}", file=sys.stderr) - return {} - except urllib.error.URLError as e: - print(f"URL error: {e.reason} for {url}", file=sys.stderr) - return {} - return {} - - -def search_semantic_scholar(query: str, limit: int = 10) -> list[dict[str, Any]]: - """Search Semantic Scholar API.""" - params = urllib.parse.urlencode({ - "query": query, - "limit": limit, - "fields": "title,authors,year,abstract,externalIds,venue,publicationDate,citationCount,url,openAccessPdf", - }) - url = f"https://api.semanticscholar.org/graph/v1/paper/search?{params}" - data = _get(url, headers={"Accept": "application/json"}) - results = [] - for paper in data.get("data", []): - ext_ids = paper.get("externalIds") or {} - pdf_info = paper.get("openAccessPdf") or {} - results.append({ - "source": "semantic_scholar", - "title": paper.get("title", ""), - "authors": [a.get("name", "") for a in paper.get("authors", [])], - "year": paper.get("year"), - "abstract": paper.get("abstract", ""), - "doi": ext_ids.get("DOI"), - "arxiv_id": ext_ids.get("ArXiv"), - "venue": paper.get("venue", ""), - "citation_count": paper.get("citationCount"), - "url": paper.get("url", ""), - "pdf_url": pdf_info.get("url"), - }) - return results - - -def search_arxiv(query: str, limit: int = 10) -> list[dict[str, Any]]: - """Search arXiv API.""" - words = query.split() - if len(words) == 1: - search_term = f"all:{query}" - elif len(words) == 2: - # Phrase search for 2-word queries - search_term = f'all:"{query}"' - else: - # Use OR of phrase and individual terms for 3+ words - # This catches exact phrase matches AND papers with all terms - phrase = f'all:"{query}"' - and_terms = " AND ".join(f"all:{w}" for w in words) - search_term = f"({phrase}) OR ({and_terms})" - params = urllib.parse.urlencode({ - "search_query": search_term, - "start": 0, - "max_results": limit, - }) - url = f"http://export.arxiv.org/api/query?{params}" - req = urllib.request.Request(url) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - xml_data = resp.read().decode() - except (urllib.error.URLError, urllib.error.HTTPError) as e: - print(f"arXiv API error: {e}", file=sys.stderr) - return [] - - import xml.etree.ElementTree as ET - root = ET.fromstring(xml_data) - ns = {"atom": "http://www.w3.org/2005/Atom"} - results = [] - for entry in root.findall("atom:entry", ns): - title = entry.findtext("atom:title", "", ns).strip().replace("\n", " ") - abstract = entry.findtext("atom:summary", "", ns).strip().replace("\n", " ") - authors = [a.findtext("atom:name", "", ns) for a in entry.findall("atom:author", ns)] - published = entry.findtext("atom:published", "", ns) - year = int(published[:4]) if published else None - arxiv_id = "" - for link in entry.findall("atom:link", ns): - href = link.get("href", "") - if "arxiv.org/abs/" in href: - arxiv_id = href.split("/abs/")[-1] - break - results.append({ - "source": "arxiv", - "title": title, - "authors": authors, - "year": year, - "abstract": abstract, - "doi": None, - "arxiv_id": arxiv_id, - "venue": "arXiv", - "citation_count": None, - "url": f"https://arxiv.org/abs/{arxiv_id}" if arxiv_id else "", - "pdf_url": f"https://arxiv.org/pdf/{arxiv_id}" if arxiv_id else None, - }) - return results - - -def search_crossref(query: str, limit: int = 10) -> list[dict[str, Any]]: - """Search CrossRef API.""" - params = urllib.parse.urlencode({ - "query": query, - "rows": limit, - "select": "DOI,title,author,published-print,abstract,container-title,is-referenced-by-count,URL,type,link", - }) - url = f"https://api.crossref.org/works?{params}" - data = _get(url, headers={"Accept": "application/json"}) - results = [] - for item in data.get("message", {}).get("items", []): - title_list = item.get("title", []) - title = title_list[0] if title_list else "" - authors = [] - for a in item.get("author", []): - name = f"{a.get('given', '')} {a.get('family', '')}".strip() - if name: - authors.append(name) - pub_date = item.get("published-print", {}).get("date-parts", [[None]]) - year = pub_date[0][0] if pub_date and pub_date[0] else None - venue_list = item.get("container-title", []) - venue = venue_list[0] if venue_list else "" - pdf_url = None - for link in item.get("link", []): - if "pdf" in link.get("content-type", ""): - pdf_url = link.get("URL") - break - results.append({ - "source": "crossref", - "title": title, - "authors": authors, - "year": year, - "abstract": item.get("abstract", ""), - "doi": item.get("DOI"), - "arxiv_id": None, - "venue": venue, - "citation_count": item.get("is-referenced-by-count"), - "url": item.get("URL", ""), - "pdf_url": pdf_url, - }) - return results - - -def lookup_doi(doi: str) -> dict[str, Any] | None: - """Look up a single paper by DOI via CrossRef.""" - url = f"https://api.crossref.org/works/{urllib.parse.quote(doi, safe='')}" - data = _get(url) - item = data.get("message") - if not item: - return None - title_list = item.get("title", []) - title = title_list[0] if title_list else "" - authors = [] - for a in item.get("author", []): - name = f"{a.get('given', '')} {a.get('family', '')}".strip() - if name: - authors.append(name) - pub_date = item.get("published-print", {}).get("date-parts", [[None]]) - year = pub_date[0][0] if pub_date and pub_date[0] else None - venue_list = item.get("container-title", []) - venue = venue_list[0] if venue_list else "" - pdf_url = None - for link in item.get("link", []): - if "pdf" in link.get("content-type", ""): - pdf_url = link.get("URL") - break - return { - "source": "crossref", - "title": title, - "authors": authors, - "year": year, - "abstract": item.get("abstract", ""), - "doi": item.get("DOI"), - "arxiv_id": None, - "venue": venue, - "citation_count": item.get("is-referenced-by-count"), - "url": item.get("URL", ""), - "pdf_url": pdf_url, - } - - -SOURCES = { - "semantic_scholar": search_semantic_scholar, - "arxiv": search_arxiv, - "crossref": search_crossref, -} - - -def _sanitize_filename(title: str) -> str: - """Generate a clean filename from paper title.""" - name = re.sub(r'[^\w\s-]', '', title.lower()) - name = re.sub(r'[\s]+', '_', name.strip()) - return name[:80] - - -def download_pdf(url: str, output_dir: str, paper: dict[str, Any]) -> str | None: - """Download a PDF and return the local path.""" - filename = _sanitize_filename(paper.get("title", "paper")) + ".pdf" - output_path = Path(output_dir) / filename - output_path.parent.mkdir(parents=True, exist_ok=True) - - req = urllib.request.Request(url, headers={ - "User-Agent": "Mozilla/5.0 (academic paper-search script)" - }) - try: - with urllib.request.urlopen(req, timeout=60) as resp: - content_type = resp.headers.get("Content-Type", "") - if "pdf" not in content_type and "octet-stream" not in content_type: - print(f"Warning: unexpected content type '{content_type}' for {url}", - file=sys.stderr) - with open(output_path, "wb") as f: - f.write(resp.read()) - print(f"Downloaded: {output_path}", file=sys.stderr) - return str(output_path) - except (urllib.error.URLError, urllib.error.HTTPError) as e: - print(f"Download failed for {url}: {e}", file=sys.stderr) - return None - - -def main(): - parser = argparse.ArgumentParser(description="Search for academic papers") - parser.add_argument("--query", "-q", help="Search query") - parser.add_argument("--doi", help="Look up a specific DOI") - parser.add_argument("--source", "-s", default="semantic_scholar", - choices=["semantic_scholar", "arxiv", "crossref", "all"], - help="API source (default: semantic_scholar)") - parser.add_argument("--limit", "-n", type=int, default=10, - help="Max results per source (default: 10)") - parser.add_argument("--format", "-f", default="json", - choices=["json", "compact"], - help="Output format (default: json)") - parser.add_argument("--download", "-d", metavar="DIR", - help="Download PDFs to DIR (requires pdf_url in results)") - parser.add_argument("--zotero", "-z", action="store_true", - help="Download PDFs and output paths for Zotero import (implies --download)") - args = parser.parse_args() - - if not args.query and not args.doi: - parser.error("Either --query or --doi is required") - - if args.zotero and not args.download: - args.download = "." - - results = [] - if args.doi: - paper = lookup_doi(args.doi) - if paper: - results.append(paper) - elif args.source == "all": - for name, func in SOURCES.items(): - try: - results.extend(func(args.query, args.limit)) - except Exception as e: - print(f"Error searching {name}: {e}", file=sys.stderr) - time.sleep(1) - else: - results = SOURCES[args.source](args.query, args.limit) - - if args.download: - for r in results: - pdf_url = r.get("pdf_url") - if pdf_url: - path = download_pdf(pdf_url, args.download, r) - r["downloaded_path"] = path - else: - r["downloaded_path"] = None - - if args.format == "json": - print(json.dumps(results, indent=2)) - else: - for i, r in enumerate(results, 1): - authors = ", ".join(r["authors"][:3]) - if len(r["authors"]) > 3: - authors += " et al." - doi_str = f" DOI: {r['doi']}" if r.get("doi") else "" - arxiv_str = f" arXiv: {r['arxiv_id']}" if r.get("arxiv_id") else "" - cite_str = f" Citations: {r['citation_count']}" if r.get("citation_count") else "" - pdf_str = f" PDF: {r['pdf_url']}" if r.get("pdf_url") else " PDF: N/A" - dl_str = "" - if r.get("downloaded_path"): - dl_str = f" Downloaded: {r['downloaded_path']}" - print(f"[{i}] {r['title']}") - print(f" {authors} ({r.get('year', '?')}) — {r.get('venue', '')}") - print(f" {r.get('url', '')}{doi_str}{arxiv_str}{cite_str}") - print(f" {pdf_str}{dl_str}") - if r.get("abstract"): - abstract = r["abstract"][:200] - if len(r["abstract"]) > 200: - abstract += "..." - print(f" {abstract}") - print() - - -if __name__ == "__main__": - main() diff --git a/agentic/cpp/skills/pdf/SKILL.md b/agentic/cpp/skills/pdf/SKILL.md deleted file mode 100644 index ddbce00..0000000 --- a/agentic/cpp/skills/pdf/SKILL.md +++ /dev/null @@ -1,112 +0,0 @@ ---- -name: pdf -description: Process PDF files - extract text, create PDFs, merge documents. Use when user asks to read PDF, create PDF, or work with PDF files. ---- - -# PDF Processing Skill - -You now have expertise in PDF manipulation. Follow these workflows: - -## Reading PDFs - -**Option 1: Quick text extraction (preferred)** -```bash -# Using pdftotext (poppler-utils) -pdftotext input.pdf - # Output to stdout -pdftotext input.pdf output.txt # Output to file - -# If pdftotext not available, try: -python3 -c " -import fitz # PyMuPDF -doc = fitz.open('input.pdf') -for page in doc: - print(page.get_text()) -" -``` - -**Option 2: Page-by-page with metadata** -```python -import fitz # pip install pymupdf - -doc = fitz.open("input.pdf") -print(f"Pages: {len(doc)}") -print(f"Metadata: {doc.metadata}") - -for i, page in enumerate(doc): - text = page.get_text() - print(f"--- Page {i+1} ---") - print(text) -``` - -## Creating PDFs - -**Option 1: From Markdown (recommended)** -```bash -# Using pandoc -pandoc input.md -o output.pdf - -# With custom styling -pandoc input.md -o output.pdf --pdf-engine=xelatex -V geometry:margin=1in -``` - -**Option 2: Programmatically** -```python -from reportlab.lib.pagesizes import letter -from reportlab.pdfgen import canvas - -c = canvas.Canvas("output.pdf", pagesize=letter) -c.drawString(100, 750, "Hello, PDF!") -c.save() -``` - -**Option 3: From HTML** -```bash -# Using wkhtmltopdf -wkhtmltopdf input.html output.pdf - -# Or with Python -python3 -c " -import pdfkit -pdfkit.from_file('input.html', 'output.pdf') -" -``` - -## Merging PDFs - -```python -import fitz - -result = fitz.open() -for pdf_path in ["file1.pdf", "file2.pdf", "file3.pdf"]: - doc = fitz.open(pdf_path) - result.insert_pdf(doc) -result.save("merged.pdf") -``` - -## Splitting PDFs - -```python -import fitz - -doc = fitz.open("input.pdf") -for i in range(len(doc)): - single = fitz.open() - single.insert_pdf(doc, from_page=i, to_page=i) - single.save(f"page_{i+1}.pdf") -``` - -## Key Libraries - -| Task | Library | Install | -|------|---------|---------| -| Read/Write/Merge | PyMuPDF | `pip install pymupdf` | -| Create from scratch | ReportLab | `pip install reportlab` | -| HTML to PDF | pdfkit | `pip install pdfkit` + wkhtmltopdf | -| Text extraction | pdftotext | `brew install poppler` / `apt install poppler-utils` | - -## Best Practices - -1. **Always check if tools are installed** before using them -2. **Handle encoding issues** - PDFs may contain various character encodings -3. **Large PDFs**: Process page by page to avoid memory issues -4. **OCR for scanned PDFs**: Use `pytesseract` if text extraction returns empty diff --git a/agentic/cpp/skills/setup-cpp-repo/SKILL.md b/agentic/cpp/skills/setup-cpp-repo/SKILL.md deleted file mode 100644 index 83201a4..0000000 --- a/agentic/cpp/skills/setup-cpp-repo/SKILL.md +++ /dev/null @@ -1,136 +0,0 @@ ---- -name: setup-cpp-repo -description: Scaffold a new C++20 repository with CMake, Google Test, Google Benchmark, CI workflows, Doxygen docs, and Chromium code style. Use when the user asks to create a new C++ project, set up a C++ library, or initialize a C++ repository with modern tooling. ---- - -# setup-cpp-repo - -## Overview - -This skill generates a complete modern C++20 project scaffold. The generated -repository is header-only by default and includes: - -- CMake build system with presets -- Google Test for unit testing -- Google Benchmark for performance benchmarks -- Doxygen documentation with doxygen-awesome-css theme -- GitHub Actions CI workflows (ASan, lint, coverage, docs) -- Chromium C++ code style via `.clang-format` -- `AGENTS.md` for AI coding assistant guidelines - -## When to Use This Skill - -Use this skill when: -- The user wants to create a new C++ library or project from scratch -- The user asks for a "C++ project template" or "C++ repo setup" -- The user needs CMake + Google Test + benchmark scaffolding -- The user wants header-only library conventions with optional SIMD-oriented - build flags and Doxygen docs - -Do **not** use this skill when: -- Working with an existing codebase (use the `cmake` skill instead) -- The project is not C++ (use a different skill) -- The user only wants a single file or snippet - -## Workflow - -### Step 1: Gather Parameters - -Ask the user for (or infer from context): -- **Project name** (required): Hyphenated lowercase identifier, e.g., `my-lib` -- **Namespace** (optional): C++ namespace. Defaults to project name with hyphens removed, e.g., `mylib` -- **Output directory** (optional): Where to create the project. Defaults to current directory. - -### Step 2: Run the Generator - -Execute the generation script: - -```bash -python3 agentic/cpp/skills/setup-cpp-repo/scripts/init_cpp_project.py \ - --name \ - [--namespace ] \ - [--output-dir ] -``` - -For concrete examples, check -`agentic/local/cpp/skills/setup-cpp-repo/EXAMPLES.md` when present. - -### Step 3: Verify the Scaffold - -After generation, the project structure should look like: - -``` -/ -├── CMakeLists.txt -├── CMakePresets.json -├── .clang-format -├── .gitignore -├── README.md -├── AGENTS.md -├── include/ -│ └── / -│ └── .hpp -├── src/ -│ ├── tests/ -│ │ └── unittests.cpp -│ ├── benchmarks/ -│ │ └── benchmarks.cpp -│ └── docs/ -│ ├── Doxyfile.in -│ └── images/ -├── scripts/ -│ └── coverage_report.sh -└── .github/ - └── workflows/ - ├── build-test.yml - ├── linter.yml - ├── coverage.yml - └── doxygen.yml -``` - -### Step 4: Initial Build and Test - -Change into the project directory and run an initial build to verify everything works: - -```bash -cd -cmake --preset release -cmake --build --preset release -j -./build/release/unittests -``` - -If the build and tests pass, the scaffold is ready. - -### Step 5: Hand Off to cmake Skill - -After project creation, use the **`cmake` skill** (`../cmake/SKILL.md`) for all subsequent build operations. The `cmake` skill documents: -- Build directory conventions with git short-hash suffixes -- How to replicate preset settings with custom build directories -- AddressSanitizer, coverage, and benchmark workflows -- Best practices for out-of-source builds - -## Customization Guide - -### Adding More Test Executables - -Edit `CMakeLists.txt` and add new `add_executable` blocks under the `if(_TESTS)` section, following the pattern of the existing `unittests` target. - -Update `scripts/coverage_report.sh` to run any new test binaries. - -Update `.github/workflows/build-test.yml` to execute new test binaries in CI. - -### Adding More Benchmark Executables - -Edit `CMakeLists.txt` and add new `add_executable` blocks under the `if(_BENCHMARKS)` section, following the pattern of the existing `benchmarks` target. - -### Adding Third-Party Dependencies - -For header-only libraries, prefer `FetchContent` in `CMakeLists.txt`. For compiled libraries, consider vendoring or using a package manager (Conan, vcpkg). - -### Modifying Doxygen Configuration - -Edit `src/docs/Doxyfile.in`. The generated version is intentionally minimal (only non-default settings). Add or override settings as needed. Run `doxygen -g` to see all available options. - -## Reference - -See `references/project_structure.md` for a detailed breakdown of every generated file and its purpose. diff --git a/agentic/cpp/skills/setup-cpp-repo/references/project_structure.md b/agentic/cpp/skills/setup-cpp-repo/references/project_structure.md deleted file mode 100644 index 6bf3236..0000000 --- a/agentic/cpp/skills/setup-cpp-repo/references/project_structure.md +++ /dev/null @@ -1,117 +0,0 @@ -# Generated Project Structure Reference - -This document describes every file and directory generated by `init_cpp_project.py` and its purpose. - -## Root Files - -### `CMakeLists.txt` -Main CMake configuration. Defines: -- C++20 standard requirements -- `MARCH` cache variable (defaults to `native`) -- Optional SIMD fallback flag when the generated project enables SIMD-specific - code paths -- `ENABLE_ADDRESS_SANITIZER` option for ASan builds -- `_COVERAGE` option for gcov instrumentation -- Build options: `_TESTS`, `_BENCHMARKS`, `_DIAGNOSTICS`, `_DOCS` -- FetchContent dependencies: Google Test, Google Benchmark, spdlog (diagnostics only), Doxygen theme -- Test executable: `unittests` -- Benchmark executable: `benchmarks` -- Custom target: `docs` (when Doxygen is enabled) - -### `CMakePresets.json` -CMake presets (version 4) with a hidden `base` preset. Defines presets for: -- `debug` — Debug build -- `release` — Release build -- `benchmarks` — Release with benchmarks enabled -- `benchmarks-diagnostic` — RelWithDebInfo with diagnostics and libpfm -- `docs` — Documentation build -- `coverage` — Debug with coverage instrumentation -- `asan` — Debug with AddressSanitizer - -### `.clang-format` -Chromium-based C++ formatting configuration. Simplified from the full Chromium style by removing Windows-specific include priorities and IPC macro block definitions. Key settings: -- `BasedOnStyle: Chromium` -- `Standard: Cpp11` -- `InsertBraces: true` -- `InsertNewlineAtEOF: true` -- `IncludeBlocks: Regroup` with generic priority categories - -### `.gitignore` -Standard C++ project ignores: -- `build/`, `.vscode/`, `Testing/` -- `plans/*`, `venv/`, `docs/*` -- `CMakeUserPresets.json` -- `_deps/`, gcov outputs (`*.gcda`, `*.gcno`, `*.gcov`) - -### `README.md` -Minimal project README used as the Doxygen main page. - -### `AGENTS.md` -Project documentation for AI coding assistants. Contains: -- Project overview and architecture conventions -- Technology stack (C++20, CMake, Google Test, Google Benchmark) -- Build commands with all CMake options -- Testing patterns and style guidelines -- Common tasks for AI agents (adding components, modifying SIMD code, adding tests) -- Performance philosophy - -## Directories - -### `include//` -Header-only library API. Contains a placeholder header (`.hpp`) with: -- Doxygen file documentation -- Example function in the project's namespace -- `#pragma once` guard - -### `src/tests/` -Unit test scaffold. Contains `unittests.cpp` with: -- Google Test includes -- Basic assertion test against the placeholder header -- `gtest_main` supplies the test runner entry point - -### `src/benchmarks/` -Benchmark scaffold. Contains `benchmarks.cpp` with: -- Google Benchmark includes -- Example benchmark using `benchmark::DoNotOptimize` -- `BENCHMARK_MAIN()` macro - -### `src/docs/` -Doxygen configuration. Contains: -- `Doxyfile.in` — Trimmed Doxygen config (~300 lines vs. 1100+ in full). Only non-default settings are specified. Key templated values: - - `PROJECT_NAME` - - `INPUT` (points to `include/` and `README.md`) - - `STRIP_FROM_PATH` (strips source dir from file paths) - - `IMAGE_PATH` - - `HTML_EXTRA_STYLESHEET` (doxygen-awesome-css) - - `USE_MDFILE_AS_MAINPAGE` -- `images/` — Empty directory for documentation images - -### `scripts/` -Utility scripts. Contains: -- `coverage_report.sh` — Runs the `coverage` CMake preset, executes tests, and generates gcov reports. Excludes `_deps/`, `third_party/`, and `src/benchmarks/` from coverage. - -### `.github/workflows/` -CI/CD workflows: - -#### `build-test.yml` -Builds the project with AddressSanitizer and runs unit tests on `ubuntu-latest`. Triggered on pushes and PRs to `main`. - -#### `linter.yml` -Runs `clang-format --dry-run --Werror` on all C/C++ files. Triggered on pushes to `main` and all PRs. - -#### `coverage.yml` -Runs the coverage script and uploads results to Codecov. Also uploads coverage artifacts. Triggered on pushes and PRs to `main`. - -#### `doxygen.yml` -Installs Doxygen, builds documentation with the `docs` preset, and deploys HTML output to GitHub Pages. Triggered on pushes to `main` and manual dispatch. - -## Template Substitution - -All generated files use these placeholders, replaced by the script: - -| Placeholder | Example input | Example output | -|-------------|---------------|----------------| -| `{{PROJECT_NAME}}` | `my-lib` | `my-lib` | -| `{{NAMESPACE}}` | `mylib` | `mylib` | -| `{{PROJECT_NAME_UPPER}}` | `MY_LIB` | `MY_LIB` | -| `{{HEADER_NAME}}` | `my_lib.hpp` | `my_lib.hpp` | diff --git a/agentic/cpp/skills/setup-cpp-repo/scripts/init_cpp_project.py b/agentic/cpp/skills/setup-cpp-repo/scripts/init_cpp_project.py deleted file mode 100755 index d76624f..0000000 --- a/agentic/cpp/skills/setup-cpp-repo/scripts/init_cpp_project.py +++ /dev/null @@ -1,1053 +0,0 @@ -#!/usr/bin/env python3 -""" -init_cpp_project.py - Scaffold a new C++20 repository following modern C++ conventions. - -Usage: - init_cpp_project.py --name [--namespace ] [--output-dir ] - -Example: - init_cpp_project.py --name my-lib --namespace mylib --output-dir . -""" - -import argparse -import os -import sys -from pathlib import Path - - -# --------------------------------------------------------------------------- -# Helper functions -# --------------------------------------------------------------------------- - -def to_upper(name: str) -> str: - """Convert project name to uppercase with underscores.""" - return name.replace("-", "_").upper() - - -def to_snake(name: str) -> str: - """Convert project name to snake_case for filenames.""" - return name.replace("-", "_") - - -# --------------------------------------------------------------------------- -# Templates -# --------------------------------------------------------------------------- - -CMAKE_LISTS_TXT = """cmake_minimum_required(VERSION 3.18) -project({{PROJECT_NAME}}) - -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) - -set(MARCH "native" CACHE STRING "march compiler flag") -add_compile_options("-march=${MARCH}") -message(STATUS "MARCH is '${MARCH}'") - -option({{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD "Disable wide SIMD instructions" OFF) -if({{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD) - add_compile_options("-mno-avx512f") - message(STATUS "{{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD is ON") -endif() - -option(ENABLE_ADDRESS_SANITIZER "Enable AddressSanitizer" OFF) -if(ENABLE_ADDRESS_SANITIZER) - add_compile_options(-fsanitize=address -fno-omit-frame-pointer) - add_link_options(-fsanitize=address) - message(STATUS "AddressSanitizer is ON") -endif() - -option({{PROJECT_NAME_UPPER}}_COVERAGE "Enable coverage instrumentation" OFF) -if({{PROJECT_NAME_UPPER}}_COVERAGE) - add_compile_options(-O0 -g --coverage) - add_link_options(--coverage) - message(STATUS "Coverage instrumentation is ON") -endif() - -# --------------------------------------------------------------------------- -# Build options -# --------------------------------------------------------------------------- -option({{PROJECT_NAME_UPPER}}_TESTS "Build unit tests" ON) -option({{PROJECT_NAME_UPPER}}_BENCHMARKS "Build benchmarks" OFF) -option({{PROJECT_NAME_UPPER}}_DIAGNOSTICS "Include diagnostic logs" OFF) -option({{PROJECT_NAME_UPPER}}_DOCS "Build Doxygen documentation" OFF) - -if({{PROJECT_NAME_UPPER}}_DIAGNOSTICS) - add_compile_definitions({{PROJECT_NAME_UPPER}}_DIAGNOSTICS) - set({{PROJECT_NAME_UPPER}}_DIAGNOSTICS_LIBS spdlog::spdlog_header_only) -endif() - -# --------------------------------------------------------------------------- -# Dependencies (fetched only when needed) -# --------------------------------------------------------------------------- -include(FetchContent) - -if({{PROJECT_NAME_UPPER}}_DIAGNOSTICS) - set(SPDLOG_BUILD_SHARED OFF CACHE BOOL "" FORCE) - set(SPDLOG_BUILD_EXAMPLE OFF CACHE BOOL "" FORCE) - set(SPDLOG_BUILD_TESTING OFF CACHE BOOL "" FORCE) - set(SPDLOG_INSTALL OFF CACHE BOOL "" FORCE) - FetchContent_Declare( - spdlog - GIT_REPOSITORY https://github.com/gabime/spdlog.git - GIT_TAG v1.14.1 - ) - FetchContent_MakeAvailable(spdlog) -endif() - -if({{PROJECT_NAME_UPPER}}_BENCHMARKS) - FetchContent_Declare( - googlebenchmark - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.9.4 - ) - set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable Google Benchmark tests") - FetchContent_MakeAvailable(googlebenchmark) -endif() - -if({{PROJECT_NAME_UPPER}}_TESTS) - if(NOT TARGET gtest_main) - FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.17.0 - ) - FetchContent_MakeAvailable(googletest) - endif() - include(GoogleTest) -endif() - -# --------------------------------------------------------------------------- -# Unit tests -# --------------------------------------------------------------------------- -if({{PROJECT_NAME_UPPER}}_TESTS) - enable_testing() - - add_executable(unittests - src/tests/unittests.cpp) - target_include_directories(unittests - PUBLIC include) - target_link_libraries(unittests - gtest_main - ${{{PROJECT_NAME_UPPER}}_DIAGNOSTICS_LIBS}) - gtest_discover_tests(unittests) -endif() - -# --------------------------------------------------------------------------- -# Benchmarks -# --------------------------------------------------------------------------- -if({{PROJECT_NAME_UPPER}}_BENCHMARKS) - add_executable(benchmarks - src/benchmarks/benchmarks.cpp) - target_include_directories(benchmarks - PUBLIC include) - target_link_libraries(benchmarks - benchmark - benchmark_main - ${{{PROJECT_NAME_UPPER}}_DIAGNOSTICS_LIBS}) -endif() - -# --------------------------------------------------------------------------- -# Documentation (Doxygen) -# --------------------------------------------------------------------------- -if({{PROJECT_NAME_UPPER}}_DOCS) - find_package(Doxygen REQUIRED) - - FetchContent_Declare( - doxygen-awesome-css - URL https://github.com/jothepro/doxygen-awesome-css/archive/refs/heads/main.zip - ) - FetchContent_MakeAvailable(doxygen-awesome-css) - - FetchContent_GetProperties(doxygen-awesome-css SOURCE_DIR AWESOME_CSS_DIR) - - set(DOXYFILE_IN ${CMAKE_CURRENT_SOURCE_DIR}/src/docs/Doxyfile.in) - set(DOXYFILE_OUT ${CMAKE_CURRENT_BINARY_DIR}/docs/Doxyfile) - configure_file(${DOXYFILE_IN} ${DOXYFILE_OUT} @ONLY) - - add_custom_target(docs - COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_OUT} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Generating API documentation with Doxygen" - VERBATIM) -endif() -""" - -CMAKE_PRESETS_JSON = """{ - "version": 4, - "cmakeMinimumRequired": { - "major": 3, - "minor": 18, - "patch": 0 - }, - "configurePresets": [ - { - "name": "base", - "hidden": true, - "cacheVariables": { - "CMAKE_EXPORT_COMPILE_COMMANDS": "ON" - } - }, - { - "name": "debug", - "displayName": "Debug", - "inherits": "base", - "binaryDir": "${sourceDir}/build/debug", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" - } - }, - { - "name": "release", - "displayName": "Release", - "inherits": "base", - "binaryDir": "${sourceDir}/build/release", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" - } - }, - { - "name": "benchmarks", - "displayName": "Benchmarks", - "inherits": "base", - "binaryDir": "${sourceDir}/build/benchmarks", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release", - "{{PROJECT_NAME_UPPER}}_BENCHMARKS": "ON" - } - }, - { - "name": "benchmarks-diagnostic", - "displayName": "Benchmarks diagnostic build", - "inherits": "base", - "binaryDir": "${sourceDir}/build/release-with-deb", - "cacheVariables": { - "BENCHMARK_ENABLE_LIBPFM": "ON", - "CMAKE_BUILD_TYPE": "RelWithDebInfo", - "{{PROJECT_NAME_UPPER}}_DIAGNOSTICS": "ON", - "{{PROJECT_NAME_UPPER}}_BENCHMARKS": "ON" - } - }, - { - "name": "docs", - "displayName": "Docs", - "inherits": "base", - "binaryDir": "${sourceDir}/build/docs", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release", - "{{PROJECT_NAME_UPPER}}_DOCS": "ON" - } - }, - { - "name": "coverage", - "displayName": "Coverage", - "inherits": "base", - "binaryDir": "${sourceDir}/build/coverage", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "{{PROJECT_NAME_UPPER}}_BENCHMARKS": "OFF", - "{{PROJECT_NAME_UPPER}}_COVERAGE": "ON" - } - }, - { - "name": "asan", - "displayName": "AddressSanitizer", - "inherits": "base", - "binaryDir": "${sourceDir}/build/asan", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "{{PROJECT_NAME_UPPER}}_BENCHMARKS": "OFF", - "ENABLE_ADDRESS_SANITIZER": "ON" - } - } - ], - "buildPresets": [ - { - "name": "debug", - "displayName": "Build Debug", - "configurePreset": "debug" - }, - { - "name": "release", - "displayName": "Build Release", - "configurePreset": "release" - }, - { - "name": "benchmarks", - "displayName": "Build Benchmarks", - "configurePreset": "benchmarks" - }, - { - "name": "benchmarks-diagnostic", - "displayName": "Benchmarks diagnostic", - "configurePreset": "benchmarks-diagnostic" - }, - { - "name": "docs", - "displayName": "Build Docs", - "configurePreset": "docs", - "targets": [ - "docs" - ] - }, - { - "name": "coverage", - "displayName": "Build Coverage", - "configurePreset": "coverage" - }, - { - "name": "asan", - "displayName": "Build AddressSanitizer", - "configurePreset": "asan" - } - ] -} -""" - -CLANG_FORMAT = """# Defines the Chromium style for automatic reformatting. -# http://clang.llvm.org/docs/ClangFormatStyleOptions.html -BasedOnStyle: Chromium -# This defaults to 'Auto'. Explicitly set it for a while, so that -# 'vector >' in existing files gets formatted to -# 'vector>'. ('Auto' means that clang-format will only use -# 'int>>' if the file already contains at least one such instance.) -Standard: Cpp11 - -# TODO(crbug.com/1392808): Remove when InsertBraces has been upstreamed into -# the Chromium style (is implied by BasedOnStyle: Chromium). -InsertBraces: true -InsertNewlineAtEOF: true - -# Sort #includes by following -# https://google.github.io/styleguide/cppguide.html#Names_and_Order_of_Includes -IncludeBlocks: Regroup -IncludeCategories: - # C system headers. - - Regex: '^<.*\\.h>' - Priority: 1 - # C++ standard library headers. - - Regex: '^<.*>' - Priority: 2 - # Project headers (quoted includes). - - Regex: '^".*"' - Priority: 3 - # Other libraries. - - Regex: '.*' - Priority: 4 -""" - -GITIGNORE = """build/ -.vscode/ -Testing/ -plans/* -venv/ -docs/* -src/docs/presentations/* -CMakeUserPresets.json -_deps/ -*.gcda -*.gcno -*.gcov -""" - -README_MD = """# {{PROJECT_NAME}} - -{{PROJECT_NAME}} is a C++20 header-only library. - -## Build - -```bash -cmake --preset release -cmake --build --preset release -j -./build/release/unittests -``` -""" - -AGENTS_MD = """# AGENTS.md - AI Coding Assistant Guidelines for {{PROJECT_NAME}} - -## Project Overview - -{{PROJECT_NAME}} is a **C++20 header-only library**. It provides [TODO: brief description]. - -## Skills - -Shared C++ agent skills live in `agentic/cpp/skills` when this repository -vendors the shared skills subtree. Project-specific examples live in -`agentic/local/cpp/skills`. - -## Architecture - -### Project Layout Conventions - -- **`include/`**: Header-only library API (all implementations here, no `.cpp` files) -- **`src/*_tests.cpp`**: Unit tests (Google Test) -- **`src/*_benchmarks.cpp`**: Performance benchmarks (Google Benchmark) -- **`src/docs/`**: Doxygen configuration - -### Key Design Decisions - -1. **Header-only library**: All code in `include/`; no compiled library. -2. **Non-owning spans**: Use `std::span` for external data where appropriate. -3. **SIMD conditional compilation**: Use `#ifdef {{PROJECT_NAME_UPPER}}_AVX512_SUPPORT` / `{{PROJECT_NAME_UPPER}}_AVX2_SUPPORT` with scalar fallbacks. -4. **Target domain**: Optimized for practical data sizes. -5. **Platform**: Linux/Unix is the primary target platform. - -### Why Header-Only? - -- **SIMD flexibility**: Users compile with their target `-march` flags. -- **Better inlining**: Compiler sees full implementation. -- **No ABI issues**: Works across compilers and standard library versions. -- **Easy integration**: Users just `#include` headers. -- **Template-friendly**: No explicit instantiation needed. - -## Technology Stack - -- **Language**: C++20 (required features: `std::span`, `std::popcount`, ``) -- **Build**: CMake >= 3.18 -- **Testing**: Google Test v1.17.0 -- **Benchmarking**: Google Benchmark v1.9.4 -- **SIMD**: AVX-512 (primary), AVX2 (fallback), scalar fallbacks -- **Style**: Chromium C++ style (`.clang-format`) - -### Dependencies - -The library itself is header-only and has **no runtime dependencies**. Build-time dependencies are managed via CMake FetchContent and controlled by options: - -| Option | Default | What it enables | -|--------|---------|-----------------| -| `{{PROJECT_NAME_UPPER}}_TESTS` | `ON` | Unit tests (fetches Google Test) | -| `{{PROJECT_NAME_UPPER}}_BENCHMARKS` | `OFF` | Benchmarks (fetches Google Benchmark) | - -## Build Commands - -```bash -# Standard build (Release) -cmake -B build/release -DCMAKE_BUILD_TYPE=Release -cmake --build build/release -j - -# Debug build -cmake -B build/debug -DCMAKE_BUILD_TYPE=Debug -cmake --build build/debug -j - -# Without wide SIMD -cmake -B build/release -D{{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD=ON -cmake --build build/release -j - -# With AddressSanitizer -cmake -B build/asan -DENABLE_ADDRESS_SANITIZER=ON -cmake --build build/asan -j - -# Custom march flag -cmake -B build/release -DMARCH=icelake-client -cmake --build build/release -j - -# Tests only (no benchmarks) -cmake -B build/release -D{{PROJECT_NAME_UPPER}}_BENCHMARKS=OFF -cmake --build build/release -j -``` - -## Testing - -### Running Tests - -```bash -./build/release/unittests -``` - -### Testing Patterns - -- **Differential testing**: Compare against naive reference implementations. -- **Randomized testing**: Random inputs with configurable seed. -- **Exhaustive short inputs**: Test all patterns for small sizes. - -## Code Style Guidelines - -1. **Formatting**: Run `clang-format` before committing (Chromium style) -2. **Namespace**: All library code in `{{NAMESPACE}}` namespace -3. **Documentation**: Use Doxygen-style comments for public API -4. **Constants**: Use `constexpr` for compile-time values -5. **Alignment**: Be aware of data alignment; prefer 64-byte aligned array allocations where performance matters - -## CI/CD Workflows - -- **build-test.yml**: Builds and runs tests with AddressSanitizer -- **linter.yml**: Clang-format checks on all C/C++ files -- **coverage.yml**: Coverage reporting with codecov upload -- **doxygen.yml**: Documentation generation and GitHub Pages deployment - -## Common Tasks for AI Agents - -### Adding a New Component - -1. Create header in `include/{{NAMESPACE}}/` with Doxygen documentation -2. Add unit tests in `src/tests/_tests.cpp` -3. Add benchmarks in `src/benchmarks/_benchmarks.cpp` -4. Update `CMakeLists.txt` with new executables -5. Run `clang-format` on new files - -### Modifying SIMD Code - -1. Provide implementations for: - - Wide SIMD (`#ifdef {{PROJECT_NAME_UPPER}}_AVX512_SUPPORT`) - - AVX2 (`#ifdef {{PROJECT_NAME_UPPER}}_AVX2_SUPPORT`) - - Scalar fallback -2. Test with `-D{{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD=ON` to verify fallback works -3. Benchmark to ensure performance is maintained - -### Adding Tests - -1. Use Google Test framework -2. Include naive reference implementation for differential testing -3. Add edge cases: empty input, single element, boundary conditions -4. Use random testing with configurable seed for reproducibility - -## Performance Philosophy - -- **Goal**: Best practical performance (not just asymptotic complexity) -- **Approach**: Benchmark-driven optimization using Google Benchmark -- **SIMD**: Leverage vectorized operations where beneficial -- **Cache efficiency**: Align data structures to cache line boundaries (64 bytes) -""" - -DOXYFILE_IN = """# Doxyfile - -DOXYFILE_ENCODING = UTF-8 -PROJECT_NAME = "{{PROJECT_NAME}}" -PROJECT_NUMBER = -PROJECT_BRIEF = -PROJECT_LOGO = -PROJECT_ICON = -OUTPUT_DIRECTORY = docs -CREATE_SUBDIRS = NO -CREATE_SUBDIRS_LEVEL = 8 -ALLOW_UNICODE_NAMES = NO -OUTPUT_LANGUAGE = English -BRIEF_MEMBER_DESC = YES -REPEAT_BRIEF = YES -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the -ALWAYS_DETAILED_SEC = NO -INLINE_INHERITED_MEMB = NO -FULL_PATH_NAMES = YES -STRIP_FROM_PATH = @CMAKE_CURRENT_SOURCE_DIR@ -STRIP_FROM_INC_PATH = -SHORT_NAMES = NO -JAVADOC_AUTOBRIEF = NO -JAVADOC_BANNER = NO -QT_AUTOBRIEF = NO -MULTILINE_CPP_IS_BRIEF = NO -PYTHON_DOCSTRING = YES -INHERIT_DOCS = YES -SEPARATE_MEMBER_PAGES = NO -TAB_SIZE = 4 -ALIASES = -OPTIMIZE_OUTPUT_FOR_C = NO -OPTIMIZE_OUTPUT_JAVA = NO -OPTIMIZE_FOR_FORTRAN = NO -OPTIMIZE_OUTPUT_VHDL = NO -OPTIMIZE_OUTPUT_SLICE = NO -EXTENSION_MAPPING = -MARKDOWN_SUPPORT = YES -MARKDOWN_STRICT = YES -TOC_INCLUDE_HEADINGS = 6 -MARKDOWN_ID_STYLE = DOXYGEN -AUTOLINK_SUPPORT = YES -AUTOLINK_IGNORE_WORDS = -BUILTIN_STL_SUPPORT = NO -CPP_CLI_SUPPORT = NO -SIP_SUPPORT = NO -IDL_PROPERTY_SUPPORT = YES -DISTRIBUTE_GROUP_DOC = NO -GROUP_NESTED_COMPOUNDS = NO -SUBGROUPING = YES -INLINE_GROUPED_CLASSES = NO -INLINE_SIMPLE_STRUCTS = NO -TYPEDEF_HIDES_STRUCT = NO -LOOKUP_CACHE_SIZE = 0 -NUM_PROC_THREADS = 1 -TIMESTAMP = NO -EXTRACT_ALL = NO -EXTRACT_PRIVATE = NO -EXTRACT_PRIV_VIRTUAL = NO -EXTRACT_PACKAGE = NO -EXTRACT_STATIC = NO -EXTRACT_LOCAL_CLASSES = YES -EXTRACT_LOCAL_METHODS = NO -EXTRACT_ANON_NSPACES = NO -RESOLVE_UNNAMED_PARAMS = YES -HIDE_UNDOC_MEMBERS = NO -HIDE_UNDOC_CLASSES = NO -HIDE_UNDOC_NAMESPACES = YES -HIDE_FRIEND_COMPOUNDS = NO -HIDE_IN_BODY_DOCS = NO -INTERNAL_DOCS = NO -CASE_SENSE_NAMES = SYSTEM -HIDE_SCOPE_NAMES = NO -HIDE_COMPOUND_REFERENCE= NO -SHOW_HEADERFILE = YES -SHOW_INCLUDE_FILES = YES -SHOW_GROUPED_MEMB_INC = NO -FORCE_LOCAL_INCLUDES = NO -INLINE_INFO = YES -SORT_MEMBER_DOCS = YES -SORT_BRIEF_DOCS = NO -SORT_MEMBERS_CTORS_1ST = NO -SORT_GROUP_NAMES = NO -SORT_BY_SCOPE_NAME = NO -STRICT_PROTO_MATCHING = NO -GENERATE_TODOLIST = YES -GENERATE_TESTLIST = YES -GENERATE_BUGLIST = YES -GENERATE_DEPRECATEDLIST= YES -ENABLED_SECTIONS = -MAX_INITIALIZER_LINES = 30 -SHOW_USED_FILES = YES -SHOW_FILES = YES -SHOW_NAMESPACES = YES -FILE_VERSION_FILTER = -LAYOUT_FILE = -CITE_BIB_FILES = -EXTERNAL_TOOL_PATH = -QUIET = NO -WARNINGS = YES -WARN_IF_UNDOCUMENTED = YES -WARN_IF_DOC_ERROR = YES -WARN_IF_INCOMPLETE_DOC = YES -WARN_NO_PARAMDOC = NO -WARN_IF_UNDOC_ENUM_VAL = NO -WARN_LAYOUT_FILE = YES -WARN_AS_ERROR = NO -WARN_FORMAT = "$file:$line: $text" -WARN_LINE_FORMAT = "at line $line of file $file" -WARN_LOGFILE = -INPUT = @CMAKE_CURRENT_SOURCE_DIR@/include \ - @CMAKE_CURRENT_SOURCE_DIR@/README.md -INPUT_ENCODING = UTF-8 -INPUT_FILE_ENCODING = -FILE_PATTERNS = *.c \ - *.cc \ - *.cxx \ - *.cpp \ - *.h \ - *.hh \ - *.hxx \ - *.hpp -RECURSIVE = YES -EXCLUDE = -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = -EXCLUDE_SYMBOLS = -EXAMPLE_PATH = -EXAMPLE_PATTERNS = * -EXAMPLE_RECURSIVE = NO -IMAGE_PATH = @CMAKE_CURRENT_SOURCE_DIR@/src/docs/images -INPUT_FILTER = -FILTER_PATTERNS = -FILTER_SOURCE_FILES = NO -FILTER_SOURCE_PATTERNS = -USE_MDFILE_AS_MAINPAGE = @CMAKE_CURRENT_SOURCE_DIR@/README.md -IMPLICIT_DIR_DOCS = YES -FORTRAN_COMMENT_AFTER = 72 -SOURCE_BROWSER = NO -INLINE_SOURCES = NO -STRIP_CODE_COMMENTS = YES -REFERENCED_BY_RELATION = NO -REFERENCES_RELATION = NO -REFERENCES_LINK_SOURCE = YES -SOURCE_TOOLTIPS = YES -USE_HTAGS = NO -VERBATIM_HEADERS = YES -CLANG_ASSISTED_PARSING = NO -CLANG_ADD_INC_PATHS = YES -CLANG_OPTIONS = -CLANG_DATABASE_PATH = -ALPHABETICAL_INDEX = YES -IGNORE_PREFIX = -GENERATE_HTML = YES -HTML_OUTPUT = html -HTML_FILE_EXTENSION = .html -HTML_HEADER = -HTML_FOOTER = -HTML_STYLESHEET = -HTML_EXTRA_STYLESHEET = @AWESOME_CSS_DIR@/doxygen-awesome.css -HTML_EXTRA_FILES = -HTML_COLORSTYLE = AUTO_LIGHT -HTML_COLORSTYLE_HUE = 220 -HTML_COLORSTYLE_SAT = 100 -HTML_COLORSTYLE_GAMMA = 80 -HTML_DYNAMIC_MENUS = YES -HTML_DYNAMIC_SECTIONS = NO -HTML_CODE_FOLDING = YES -HTML_COPY_CLIPBOARD = YES -HTML_PROJECT_COOKIE = -HTML_INDEX_NUM_ENTRIES = 100 -GENERATE_DOCSET = NO -DOCSET_FEEDNAME = "Doxygen generated docs" -DOCSET_FEEDURL = -DOCSET_BUNDLE_ID = org.doxygen.Project -DOCSET_PUBLISHER_ID = org.doxygen.Publisher -DOCSET_PUBLISHER_NAME = Publisher -GENERATE_HTMLHELP = NO -CHM_FILE = -HHC_LOCATION = -GENERATE_CHI = NO -CHM_INDEX_ENCODING = -BINARY_TOC = NO -TOC_EXPAND = NO -SITEMAP_URL = -GENERATE_QHP = NO -QCH_FILE = -QHP_NAMESPACE = org.doxygen.Project -QHP_VIRTUAL_FOLDER = doc -QHP_CUST_FILTER_NAME = -QHP_CUST_FILTER_ATTRS = -QHP_SECT_FILTER_ATTRS = -QHG_LOCATION = -GENERATE_ECLIPSEHELP = NO -ECLIPSE_DOC_ID = org.doxygen.Project -DISABLE_INDEX = NO -GENERATE_TREEVIEW = YES -PAGE_OUTLINE_PANEL = YES -FULL_SIDEBAR = NO -ENUM_VALUES_PER_LINE = 4 -SHOW_ENUM_VALUES = NO -TREEVIEW_WIDTH = 250 -EXT_LINKS_IN_WINDOW = NO -OBFUSCATE_EMAILS = YES -HTML_FORMULA_FORMAT = png -FORMULA_FONTSIZE = 10 -FORMULA_MACROFILE = -USE_MATHJAX = NO -MATHJAX_VERSION = MathJax_2 -MATHJAX_FORMAT = HTML-CSS -MATHJAX_RELPATH = -MATHJAX_EXTENSIONS = -MATHJAX_CODEFILE = -SEARCHENGINE = YES -SERVER_BASED_SEARCH = NO -EXTERNAL_SEARCH = NO -SEARCHENGINE_URL = -SEARCHDATA_FILE = searchdata.xml -EXTERNAL_SEARCH_ID = -EXTRA_SEARCH_MAPPINGS = -GENERATE_LATEX = NO -""" - -COVERAGE_REPORT_SH = """#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -BUILD_DIR="${ROOT_DIR}/build/coverage" - -cmake --preset coverage -cmake --build --preset coverage - -"${BUILD_DIR}/unittests" - -cd "${BUILD_DIR}" -find . -name "*.gcda" > gcov_files.txt -while read -r f; do - case "${f}" in - *"/_deps/"*|*"/third_party/"*|*"/src/benchmarks/"*) - ;; - *) - gcov -pb "${f}" >> coverage.txt - ;; - esac -done < gcov_files.txt -echo "gcov report written to ${BUILD_DIR}/coverage.txt" -""" - -BUILD_TEST_YML = """name: Tests (ASan) - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build-and-test: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Create Build Directory - run: mkdir build - - - name: Configure CMake - working-directory: ./build - run: cmake -D{{PROJECT_NAME_UPPER}}_DISABLE_WIDE_SIMD=ON -DENABLE_ADDRESS_SANITIZER=ON -D{{PROJECT_NAME_UPPER}}_BENCHMARKS=OFF .. - - - name: Build Project - working-directory: ./build - run: make -j - - - name: Run Unittests - working-directory: ./build - run: ./unittests -""" - -LINTER_YML = """name: Clang Format Lint - -on: - pull_request: - push: - branches: [main] - -jobs: - clang-format: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install clang-format - run: sudo apt-get update && sudo apt-get install -y clang-format - - - name: Run clang-format check - run: | - mapfile -t FILES < <(find include src -type f \\( -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.c' -o -name '*.h' \\)) - clang-format --version - if [ ${#FILES[@]} -eq 0 ]; then - echo "No C/C++ files found." - exit 0 - fi - - clang-format --dry-run --Werror "${FILES[@]}" -""" - -COVERAGE_YML = """name: coverage - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - coverage: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Create Build Directory - run: mkdir build - - - name: Run coverage - run: ./scripts/coverage_report.sh - - - name: Upload to Codecov - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: build/coverage/coverage.txt - flags: gcov - fail_ci_if_error: false - - - name: Upload coverage artifacts - uses: actions/upload-artifact@v4 - with: - name: coverage-gcov - path: | - build/coverage/coverage.txt - build/coverage/*.gcov -""" - -DOXYGEN_YML = """# Simple workflow for deploying static content to GitHub Pages -name: Deploy static content to Pages - -on: - # Runs on pushes targeting the default branch - push: - branches: ["main"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - # Single deploy job since we're just deploying - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Doxygen v1.13.2 - run: | - transformed_version=$(echo "1.13.2" | tr '.' '_') - wget https://github.com/doxygen/doxygen/releases/download/Release_${transformed_version}/doxygen-1.13.2.linux.bin.tar.gz - tar -xzf doxygen-1.13.2.linux.bin.tar.gz - sudo mv doxygen-1.13.2/bin/doxygen /usr/local/bin/doxygen - shell: bash - - name: Cmake configure - run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -D{{PROJECT_NAME_UPPER}}_DOCS=ON -D{{PROJECT_NAME_UPPER}}_TESTS=OFF -D{{PROJECT_NAME_UPPER}}_BENCHMARKS=OFF - - name: Build docs - run: cmake --build ${{github.workspace}}/build --target docs - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - # Upload entire repository - path: ${{github.workspace}}/build/docs/html - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 -""" - -HEADER_HPP = """#pragma once - -/** - * @file {{HEADER_NAME}} - * @brief Main header for the {{PROJECT_NAME}} library - */ - -namespace {{NAMESPACE}} { - -/** - * @brief Example function. - * - * TODO: Replace with actual library functionality. - */ -inline int example() { - return 42; -} - -} // namespace {{NAMESPACE}} -""" - -UNITTESTS_CPP = """#include - -#include "{{NAMESPACE}}/{{HEADER_NAME}}" - -TEST(ExampleTest, BasicAssertion) { - EXPECT_EQ({{NAMESPACE}}::example(), 42); -} -""" - -BENCHMARKS_CPP = """#include - -#include "{{NAMESPACE}}/{{HEADER_NAME}}" - -static void BM_Example(benchmark::State& state) { - for (auto _ : state) { - benchmark::DoNotOptimize({{NAMESPACE}}::example()); - } -} - -BENCHMARK(BM_Example); - -BENCHMARK_MAIN(); -""" - - -# --------------------------------------------------------------------------- -# Generation logic -# --------------------------------------------------------------------------- - -def generate(args: argparse.Namespace) -> None: - project_name = args.name - namespace = args.namespace or project_name.replace("-", "") - project_name_upper = to_upper(project_name) - header_name = f"{to_snake(project_name)}.hpp" - output_dir = Path(args.output_dir).resolve() / project_name - - if output_dir.exists(): - print(f"Error: output directory already exists: {output_dir}") - sys.exit(1) - - substitutions = { - "{{PROJECT_NAME}}": project_name, - "{{NAMESPACE}}": namespace, - "{{PROJECT_NAME_UPPER}}": project_name_upper, - "{{HEADER_NAME}}": header_name, - } - - def sub(text: str) -> str: - for key, value in substitutions.items(): - text = text.replace(key, value) - return text - - # Create directories - (output_dir / "include" / namespace).mkdir(parents=True) - (output_dir / "src" / "tests").mkdir(parents=True) - (output_dir / "src" / "benchmarks").mkdir(parents=True) - (output_dir / "src" / "docs").mkdir(parents=True) - (output_dir / "src" / "docs" / "images").mkdir(parents=True) - (output_dir / "scripts").mkdir(parents=True) - (output_dir / ".github" / "workflows").mkdir(parents=True) - - # Write files - files = { - output_dir / "CMakeLists.txt": sub(CMAKE_LISTS_TXT), - output_dir / "CMakePresets.json": sub(CMAKE_PRESETS_JSON), - output_dir / ".clang-format": sub(CLANG_FORMAT), - output_dir / ".gitignore": sub(GITIGNORE), - output_dir / "README.md": sub(README_MD), - output_dir / "AGENTS.md": sub(AGENTS_MD), - output_dir / "src" / "docs" / "Doxyfile.in": sub(DOXYFILE_IN), - output_dir / "scripts" / "coverage_report.sh": sub(COVERAGE_REPORT_SH), - output_dir / ".github" / "workflows" / "build-test.yml": sub(BUILD_TEST_YML), - output_dir / ".github" / "workflows" / "linter.yml": sub(LINTER_YML), - output_dir / ".github" / "workflows" / "coverage.yml": sub(COVERAGE_YML), - output_dir / ".github" / "workflows" / "doxygen.yml": sub(DOXYGEN_YML), - output_dir / "include" / namespace / header_name: sub(HEADER_HPP), - output_dir / "src" / "tests" / "unittests.cpp": sub(UNITTESTS_CPP), - output_dir / "src" / "benchmarks" / "benchmarks.cpp": sub(BENCHMARKS_CPP), - } - - for path, content in files.items(): - path.write_text(content) - print(f"Created: {path.relative_to(output_dir.parent)}") - - # Make coverage script executable - (output_dir / "scripts" / "coverage_report.sh").chmod(0o755) - - print(f"\\nProject '{project_name}' generated successfully at {output_dir}") - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Scaffold a new C++20 repository following modern C++ conventions." - ) - parser.add_argument("--name", required=True, help="Project name (e.g., my-lib)") - parser.add_argument( - "--namespace", - help="C++ namespace (defaults to project name with hyphens removed)", - ) - parser.add_argument( - "--output-dir", - default=".", - help="Output directory (default: current directory)", - ) - args = parser.parse_args() - generate(args) - - -if __name__ == "__main__": - main() diff --git a/agentic/local/cpp/skills/cmake/EXAMPLES.md b/agentic/local/cpp/skills/cmake/EXAMPLES.md new file mode 100644 index 0000000..95cae08 --- /dev/null +++ b/agentic/local/cpp/skills/cmake/EXAMPLES.md @@ -0,0 +1,73 @@ +# Pixie CMake Examples + +Use these notes with `agentic/cpp/skills/cmake/SKILL.md` for Pixie's local +presets and coverage workflow. + +## Coverage + +Prefer the repository script when checking coverage: + +```bash +./scripts/coverage_report.sh +``` + +The script uses the `coverage` preset, where Pixie's coverage option is +`PIXIE_COVERAGE=ON`, not the generic `ENABLE_COVERAGE=ON` spelling used in +some other CMake projects. It also clears old `.gcda` and `.gcov` files before +running tests, which avoids gcov checksum warnings after recompilation. + +The generated report is: + +```text +build/coverage/coverage.txt +``` + +Useful focused targets while iterating on coverage: + +```bash +cmake --build build/coverage --target rmq_tests -j +./build/coverage/rmq_tests + +cmake --build build/coverage --target test_rmm -j +./build/coverage/test_rmm +``` + +Run the full script before reporting final coverage numbers, because individual +test-binary runs can leave stale profile counters when source checksums change. + +## Codecov Interpretation + +Pixie uploads `gcov -pb` output to Codecov. The `-b` flag includes branch +probabilities, so Codecov reports "partial" coverage for executed lines whose +branch paths were not all covered. A local line-only summary can therefore look +better than the Codecov project or patch percentage. + +When a PR coverage check looks unexpectedly low: + +1. Check the Codecov file table for missing and partial counts. +2. Check `scripts/coverage_report.sh` and `.github/workflows/coverage.yml` to + confirm the uploaded file and flags. +3. For header-only templates, remember that the same header may appear in more + than one translation-unit gcov report. Inspect the generated `.gcov` file or + Codecov aggregate before drawing conclusions from the first `coverage.txt` + block. + +## RMQ/RmM Coverage Strategy + +Random differential tests are necessary but not sufficient for Pixie's RMQ and +RmM structures. The best coverage improvements usually come from deterministic +input shapes that force a specific traversal branch: + +- HybridBTree leaf selector paths: prefix answer, suffix answer, embedded leaf + minimum answer, and unsupported interior range fallback. +- HybridBTree and CartesianHybridBTree internal query paths where the local + selector chooses a border child whose subtree minimum is outside the queried + range, forcing border recursion plus middle-child comparison. +- Top sparse-overlay paths where the padded sparse candidate is inside the + query and where it misses and delegates borders to the tree. +- RmMBTree partial last-block scans, forward/backward search no-hit cases, + backward boundary-only returns, and `minselect` ranks across node boundaries. + +Do not chase every uncovered branch. Low-value coverage includes allocator +failure branches, impossible index-size `length_error` paths, and private +defensive `npos` checks that cannot be reached through valid public API calls. diff --git a/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md b/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md new file mode 100644 index 0000000..6f4854c --- /dev/null +++ b/agentic/local/cpp/skills/optimization-experiment/EXAMPLES.md @@ -0,0 +1,198 @@ +# Pixie Optimization Experiment Examples + +Use these notes with `agentic/cpp/skills/optimization-experiment/SKILL.md` for +Pixie-specific optimization work. + +## Example: `excess_min_128` + +The `excess_min_128` experiment is a good template for future hot-path work: + +1. **Lock semantics first**: preserve inclusive prefix range `[left, right]`, + prefix offsets `[0, 128]`, first-min tie breaking, and invalid-range sentinel + behavior. +2. **Add focused benchmarks before trusting results**: include full-range, + RMQ-style `[0, 127]`, aligned short ranges, non-aligned short ranges, + cross-word ranges, point ranges, and a reproducible random-range benchmark. +3. **Keep candidates same-API**: experimental variants should accept the same + parameters and return the same result type as production, so benchmarks can + swap implementations without adapter logic. +4. **Use experimental space for ideas**: keep exploratory variants in + `include/pixie/experimental/` unless they are clearly production-ready. +5. **Promote narrowly**: when a variant wins only under a specific condition, + promote only that condition. For `excess_min_128`, byte-LUT was useful only + for byte-aligned short ranges, so the production fallback was narrowed instead + of applied to every short range. +6. **Mark unlikely cold dispatches**: if a promoted optimization is for a narrow + special case, mark the branch unlikely when that matches expected workload. +7. **Persist benchmark evidence in the repository**: add a top-of-file + benchmark note in `/** ... */` or `/* ... */` form to the experimental + header, production header, benchmark file, or experiment log. Include metric + units, fixed decimal precision, and visible separators between logical + result blocks. Do not include machine-local JSON paths such as `/tmp/...` in + source comments. + +## Benchmark Pattern + +Prefer a diagnostic run that can explain timing changes, not just report them: + +```bash +taskset -c 0 build/benchmarks-diagnostic_local/excess_positions_benchmarks \ + --benchmark_filter='BM_ExcessMin128(_|/|$)' \ + --benchmark_repetitions=3 \ + --benchmark_perf_counters=CYCLES,INSTRUCTIONS,CACHE-MISSES \ + --benchmark_counters_tabular=true \ + --benchmark_out=/tmp/pixie_excess_min_counters.json \ + --benchmark_out_format=json +``` + +For final comparison tables, include at least: + +- CPU time +- cycles +- instructions +- cache misses when collected +- a random or mixed workload row +- the rows that justified any narrowed production dispatch condition + +## Example: RMQ Optimization Report + +The current RMQ work is the preferred template for larger optimization +experiments. It records the command shape, units, precision, and the main +tradeoff dimensions: query latency, build time, absolute memory, and normalized +memory. + +Keep this kind of report at the top of the relevant header or experiment log: + +```text +RMQ benchmark snapshot, 2026-06-11. + +Query command: + taskset -c 0 ./build/release/bench_rmq + --benchmark_filter='^(rmq_sparse_table|rmq_segment_tree|rmq_hybrid_btree|rmq_cartesian_rmm|rmq_cartesian_hybrid_btree|rmq_sdsl_sct)/' + --benchmark_min_time=0.25s + +Build/memory command: + taskset -c 0 ./build/release/bench_rmq + --benchmark_filter='^(rmq_build_sparse_table|rmq_build_segment_tree|rmq_build_hybrid_btree|rmq_build_cartesian_rmm|rmq_build_cartesian_hybrid_btree|rmq_build_sdsl_sct)/' + --benchmark_min_time=0.20s + +Query CPU time. "max width" is the benchmark's maximum sampled query width. +Sparse-table rows are intentionally not registered above 2^22. "cartesian hybrid" +is the `rmq_cartesian_hybrid_btree` row. + +| N | max width | sparse table (ns) | segment tree (ns) | hybrid btree (ns) | cartesian hybrid (ns) | sdsl sct (ns) | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^10 | 64 | 10.1 | 35.9 | 34.6 | 66.4 | 111.2 | +| 2^10 | 2^10 | 13.1 | 58.8 | 41.0 | 118.6 | 231.3 | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^14 | 64 | 14.7 | 43.6 | 33.5 | 67.3 | 125.4 | +| 2^14 | 4096 | 15.9 | 76.7 | 38.1 | 66.3 | 430.8 | +| 2^14 | 2^14 | 15.8 | 93.5 | 33.7 | 49.2 | 532.1 | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^18 | 64 | 63.0 | 103.7 | 50.9 | 74.7 | 141.3 | +| 2^18 | 4096 | 52.2 | 183.0 | 52.5 | 72.6 | 586.7 | +| 2^18 | 2^18 | 40.9 | 299.1 | 46.0 | 27.6 | 791.0 | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^22 | 64 | 97.2 | 169.1 | 137.4 | 256.9 | 263.2 | +| 2^22 | 4096 | 93.4 | 298.7 | 101.9 | 251.9 | 726.7 | +| 2^22 | 2^18 | 68.0 | 437.3 | 82.2 | 86.4 | 1053.0 | +| 2^22 | 2^22 | 60.7 | 485.3 | 45.5 | 22.3 | 1129.7 | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^24 | 64 | - | 208.0 | 166.6 | 480.4 | 549.6 | +| 2^24 | 4096 | - | 357.2 | 222.2 | 535.4 | 864.8 | +| 2^24 | 2^18 | - | 472.3 | 143.0 | 83.5 | 2824.2 | +| 2^24 | 2^22 | - | 510.7 | 47.1 | 25.3 | 1169.8 | +| 2^24 | 2^24 | - | 546.5 | 43.4 | 20.1 | 1239.6 | +| -----: | ----------: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^26 | 64 | - | 273.5 | 342.3 | 567.5 | 710.4 | +| 2^26 | 4096 | - | 938.4 | 308.9 | 688.5 | 1701.6 | +| 2^26 | 2^18 | - | 802.5 | 175.6 | 195.1 | 2609.4 | +| 2^26 | 2^22 | - | 1690.7 | 64.0 | 45.2 | 2387.0 | +| 2^26 | 2^26 | - | 1003.7 | 51.0 | 24.1 | 2152.0 | + +Build CPU time. Sparse-table build rows are intentionally not registered +above 2^22. + +| N | sparse table (ms) | segment tree (ms) | hybrid btree (ms) | cartesian hybrid (ms) | sdsl sct (ms) | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^10 | 0.005 | 0.001 | 0.001 | 0.009 | 0.007 | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^14 | 1.096 | 0.015 | 0.018 | 0.058 | 0.162 | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^18 | 28.405 | 0.292 | 0.301 | 1.985 | 2.484 | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^22 | 860.619 | 92.812 | 6.077 | 31.552 | 40.262 | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^24 | - | 426.733 | 21.802 | 124.868 | 158.091 | +| -----: | ------------------: | ------------------: | ----------------------: | ------------------: | --------------: | +| 2^26 | - | 1432.849 | 86.641 | 518.439 | 647.360 | + +Owned auxiliary memory. Benchmarks use 64-bit signed integer values and +64-bit indexes. The external input values are not owned by RMQ indexes and +are excluded. + +| N | sparse table (MiB) | segment tree (MiB) | hybrid btree (MiB) | cartesian hybrid (MiB) | sdsl sct (MiB) | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^10 | 0.071 | 0.016 | 0.009 | 0.016 | 0.001 | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^14 | 1.626 | 0.250 | 0.011 | 0.022 | 0.005 | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^18 | 34.001 | 4.000 | 0.065 | 0.186 | 0.079 | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^22 | 672.001 | 64.000 | 0.801 | 2.632 | 1.262 | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^24 | - | 256.000 | 3.155 | 7.525 | 5.051 | +| -----: | -------------------: | -------------------: | -----------------------: | -------------------: | ---------------: | +| 2^26 | - | 1024.000 | 8.340 | 30.530 | 20.224 | + +Owned auxiliary memory normalized by indexed value count (bits/value). + +| N | sparse table | segment tree | hybrid btree | cartesian hybrid | sdsl sct | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^10 | 579.188 | 128.438 | 71.438 | 127.500 | 7.312 | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^14 | 832.262 | 128.027 | 5.434 | 11.102 | 2.770 | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^18 | 1088.020 | 128.002 | 2.089 | 5.957 | 2.534 | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^22 | 1344.002 | 128.000 | 1.603 | 5.264 | 2.524 | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^24 | - | 128.000 | 1.577 | 3.763 | 2.526 | +| -----: | -------------: | -------------: | -----------------: | -------------: | ---------: | +| 2^26 | - | 128.000 | 1.042 | 3.816 | 2.528 | +``` + +## Documentation Pattern + +For experimental algorithms, use Doxygen block comments: + +```cpp +/** + * @brief Short algorithm name and purpose. + * + * @details Workflow: + * + * input -> transform -> candidate result -> final reduction + * + * Explain the non-obvious tradeoff, such as avoiding lane crossing, reducing + * loop iterations, or paying scalar boundary work. + */ +``` + +Keep long benchmark tables as a top-level `/** ... */` note when they describe +the whole experimental header rather than one symbol. Do not leave measurements +only in chat or `/tmp`; persist a top-of-file snapshot before finishing. + +## Non-Obvious Lessons From This Session + +- A faster micro-kernel can regress neighboring ranges if dispatch is too broad; + benchmark aligned and non-aligned cases separately. +- Random-range benchmarks are useful as a sanity check against overfitting fixed + rows. +- Hardware counters helped distinguish lower-level cost: cycles and instructions + were more actionable than cache misses for the `excess_min_128` candidates. +- Production and experimental code must stay visually distinct. A benchmark win + in `pixie::experimental` is only a candidate, not a caller-visible change. +- When result names encode width unnecessarily, prefer the stable semantic name + (`ExcessResult`) over width-specific detail (`ExcessResult128`) unless the API + truly needs multiple widths. diff --git a/include/pixie/bits.h b/include/pixie/bits.h index 7d26597..4fb4f57 100644 --- a/include/pixie/bits.h +++ b/include/pixie/bits.h @@ -2,6 +2,8 @@ #include +#include +#include #include #include #include @@ -45,6 +47,87 @@ static inline const __m256i mask_first_half = _mm256_setr_epi8( // clang-format on #endif +static inline constexpr int8_t excess_nibble_min_offset[16] = { + 4, 4, 4, 4, 2, 2, 1, 1, 3, 3, 1, 1, 2, 2, 1, 1}; + +#if defined(__SSSE3__) && defined(__SSE4_1__) +#define PIXIE_SSE41_SUPPORT +// clang-format off +static inline const __m128i excess_lut_delta_sse = _mm_setr_epi8( + -4, -2, -2, 0, + -2, 0, 0, 2, + -2, 0, 0, 2, + 0, 2, 2, 4); +static inline const __m128i excess_lut_pos0_sse = _mm_setr_epi8( + -1, 1, -1, 1, + -1, 1, -1, 1, + -1, 1, -1, 1, + -1, 1, -1, 1); +static inline const __m128i excess_lut_pos1_sse = _mm_setr_epi8( + -2, 0, 0, 2, + -2, 0, 0, 2, + -2, 0, 0, 2, + -2, 0, 0, 2); +static inline const __m128i excess_lut_pos2_sse = _mm_setr_epi8( + -3, -1, -1, 1, + -1, 1, 1, 3, + -3, -1, -1, 1, + -1, 1, 1, 3); +static inline const __m128i excess_lut_min_sse = _mm_setr_epi8( + -4, -2, -2, 0, + -2, 0, -1, 1, + -3, -1, -1, 1, + -2, 0, -1, 1); +static inline const __m128i excess_lut_nibble_index_sse = _mm_setr_epi8( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15); +static inline const __m128i excess_lut_low_nibble_index_sse = _mm_setr_epi8( + 0, 2, 4, 6, + 8, 10, 12, 14, + 16, 18, 20, 22, + 24, 26, 28, 30); +static inline const __m128i excess_lut_high_nibble_index_sse = _mm_setr_epi8( + 1, 3, 5, 7, + 9, 11, 13, 15, + 17, 19, 21, 23, + 25, 27, 29, 31); +static inline const __m128i excess_lut_nibble_mask_sse = _mm_set1_epi8(0x0F); +// clang-format on +#endif + +#ifdef PIXIE_SSE41_SUPPORT +static inline __m128i excess_nibbles_64_sse(const uint64_t* s) noexcept { + const __m128i word_vec = _mm_loadl_epi64(reinterpret_cast(s)); + const __m128i lo_nibbles = + _mm_and_si128(word_vec, excess_lut_nibble_mask_sse); + const __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(word_vec, 4), excess_lut_nibble_mask_sse); + return _mm_unpacklo_epi8(lo_nibbles, hi_nibbles); +} + +static inline __m128i excess_prefix_sum_16x_i8(__m128i v) noexcept { + __m128i x = v; + __m128i t = _mm_slli_si128(x, 1); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 2); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 4); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 8); + return _mm_add_epi8(x, t); +} + +static inline int excess_horizontal_min_i8(__m128i v) noexcept { + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 8)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 4)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 2)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 1)); + return static_cast(static_cast(_mm_extract_epi8(v, 0))); +} +#endif + /** * @brief Test 16 int16 RmM btree child ranges for a node-local target. * @details Each lane represents one child summary. The function checks whether @@ -797,16 +880,210 @@ static inline const __m256i excess_lut_pos2 = _mm256_setr_epi8( -1, 1, 1, 3, -3, -1, -1, 1, -1, 1, 1, 3); +static inline const __m256i excess_lut_min = _mm256_setr_epi8( + -4, -2, -2, 0, + -2, 0, -1, 1, + -3, -1, -1, 1, + -2, 0, -1, 1, + -4, -2, -2, 0, + -2, 0, -1, 1, + -3, -1, -1, 1, + -2, 0, -1, 1); +static inline constexpr int8_t excess_lut_min_offset[16] = { + 4, 4, 4, 4, 2, 2, 1, 1, 3, 3, 1, 1, 2, 2, 1, 1}; static inline const __m256i excess_lut_pack_multiplier = _mm256_set1_epi16(0x1001); static inline const __m256i excess_lut_bit0 = _mm256_set1_epi8(1); static inline const __m256i excess_lut_bit1 = _mm256_set1_epi8(2); static inline const __m256i excess_lut_bit2 = _mm256_set1_epi8(4); static inline const __m256i excess_lut_bit3 = _mm256_set1_epi8(8); +static inline const __m256i excess_lut_nibble_index = _mm256_setr_epi8( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, + 28, 29, 30, 31); static inline const __m128i excess_lut_nibble_mask = _mm_set1_epi8(0x0F); // clang-format on + +static inline __m256i excess_nibbles_128_avx2(const uint64_t* s) noexcept { + __m128i word_vec = _mm_loadu_si128(reinterpret_cast(s)); + __m128i lo_nibbles = _mm_and_si128(word_vec, excess_lut_nibble_mask); + __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(word_vec, 4), excess_lut_nibble_mask); + + __m128i unpack_lo = _mm_unpacklo_epi8(lo_nibbles, hi_nibbles); + __m128i unpack_hi = _mm_unpackhi_epi8(lo_nibbles, hi_nibbles); + + return _mm256_inserti128_si256(_mm256_castsi128_si256(unpack_lo), unpack_hi, + 1); +} + +static inline __m256i excess_bit_masks_16x_i16() noexcept { + return _mm256_setr_epi16(0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, + 0x0040, 0x0080, 0x0100, 0x0200, 0x0400, 0x0800, + 0x1000, 0x2000, 0x4000, + static_cast(0x8000)); +} + +static inline __m256i excess_prefix_sum_16x_i16(__m256i v) noexcept { + __m256i x = v; + __m256i t = _mm256_slli_si256(x, 2); + x = _mm256_add_epi16(x, t); + t = _mm256_slli_si256(x, 4); + x = _mm256_add_epi16(x, t); + t = _mm256_slli_si256(x, 8); + x = _mm256_add_epi16(x, t); + + __m128i lo = _mm256_extracti128_si256(x, 0); + __m128i hi = _mm256_extracti128_si256(x, 1); + const int16_t carry = static_cast(_mm_extract_epi16(lo, 7)); + hi = _mm_add_epi16(hi, _mm_set1_epi16(carry)); + + __m256i out = _mm256_castsi128_si256(lo); + return _mm256_inserti128_si256(out, hi, 1); +} #endif +/** + * @brief Minimum prefix excess in a 128-bit bitstring range. + * @details Prefix positions are offsets in `[0, 128]`; position 0 is the + * empty prefix and position `k` is the excess after consuming the first `k` + * bits. The query range `[left, right]` is inclusive. Ties return the first + * offset attaining the minimum. Invalid ranges return offset 128 as a + * sentinel. + */ +struct ExcessResult { + int min_excess = 0; + size_t offset = 128; +}; + +/** + * @brief Pair of boundary minimum results for adjacent BP query blocks. + * + * @details `suffix` is local to the left/suffix block and `prefix` is local to + * the right/prefix block. + */ +struct ExcessBoundaryPairResult { + ExcessResult suffix; + ExcessResult prefix; +}; + +constexpr int8_t excess_byte_delta_value(uint8_t x) { + return static_cast(2 * std::popcount(x) - 8); +} + +constexpr int8_t excess_byte_min_prefix_value(uint8_t x) { + int cur = 0; + int best = 0; + for (int bit = 0; bit < 8; ++bit) { + cur += ((x >> bit) & 1u) != 0 ? 1 : -1; + if (bit == 0 || cur < best) { + best = cur; + } + } + return static_cast(best); +} + +constexpr int8_t excess_byte_min_prefix_offset_value(uint8_t x) { + int cur = 0; + int best = 0; + int best_offset = 1; + for (int bit = 0; bit < 8; ++bit) { + cur += ((x >> bit) & 1u) != 0 ? 1 : -1; + if (bit == 0 || cur < best) { + best = cur; + best_offset = bit + 1; + } + } + return static_cast(best_offset); +} + +constexpr int8_t excess_nibble_min_prefix_offset_value(uint8_t x, int bits) { + int cur = 0; + int best = 0; + int best_offset = 1; + for (int bit = 0; bit < bits; ++bit) { + cur += ((x >> bit) & 1u) != 0 ? 1 : -1; + if (bit == 0 || cur < best) { + best = cur; + best_offset = bit + 1; + } + } + return static_cast(best_offset); +} + +template +constexpr std::array excess_make_byte_lut(Fn fn) { + std::array out{}; + for (size_t i = 0; i < out.size(); ++i) { + out[i] = fn(static_cast(i)); + } + return out; +} + +static inline constexpr std::array excess_byte_delta_lut = + excess_make_byte_lut([](uint8_t x) { return excess_byte_delta_value(x); }); +static inline constexpr std::array excess_byte_min_lut = + excess_make_byte_lut( + [](uint8_t x) { return excess_byte_min_prefix_value(x); }); +static inline constexpr std::array excess_byte_min_offset_lut = + excess_make_byte_lut( + [](uint8_t x) { return excess_byte_min_prefix_offset_value(x); }); +static inline constexpr std::array, 4> + excess_partial_nibble_min_offset_lut = [] { + std::array, 4> out{}; + for (size_t width = 1; width < out.size(); ++width) { + for (size_t nibble = 0; nibble < out[width].size(); ++nibble) { + out[width][nibble] = excess_nibble_min_prefix_offset_value( + static_cast(nibble), static_cast(width)); + } + } + return out; + }(); + +/** + * @brief Compute record-low mask for a single byte relative to a threshold. + * + * @details For each bit position in the byte (0..7), computes the local + * excess starting from 0. Sets the corresponding bit in the result mask + * iff the local excess is strictly less than @p threshold. + */ +constexpr uint8_t excess_byte_record_lows_mask(uint8_t byte, int threshold) { + int cur = 0; + uint8_t mask = 0; + for (int bit = 0; bit < 8; ++bit) { + cur += ((byte >> bit) & 1u) ? 1 : -1; + if (cur < threshold) { + mask |= static_cast(1u << bit); + } + } + return mask; +} + +/** + * @brief LUT for record-low masks within a single byte. + * + * @details For each byte value (256) and each possible gap g in [0, 7], + * stores a bitmask of positions whose local excess is strictly less than + * -g. Gap g = start_excess - best_excess; when g >= 8 no new record low + * is possible because the byte's minimum local excess is -8. + */ +static inline constexpr std::array, 256> + excess_byte_record_lows_lut = [] { + std::array, 256> out{}; + for (size_t byte = 0; byte < 256; ++byte) { + for (int g = 0; g < 8; ++g) { + out[byte][g] = + excess_byte_record_lows_mask(static_cast(byte), -g); + } + } + return out; + }(); + /** * @brief Find every prefix whose excess equals target_x in a 128-bit bitstring. * @@ -840,22 +1117,13 @@ static inline int excess_positions_128(const uint64_t* s, const __m256i vbit1 = excess_lut_bit1; const __m256i vbit2 = excess_lut_bit2; const __m256i vbit3 = excess_lut_bit3; - const __m128i vnibble_mask = excess_lut_nibble_mask; const int d = 2 * target_x - block_delta; if (d < -128 || d > 128) { return block_delta; } - __m128i word_vec = _mm_loadu_si128((const __m128i*)s); - __m128i lo_nibbles = _mm_and_si128(word_vec, vnibble_mask); - __m128i hi_nibbles = _mm_and_si128(_mm_srli_epi16(word_vec, 4), vnibble_mask); - - __m128i unpack_lo = _mm_unpacklo_epi8(lo_nibbles, hi_nibbles); - __m128i unpack_hi = _mm_unpackhi_epi8(lo_nibbles, hi_nibbles); - - __m256i nibbles = - _mm256_inserti128_si256(_mm256_castsi128_si256(unpack_lo), unpack_hi, 1); + __m256i nibbles = excess_nibbles_128_avx2(s); __m256i ps = _mm256_shuffle_epi8(vdelta, nibbles); ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 1)); @@ -935,6 +1203,577 @@ static inline int prefix_excess_128(const uint64_t* s, return 2 * ones - static_cast(end_offset); } +/** + * @brief Prefix excess in a 64-bit bitstring. + * + * @details `excess(i) = 2 * popcount(bits[0..i)) - i` for `i` in `[0, 64]`. + * + * @param s One little-endian 64-bit word (bit 0 of `s[0]` is first). + * @param end_offset Exclusive prefix boundary, clamped to `[0, 64]`. + * @return Prefix excess on `[0, end_offset)`. + */ +static inline int prefix_excess_64(const uint64_t* s, + size_t end_offset) noexcept { + end_offset = end_offset > 64 ? 64 : end_offset; + if (end_offset == 0) { + return 0; + } + const int ones = static_cast( + std::popcount(s[0] & first_bits_mask(static_cast(end_offset)))); + return 2 * ones - static_cast(end_offset); +} + +static inline ExcessResult excess_min_128_byte_lut_short( + const uint64_t* s, + size_t left, + size_t right) noexcept { + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + size_t bit = left; + for (; bit < right && (bit & 7u) != 0; ++bit) { + current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } + + for (; bit + 8 <= right; bit += 8) { + const uint8_t byte = + static_cast((s[bit >> 6] >> (bit & 63)) & 0xFFu); + const int candidate = current + excess_byte_min_lut[byte]; + if (candidate < best) { + best = candidate; + best_offset = bit + static_cast(excess_byte_min_offset_lut[byte]); + } + current += excess_byte_delta_lut[byte]; + } + + for (; bit < right; ++bit) { + current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } + + return {best, best_offset}; +} + +/** + * @brief Return the minimum prefix excess and first attaining offset. + * + * @details Prefix positions are offsets in `[0, 64]`; position 0 is the empty + * prefix and position `k` is the excess after consuming the first `k` bits. + * The query range `[left, right]` is inclusive. Ties return the first offset + * attaining the minimum. Invalid ranges return the default `ExcessResult` + * sentinel. + * + * @param s One little-endian 64-bit word (bit 0 of `s[0]` is first). + * @param left First prefix position to consider, inclusive. + * @param right Last prefix position to consider, inclusive. + * @return Minimum excess and first local offset attaining it. + */ +static inline ExcessResult excess_min_64(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 64); + right = std::min(right, 64); + + int best = prefix_excess_64(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + +#ifdef PIXIE_SSE41_SUPPORT + int current = best; + size_t bit = left; + for (; bit < right && (bit & 3u) != 0; ++bit) { + current += ((s[0] >> bit) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } + + const size_t first_full_nibble = bit >> 2; + const size_t last_full_nibble = right >> 2; + const size_t right_partial_width = bit < right ? (right & 3u) : 0; + const size_t end_nibble = + last_full_nibble + (right_partial_width == 0 ? 0 : 1); + if (first_full_nibble < end_nibble) { + const __m128i nibbles = excess_nibbles_64_sse(s); + + __m128i ps = _mm_shuffle_epi8(excess_lut_delta_sse, nibbles); + ps = _mm_add_epi8(ps, _mm_slli_si128(ps, 1)); + ps = _mm_add_epi8(ps, _mm_slli_si128(ps, 2)); + ps = _mm_add_epi8(ps, _mm_slli_si128(ps, 4)); + ps = _mm_add_epi8(ps, _mm_slli_si128(ps, 8)); + + const __m128i excl_ps = _mm_slli_si128(ps, 1); + __m128i local_min = _mm_shuffle_epi8(excess_lut_min_sse, nibbles); + if (right_partial_width != 0) { + __m128i partial_min = _mm_shuffle_epi8(excess_lut_pos0_sse, nibbles); + if (right_partial_width >= 2) { + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos1_sse, nibbles)); + } + if (right_partial_width >= 3) { + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos2_sse, nibbles)); + } + local_min = _mm_blendv_epi8( + local_min, partial_min, + _mm_cmpeq_epi8(excess_lut_nibble_index_sse, + _mm_set1_epi8(static_cast(last_full_nibble)))); + } + const __m128i partial_candidates = _mm_add_epi8(excl_ps, local_min); + + const __m128i idx = excess_lut_nibble_index_sse; + const int first_minus_one_value = static_cast(first_full_nibble) - 1; + const __m128i first_minus_one = + _mm_set1_epi8(static_cast(first_minus_one_value)); + const __m128i last = _mm_set1_epi8(static_cast(end_nibble)); + const __m128i active = _mm_and_si128(_mm_cmpgt_epi8(idx, first_minus_one), + _mm_cmpgt_epi8(last, idx)); + const __m128i masked_candidates = + _mm_blendv_epi8(_mm_set1_epi8(127), partial_candidates, active); + + __m128i min128 = masked_candidates; + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 8)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 4)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 2)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 1)); + + const int candidate_min = + static_cast(static_cast(_mm_extract_epi8(min128, 0))); + if (candidate_min < best) { + const __m128i equal_min = _mm_cmpeq_epi8( + masked_candidates, _mm_set1_epi8(static_cast(candidate_min))); + const uint32_t equal_mask = + static_cast(_mm_movemask_epi8(equal_min)); + const uint32_t nibble_index = std::countr_zero(equal_mask); + const uint8_t nibble = + static_cast((s[0] >> (nibble_index * 4u)) & 0xFu); + best = candidate_min; + if (right_partial_width != 0 && nibble_index == last_full_nibble) { + int local = 0; + int local_best = 0; + size_t local_offset = 1; + for (size_t i = 0; i < right_partial_width; ++i) { + local += ((nibble >> i) & 1u) != 0 ? 1 : -1; + if (i == 0 || local < local_best) { + local_best = local; + local_offset = i + 1; + } + } + best_offset = static_cast(nibble_index) * 4u + local_offset; + } else { + best_offset = static_cast(nibble_index) * 4u + + static_cast(excess_nibble_min_offset[nibble]); + } + } + + bit = end_nibble * 4; + } + + for (; bit < right; ++bit) { + current += ((s[0] >> bit) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } +#else + int current = best; + for (size_t bit = left; bit < right; ++bit) { + current += ((s[0] >> bit) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } +#endif + + return {best, best_offset}; +} + +/** + * @brief Return the minimum prefix excess and first attaining offset. + * @param s 2 little-endian uint64_t words (bit 0 of s[0] is the first bit). + * @param left First prefix position to consider, inclusive. + * @param right Last prefix position to consider, inclusive. + */ +static inline ExcessResult excess_min_128(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + if (right - left <= 32 && (left & 7u) == 0 && (right & 7u) == 0) + [[unlikely]] { + return excess_min_128_byte_lut_short(s, left, right); + } + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + +#ifdef PIXIE_AVX2_SUPPORT + int current = best; + size_t bit = left; + for (; bit < right && (bit & 3u) != 0; ++bit) { + current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } + } + + const size_t first_nibble = bit >> 2; + const size_t last_full_nibble = right >> 2; + const size_t right_partial_width = bit < right ? (right & 3u) : 0; + const size_t end_nibble = + last_full_nibble + (right_partial_width == 0 ? 0 : 1); + if (first_nibble < end_nibble) { + const __m128i bytes = _mm_loadu_si128(reinterpret_cast(s)); + const __m128i lo_nibbles = _mm_and_si128(bytes, excess_lut_nibble_mask_sse); + const __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(bytes, 4), excess_lut_nibble_mask_sse); + const __m128i lo_delta = _mm_shuffle_epi8(excess_lut_delta_sse, lo_nibbles); + const __m128i hi_delta = _mm_shuffle_epi8(excess_lut_delta_sse, hi_nibbles); + const __m128i byte_delta = _mm_add_epi8(lo_delta, hi_delta); + const __m128i byte_prefix = excess_prefix_sum_16x_i8(byte_delta); + const __m128i byte_prefix_before = _mm_slli_si128(byte_prefix, 1); + + __m128i lo_local_min = _mm_shuffle_epi8(excess_lut_min_sse, lo_nibbles); + __m128i hi_local_min = _mm_shuffle_epi8(excess_lut_min_sse, hi_nibbles); + + const __m128i byte_index = excess_lut_nibble_index_sse; + if (right_partial_width != 0) { + const bool partial_is_high = (last_full_nibble & 1u) != 0; + const size_t partial_byte = last_full_nibble >> 1; + const __m128i partial_source = partial_is_high ? hi_nibbles : lo_nibbles; + __m128i partial_min = + _mm_shuffle_epi8(excess_lut_pos0_sse, partial_source); + if (right_partial_width >= 2) { + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos1_sse, partial_source)); + } + if (right_partial_width >= 3) { + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos2_sse, partial_source)); + } + const __m128i partial_lane = _mm_cmpeq_epi8( + byte_index, _mm_set1_epi8(static_cast(partial_byte))); + if (partial_is_high) { + hi_local_min = _mm_blendv_epi8(hi_local_min, partial_min, partial_lane); + } else { + lo_local_min = _mm_blendv_epi8(lo_local_min, partial_min, partial_lane); + } + } + + const __m128i lo_candidates = + _mm_add_epi8(byte_prefix_before, lo_local_min); + const __m128i hi_candidates = + _mm_add_epi8(_mm_add_epi8(byte_prefix_before, lo_delta), hi_local_min); + + __m128i masked_lo = lo_candidates; + __m128i masked_hi = hi_candidates; + if (first_nibble != 0 || end_nibble != 32) { + const __m128i first_minus_one = _mm_set1_epi8( + static_cast(static_cast(first_nibble) - 1)); + const __m128i last = _mm_set1_epi8(static_cast(end_nibble)); + const __m128i lo_active = _mm_and_si128( + _mm_cmpgt_epi8(excess_lut_low_nibble_index_sse, first_minus_one), + _mm_cmpgt_epi8(last, excess_lut_low_nibble_index_sse)); + const __m128i hi_active = _mm_and_si128( + _mm_cmpgt_epi8(excess_lut_high_nibble_index_sse, first_minus_one), + _mm_cmpgt_epi8(last, excess_lut_high_nibble_index_sse)); + masked_lo = _mm_blendv_epi8(_mm_set1_epi8(127), lo_candidates, lo_active); + masked_hi = _mm_blendv_epi8(_mm_set1_epi8(127), hi_candidates, hi_active); + } + + const int candidate_min = + excess_horizontal_min_i8(_mm_min_epi8(masked_lo, masked_hi)); + if (candidate_min < best) { + const __m128i min_vec = _mm_set1_epi8(static_cast(candidate_min)); + const uint32_t lo_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(masked_lo, min_vec))); + const uint32_t hi_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(masked_hi, min_vec))); + const uint32_t lo_nibble_index = + lo_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(lo_equal_mask)) * 2u; + const uint32_t hi_nibble_index = + hi_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(hi_equal_mask)) * 2u + + 1u; + const uint32_t nibble_index = std::min(lo_nibble_index, hi_nibble_index); + const uint32_t byte_offset = nibble_index >> 1u; + const uint64_t byte_word = s[byte_offset >> 3u]; + const uint8_t byte = static_cast( + (byte_word >> ((byte_offset & 7u) * 8u)) & 0xFFu); + const uint8_t nibble = (nibble_index & 1u) == 0 + ? static_cast(byte & 0xFu) + : static_cast((byte >> 4u) & 0xFu); + const size_t local_offset = + right_partial_width != 0 && nibble_index == last_full_nibble + ? static_cast( + excess_partial_nibble_min_offset_lut[right_partial_width] + [nibble]) + : static_cast(excess_lut_min_offset[nibble]); + best = candidate_min; + best_offset = static_cast(nibble_index) * 4u + local_offset; + } + } +#else + int current = 0; + for (size_t bit = 0; bit < right; ++bit) { + current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (offset >= left && current < best) { + best = current; + best_offset = offset; + } + } +#endif + + return {best, best_offset}; +} + +/** + * @brief Compute disjoint suffix/prefix boundary minima for two 64-bit blocks. + * + * @details Computes `excess_min_64(suffix_s, suffix_left, 63)` and + * `excess_min_64(prefix_s, 0, prefix_right)`. The individual calls use the + * SSE nibble-LUT path when available. + * + * @param suffix_s Left boundary block. + * @param suffix_left First local prefix offset in the suffix range. + * @param prefix_s Right boundary block. + * @param prefix_right Last local prefix offset in the prefix range. + * @return Pair of local minimum results. + */ +static inline ExcessBoundaryPairResult excess_min_64_disjoint_suffix_prefix( + const uint64_t* suffix_s, + size_t suffix_left, + const uint64_t* prefix_s, + size_t prefix_right) noexcept { + suffix_left = std::min(suffix_left, 64); + prefix_right = std::min(prefix_right, 64); + return {excess_min_64(suffix_s, suffix_left, 63), + excess_min_64(prefix_s, 0, prefix_right)}; +} + +/** + * @brief Compute disjoint suffix/prefix boundary minima for two 128-bit blocks. + * + * @details Computes `excess_min_128(suffix_s, suffix_left, 127)` and + * `excess_min_128(prefix_s, 0, prefix_right)`. When `suffix_left > + * prefix_right`, the AVX2 path blends active whole nibbles from both blocks + * into one synthetic lane vector, fills the gap with neutral zero-delta + * nibbles, and shares the prefix-sum/reduction work. Boundary fragments that + * do not cover a whole nibble are handled by scalar or partial-nibble logic. + * Non-disjoint or out-of-shape inputs fall back to the two independent + * production calls. + * + * @param suffix_s Left boundary block. + * @param suffix_left First local prefix offset in the suffix range. + * @param prefix_s Right boundary block. + * @param prefix_right Last local prefix offset in the prefix range. + * @return Pair of local minimum results. + */ +static inline ExcessBoundaryPairResult excess_min_128_disjoint_suffix_prefix( + const uint64_t* suffix_s, + size_t suffix_left, + const uint64_t* prefix_s, + size_t prefix_right) noexcept { + suffix_left = std::min(suffix_left, 128); + prefix_right = std::min(prefix_right, 128); + if (suffix_left <= prefix_right || suffix_left > 127 || prefix_right > 127) { + return {excess_min_128(suffix_s, suffix_left, 127), + excess_min_128(prefix_s, 0, prefix_right)}; + } + +#ifdef PIXIE_AVX2_SUPPORT + ExcessResult prefix{0, 0}; + + int suffix_best = prefix_excess_128(suffix_s, suffix_left); + ExcessResult suffix{suffix_best, suffix_left}; + size_t suffix_bit = suffix_left; + int suffix_current = suffix_best; + for (; suffix_bit < 127 && (suffix_bit & 3u) != 0; ++suffix_bit) { + suffix_current += + ((suffix_s[suffix_bit >> 6] >> (suffix_bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = suffix_bit + 1; + if (suffix_current < suffix.min_excess) { + suffix = {suffix_current, offset}; + } + } + + const size_t prefix_last_nibble = prefix_right >> 2; + const size_t prefix_partial_width = prefix_right & 3u; + const size_t prefix_end_nibble = + prefix_last_nibble + (prefix_partial_width == 0 ? 0 : 1); + const size_t suffix_first_nibble = suffix_bit < 127 ? suffix_bit >> 2 : 32; + const int prefix_artificial_delta = + prefix_excess_128(prefix_s, prefix_end_nibble * 4); + + const __m256i idx = excess_lut_nibble_index; + const __m256i prefix_active = _mm256_cmpgt_epi8( + _mm256_set1_epi8(static_cast(prefix_end_nibble)), idx); + const __m256i suffix_active = _mm256_cmpgt_epi8( + idx, _mm256_set1_epi8(static_cast(suffix_first_nibble) - 1)); + + const __m256i prefix_nibbles = excess_nibbles_128_avx2(prefix_s); + const __m256i suffix_nibbles = excess_nibbles_128_avx2(suffix_s); + __m256i nibbles = + _mm256_blendv_epi8(_mm256_set1_epi8(3), prefix_nibbles, prefix_active); + nibbles = _mm256_blendv_epi8(nibbles, suffix_nibbles, suffix_active); + + __m256i ps = _mm256_shuffle_epi8(excess_lut_delta, nibbles); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 1)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 2)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 4)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 8)); + + __m128i ps_lo = _mm256_castsi256_si128(ps); + __m128i ps_hi = _mm256_extracti128_si256(ps, 1); + __m128i carry = + _mm_set1_epi8(static_cast(_mm_extract_epi8(ps_lo, 15))); + ps_hi = _mm_add_epi8(ps_hi, carry); + ps = _mm256_inserti128_si256(_mm256_castsi128_si256(ps_lo), ps_hi, 1); + + __m256i b = _mm256_permute2x128_si256(ps, ps, 0x08); + const __m256i excl_ps = _mm256_alignr_epi8(ps, b, 15); + + __m256i local_min = _mm256_shuffle_epi8(excess_lut_min, nibbles); + if (prefix_partial_width != 0) { + __m256i partial_min = _mm256_shuffle_epi8(excess_lut_pos0, nibbles); + if (prefix_partial_width >= 2) { + partial_min = _mm256_min_epi8( + partial_min, _mm256_shuffle_epi8(excess_lut_pos1, nibbles)); + } + if (prefix_partial_width >= 3) { + partial_min = _mm256_min_epi8( + partial_min, _mm256_shuffle_epi8(excess_lut_pos2, nibbles)); + } + local_min = _mm256_blendv_epi8( + local_min, partial_min, + _mm256_cmpeq_epi8( + idx, _mm256_set1_epi8(static_cast(prefix_last_nibble)))); + } + __m256i suffix_partial_min = _mm256_min_epi8( + _mm256_shuffle_epi8(excess_lut_pos0, nibbles), + _mm256_min_epi8(_mm256_shuffle_epi8(excess_lut_pos1, nibbles), + _mm256_shuffle_epi8(excess_lut_pos2, nibbles))); + const __m256i suffix_tail_active = _mm256_and_si256( + suffix_active, _mm256_cmpeq_epi8(idx, _mm256_set1_epi8(31))); + local_min = + _mm256_blendv_epi8(local_min, suffix_partial_min, suffix_tail_active); + + const __m256i base_candidates = _mm256_add_epi8(excl_ps, local_min); + const __m256i sentinel = _mm256_set1_epi8(127); + + const __m256i prefix_candidates = + _mm256_blendv_epi8(sentinel, base_candidates, prefix_active); + const __m256i suffix_candidates = _mm256_blendv_epi8( + sentinel, + _mm256_add_epi8(base_candidates, + _mm256_set1_epi8(static_cast( + suffix_current - prefix_artificial_delta))), + suffix_active); + + auto reduce_min = [](__m256i values) { + __m128i min128 = _mm_min_epi8(_mm256_castsi256_si128(values), + _mm256_extracti128_si256(values, 1)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 8)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 4)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 2)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 1)); + return static_cast(static_cast(_mm_extract_epi8(min128, 0))); + }; + + auto local_offset = [](uint8_t nibble, size_t width_value) { + if (width_value == 0 || width_value == 4) { + return static_cast(excess_lut_min_offset[nibble]); + } + int current = 0; + int best = 0; + size_t best_offset = 1; + for (size_t i = 0; i < width_value; ++i) { + current += ((nibble >> i) & 1u) != 0 ? 1 : -1; + if (i == 0 || current < best) { + best = current; + best_offset = i + 1; + } + } + return best_offset; + }; + + const int prefix_min = reduce_min(prefix_candidates); + if (prefix_min < prefix.min_excess) { + const uint32_t mask = static_cast(_mm256_movemask_epi8( + _mm256_cmpeq_epi8(prefix_candidates, + _mm256_set1_epi8(static_cast(prefix_min))))); + const uint32_t prefix_lane = std::countr_zero(mask); + const uint64_t word = prefix_s[prefix_lane >> 4]; + const uint8_t nibble = + static_cast((word >> ((prefix_lane & 15u) * 4u)) & 0xFu); + prefix.min_excess = prefix_min; + const size_t width = + prefix_partial_width != 0 && prefix_lane == prefix_last_nibble + ? prefix_partial_width + : 4; + prefix.offset = + static_cast(prefix_lane) * 4u + local_offset(nibble, width); + } + + const int suffix_min = reduce_min(suffix_candidates); + if (suffix_min < suffix.min_excess) { + const uint32_t mask = static_cast(_mm256_movemask_epi8( + _mm256_cmpeq_epi8(suffix_candidates, + _mm256_set1_epi8(static_cast(suffix_min))))); + const uint32_t suffix_lane = std::countr_zero(mask); + const uint64_t word = suffix_s[suffix_lane >> 4]; + const uint8_t nibble = + static_cast((word >> ((suffix_lane & 15u) * 4u)) & 0xFu); + suffix.min_excess = suffix_min; + const size_t width = suffix_lane == 31 ? 3 : 4; + suffix.offset = + static_cast(suffix_lane) * 4u + local_offset(nibble, width); + } + + return {suffix, prefix}; +#else + return {excess_min_128(suffix_s, suffix_left, 127), + excess_min_128(prefix_s, 0, prefix_right)}; +#endif +} + /** * @brief Find the first prefix reaching target_x in a 128-bit bitstring. * @@ -1053,6 +1892,276 @@ static inline void excess_positions_512(const uint64_t* s, } } +/** + * @brief Find every strict record-low prefix excess position in a 128-bit + * bitstring. + * + * @details A position i (1 <= i <= 128) is a strict record low if its prefix + * excess is strictly smaller than every prefix excess at positions j < i. + * Position 0 (empty prefix, excess 0) is always a record low vacuously but is + * not represented in the output bitmask. Bit b of out[w] is set iff position + * (w*64 + b + 1) is a strict record low. + * + * @param s 2 little-endian uint64_t words (bit 0 of s[0] is the first bit). + * @param out 2 uint64_t words receiving the result bitmask. + */ +static inline void excess_record_lows_128(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + for (size_t i = 0; i < 128; ++i) { + const uint64_t w = s[i >> 6]; + const int bit = static_cast((w >> (i & 63)) & 1ull); + cur += bit ? +1 : -1; + if (cur < best) { + best = cur; + out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } +} + +/** + * @brief Byte-LUT accelerated record-low scan for 128-bit blocks. + * + * @details Uses precomputed per-byte minimum and delta LUTs to skip whole + * bytes that cannot contain a record low. When a byte's local minimum is + * below the running global best, the byte is scanned bit-by-bit; otherwise + * the byte is skipped and only the running excess is updated. + */ +static inline void excess_record_lows_128_byte_lut(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + for (size_t byte_idx = 0; byte_idx < 16; ++byte_idx) { + const size_t bit_base = byte_idx * 8; + const uint8_t byte = + static_cast((s[bit_base >> 6] >> (bit_base & 63)) & 0xFFu); + const int byte_min = excess_byte_min_lut[byte]; + if (cur + byte_min < best) { + for (size_t i = 0; i < 8; ++i) { + const int bit = static_cast((byte >> i) & 1u); + cur += bit ? +1 : -1; + if (cur < best) { + best = cur; + const size_t pos = bit_base + i; + out[pos >> 6] |= (uint64_t{1} << (pos & 63)); + } + } + } else { + cur += excess_byte_delta_lut[byte]; + } + } +} + +/** + * @brief LUT-based record-low scan for 128-bit blocks (branchless per byte). + * + * @details Precomputed per-byte LUT: for each byte value and each possible + * threshold (cur - best, clamped to 0..7), returns a bitmask of positions + * within the byte whose excess is strictly below the threshold. The LUT + * eliminates all bit-by-bit scanning and branching inside the hot loop. + */ +static inline void excess_record_lows_128_lut(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + for (size_t byte_idx = 0; byte_idx < 16; ++byte_idx) { + const size_t bit_base = byte_idx * 8; + const uint8_t byte = + static_cast((s[bit_base >> 6] >> (bit_base & 63)) & 0xFFu); + const int gap = cur - best; + const int idx = gap > 7 ? 7 : (gap < 0 ? 0 : gap); + const uint8_t mask = + excess_byte_record_lows_lut[byte][static_cast(idx)]; + if (mask != 0) { + // Recompute absolute excesses for masked positions to update cur/best. + int local = 0; + int local_best = 0; + uint8_t local_mask = 0; + for (int bit = 0; bit < 8; ++bit) { + local += ((byte >> bit) & 1u) ? 1 : -1; + if (local < local_best) { + local_best = local; + local_mask |= static_cast(1u << bit); + } + } + // Only output positions whose absolute excess is < best. + uint8_t out_mask = 0; + local = 0; + for (int bit = 0; bit < 8; ++bit) { + local += ((byte >> bit) & 1u) ? 1 : -1; + if (cur + local < best) { + out_mask |= static_cast(1u << bit); + best = cur + local; + } + } + if (out_mask != 0) { + const uint64_t word = static_cast(out_mask) + << (bit_base & 63); + out[bit_base >> 6] |= word; + } + } + cur += excess_byte_delta_lut[byte]; + } +} + +#ifdef PIXIE_AVX2_SUPPORT +/** + * @brief AVX2 record-low scan for 128-bit blocks. + * + * @details Expands the 128-bit block into eight 16-bit chunks, computes + * vector prefix sums, then extracts record lows with a running scalar + * minimum. This avoids the unpredictable branch in the scalar loop. + */ +static inline void excess_record_lows_128_avx2(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + + const __m256i masks = excess_bit_masks_16x_i16(); + const __m256i zero = _mm256_setzero_si256(); + const __m256i pos = _mm256_set1_epi16(1); + const __m256i neg = _mm256_set1_epi16(-1); + + for (size_t chunk = 0; chunk < 8; ++chunk) { + const size_t chunk_bit = chunk * 16; + const uint16_t bits = + chunk < 4 + ? static_cast((s[0] >> (chunk * 16)) & 0xFFFFu) + : static_cast((s[1] >> ((chunk - 4) * 16)) & 0xFFFFu); + + const __m256i vb = _mm256_set1_epi16(static_cast(bits)); + const __m256i m = _mm256_and_si256(vb, masks); + const __m256i is_zero = _mm256_cmpeq_epi16(m, zero); + const __m256i steps = _mm256_blendv_epi8(pos, neg, is_zero); + const __m256i pref_rel = excess_prefix_sum_16x_i16(steps); + const __m256i pref_abs = _mm256_add_epi16( + pref_rel, _mm256_set1_epi16(static_cast(cur))); + + alignas(32) int16_t vals[16]; + _mm256_store_si256(reinterpret_cast<__m256i*>(vals), pref_abs); + + for (size_t lane = 0; lane < 16; ++lane) { + const int val = vals[lane]; + if (val < best) { + best = val; + const size_t pos_idx = chunk_bit + lane; + out[pos_idx >> 6] |= (uint64_t{1} << (pos_idx & 63)); + } + } + + cur += 2 * static_cast(std::popcount(bits)) - 16; + } +} + +/** + * @brief AVX2 nibble-LUT record-low scan for 128-bit blocks. + * + * @details Loads 128 bits as 32 nibbles in one AVX2 register. + * Uses precomputed LUTs to get per-nibble excess values, prefix-sums + * them with 32-lane i8 operations, then compares against the running + * scalar minimum. Record-low positions are written into the output + * bitmask bit-by-bit. + */ +static inline void excess_record_lows_128_nibble_lut(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + + const __m256i vdelta = excess_lut_delta; + const __m256i vmin = excess_lut_min; + + __m256i nibbles = excess_nibbles_128_avx2(s); + + // Compute inclusive prefix sums of per-nibble total excess changes. + __m256i ps = _mm256_shuffle_epi8(vdelta, nibbles); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 1)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 2)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 4)); + ps = _mm256_add_epi8(ps, _mm256_slli_si256(ps, 8)); + + __m128i ps_lo = _mm256_castsi256_si128(ps); + __m128i ps_hi = _mm256_extracti128_si256(ps, 1); + __m128i carry = + _mm_set1_epi8(static_cast(_mm_extract_epi8(ps_lo, 15))); + ps_hi = _mm_add_epi8(ps_hi, carry); + ps = _mm256_inserti128_si256(_mm256_castsi128_si256(ps_lo), ps_hi, 1); + + // Exclusive prefix: shift in zero at start. + __m256i b = _mm256_permute2x128_si256(ps, ps, 0x08); + __m256i excl_ps = _mm256_alignr_epi8(ps, b, 15); + + // Local minima relative to each nibble start. + __m256i local_min = _mm256_shuffle_epi8(vmin, nibbles); + + alignas(32) int8_t excl[32]; + _mm256_store_si256(reinterpret_cast<__m256i*>(excl), excl_ps); + + alignas(32) int8_t nibble_min[32]; + _mm256_store_si256(reinterpret_cast<__m256i*>(nibble_min), local_min); + + alignas(32) int8_t nibble_vals[32]; + _mm256_store_si256(reinterpret_cast<__m256i*>(nibble_vals), nibbles); + + for (int n = 0; n < 32; ++n) { + const int nibble_base = cur + excl[n]; + const int nibble_best = nibble_base + nibble_min[n]; + if (nibble_best < best) { + // This nibble contains at least one record low — scan bit-by-bit. + const uint8_t nibble = static_cast(nibble_vals[n]); + int local = 0; + for (int bit = 0; bit < 4; ++bit) { + local += ((nibble >> bit) & 1u) ? 1 : -1; + const int val = nibble_base + local; + if (val < best) { + best = val; + const size_t pos = static_cast(n) * 4 + bit; + out[pos >> 6] |= (uint64_t{1} << (pos & 63)); + } + } + } + } +} +#endif + +/** + * @brief Find every strict record-low prefix excess position in a 512-bit + * bitstring. + * + * @details A position i (1 <= i <= 512) is a strict record low if its prefix + * excess is strictly smaller than every prefix excess at positions j < i. + * Position 0 is not represented in the output bitmask. Bit b of out[w] is set + * iff position (w*64 + b + 1) is a strict record low. + * + * @param s 8 little-endian uint64_t words (bit 0 of s[0] is the first bit). + * @param out 8 uint64_t words receiving the result bitmask. + */ +static inline void excess_record_lows_512(const uint64_t* s, + uint64_t* out) noexcept { + out[0] = out[1] = out[2] = out[3] = 0; + out[4] = out[5] = out[6] = out[7] = 0; + int best = 0; + int global = 0; + for (int k = 0; k < 4; ++k) { + const uint64_t* block = s + 2 * k; + uint64_t* block_out = out + 2 * k; + for (size_t i = 0; i < 128; ++i) { + const uint64_t w = block[i >> 6]; + const int bit = static_cast((w >> (i & 63)) & 1ull); + global += bit ? +1 : -1; + if (global < best) { + best = global; + block_out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } + } +} + /** * @brief Calculates 32 bit ranks of every 8th bit, result is stored as 32 diff --git a/include/pixie/bitvector.h b/include/pixie/bitvector.h index 8a27553..ab6427f 100644 --- a/include/pixie/bitvector.h +++ b/include/pixie/bitvector.h @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include @@ -45,7 +47,8 @@ namespace pixie { * overhead). * - Basic blocks of 512 bits with 16-bit ranks (~3.125% * overhead). - * - Select samples every 16384 bits (~0.39% overhead). + * - Optional select samples every 16384 bits for 1-bits, 0-bits, or both + * (~0.39% overhead per enabled direction). * * * Rank: 2 table lookups plus SIMD popcount in the 512-bit block. @@ -58,9 +61,21 @@ namespace pixie { * - SIMD linear scan to find the basic block. * * This variant does - * not interleave data and index, favoring simpler scans. + * not interleave data and index, favoring simpler scans. Rank metadata and + * enabled select samples are built in one scan over the source words. */ class BitVector { + public: + /** + * @brief Select directions to index during construction. + */ + enum class SelectSupport : uint8_t { + kNone = 0, + kSelect1 = 1, + kSelect0 = 2, + kBoth = 3, + }; + private: constexpr static size_t kWordSize = 64; constexpr static size_t kSuperBlockRankIntSize = 64; @@ -76,14 +91,29 @@ class BitVector { AlignedStorage super_block_rank_; // 64-bit global prefix sums AlignedStorage basic_block_rank_; // 16-bit local prefix sums - AlignedStorage select1_samples_; // 64-bit global positions - AlignedStorage select0_samples_; // 64-bit global positions + AlignedStorage select_samples_; // 64-bit global positions size_t num_bits_{}; size_t padded_size_{}; size_t max_rank_{}; + size_t select1_sample_begin_{}; + size_t select1_sample_count_{}; + size_t select0_sample_begin_{}; + size_t select0_sample_count_{}; + SelectSupport select_support_ = SelectSupport::kNone; + bool select0_samples_reversed_ = false; std::span bits_; + static bool builds_select1(SelectSupport support) { + return (static_cast(support) & + static_cast(SelectSupport::kSelect1)) != 0; + } + + static bool builds_select0(SelectSupport support) { + return (static_cast(support) & + static_cast(SelectSupport::kSelect0)) != 0; + } + size_t logical_word_count() const { return (num_bits_ + kWordSize - 1) / kWordSize; } @@ -157,10 +187,160 @@ class BitVector { return num_bits_; } + static size_t select_sample_count_for_rank(size_t rank_count) { + return 1 + rank_count / kSelectSampleFrequency; + } + + static size_t select_sample_upper_bound(size_t bit_count) { + return select_sample_count_for_rank(bit_count); + } + + struct SelectSampleWriter { + std::span words; + size_t next = 0; + size_t count = 0; + size_t capacity = 0; + bool enabled = false; + bool reversed = false; + + SelectSampleWriter() = default; + + SelectSampleWriter(std::span words, + size_t begin, + size_t capacity, + bool enabled, + bool reversed) + : words(words), + next(begin), + capacity(capacity), + enabled(enabled), + reversed(reversed) {} + + void append(uint64_t sample) { + if (!enabled) { + return; + } + if (count >= capacity) [[unlikely]] { + throw std::invalid_argument( + "BitVector one_count hint is inconsistent with input bits"); + } + words[next] = sample; + ++count; + if (reversed) { + if (next != 0) { + --next; + } + } else { + ++next; + } + } + }; + + struct SelectSampleWriters { + SelectSampleWriter ones; + SelectSampleWriter zeros; + bool shrink_after_build = false; + }; + + SelectSampleWriters initialize_select_sample_writers( + bool need_select1, + bool need_select0, + std::optional one_count) { + select1_sample_begin_ = 0; + select1_sample_count_ = 0; + select0_sample_begin_ = 0; + select0_sample_count_ = 0; + select0_samples_reversed_ = false; + select_samples_.resize(0); + + SelectSampleWriters writers; + if (!need_select1 && !need_select0) { + return writers; + } + + const std::optional zero_count = + one_count ? std::optional(num_bits_ - *one_count) + : std::nullopt; + if (need_select1 && need_select0) { + const size_t one_sample_capacity = + one_count ? select_sample_count_for_rank(*one_count) + : 2 + num_bits_ / kSelectSampleFrequency; + const size_t zero_sample_capacity = + zero_count ? select_sample_count_for_rank(*zero_count) + : 2 + num_bits_ / kSelectSampleFrequency; + const size_t total_samples = + one_count ? one_sample_capacity + zero_sample_capacity + : 2 + num_bits_ / kSelectSampleFrequency; + select_samples_.resize(total_samples * kWordSize); + auto samples = select_samples_.As64BitInts(); + select1_sample_begin_ = 0; + select0_samples_reversed_ = true; + writers.ones = + SelectSampleWriter(samples, 0, one_sample_capacity, true, false); + writers.zeros = SelectSampleWriter(samples, total_samples - 1, + zero_sample_capacity, true, true); + writers.ones.append(0); + writers.zeros.append(0); + return writers; + } + + const size_t sample_capacity = + need_select1 ? (one_count ? select_sample_count_for_rank(*one_count) + : select_sample_upper_bound(num_bits_)) + : (zero_count ? select_sample_count_for_rank(*zero_count) + : select_sample_upper_bound(num_bits_)); + select_samples_.resize(sample_capacity * kWordSize); + auto samples = select_samples_.As64BitInts(); + writers.shrink_after_build = !one_count; + if (need_select1) { + select1_sample_begin_ = 0; + writers.ones = + SelectSampleWriter(samples, 0, sample_capacity, true, false); + writers.ones.append(0); + } else { + select0_sample_begin_ = 0; + writers.zeros = + SelectSampleWriter(samples, 0, sample_capacity, true, false); + writers.zeros.append(0); + } + return writers; + } + + void finalize_select_sample_writers(SelectSampleWriters writers) { + select1_sample_count_ = writers.ones.count; + select0_sample_count_ = writers.zeros.count; + if (writers.zeros.reversed) { + select0_sample_begin_ = writers.zeros.next + 1; + } + if (writers.shrink_after_build) { + const size_t sample_count = select1_sample_count_ != 0 + ? select1_sample_count_ + : select0_sample_count_; + select_samples_.resize(sample_count * kWordSize); + select_samples_.shrink_to_fit(); + } + } + + uint64_t select1_sample(size_t sample_index) const { + auto samples = select_samples_.AsConst64BitInts(); + return samples[select1_sample_begin_ + sample_index]; + } + + uint64_t select0_sample(size_t sample_index) const { + auto samples = select_samples_.AsConst64BitInts(); + if (select0_samples_reversed_) { + return samples[select0_sample_begin_ + select0_sample_count_ - 1 - + sample_index]; + } + return samples[select0_sample_begin_ + sample_index]; + } + /** - * @brief Precompute rank for fast queries. + * @brief Precompute rank and requested select samples in one word scan. */ - void build_rank() { + void build_rank_select(SelectSupport support, + std::optional one_count) { + select_support_ = support; size_t num_superblocks = 8 + (padded_size_ == 0 ? 0 : (padded_size_ - 1) / kSuperBlockSize); // Add more blocks to ease SIMD processing @@ -174,8 +354,21 @@ class BitVector { auto super_block_rank = super_block_rank_.As64BitInts(); auto basic_block_rank = basic_block_rank_.As16BitInts(); + const bool need_select1 = builds_select1(support); + const bool need_select0 = builds_select0(support); + if (one_count && *one_count > num_bits_) { + throw std::invalid_argument( + "BitVector one_count hint cannot exceed num_bits"); + } + auto select_writers = + initialize_select_sample_writers(need_select1, need_select0, one_count); + uint64_t super_block_sum = 0; uint64_t basic_block_sum = 0; + uint64_t milestone = kSelectSampleFrequency; + uint64_t milestone0 = kSelectSampleFrequency; + uint64_t rank = 0; + uint64_t rank0 = 0; for (size_t i = 0; i / kBasicBlockSize < basic_block_rank.size(); i += kWordSize) { @@ -189,58 +382,32 @@ class BitVector { static_cast(basic_block_sum); } if (i / kWordSize < logical_word_count()) { - basic_block_sum += std::popcount(logical_word(i / kWordSize)); + const size_t word_index = i / kWordSize; + const uint64_t word = logical_word(word_index); + const size_t word_bits = logical_word_bits(word_index); + const uint64_t ones = std::popcount(word); + const uint64_t zeros = word_bits - ones; + if (need_select1 && rank + ones >= milestone) { + const auto pos = select_64(word, milestone - rank - 1); + // TODO: try including global rank into select samples to save + // a cache miss on global rank scan + select_writers.ones.append((64 * word_index + pos) / kSuperBlockSize); + milestone += kSelectSampleFrequency; + } + if (need_select0 && rank0 + zeros >= milestone0) { + const uint64_t zero_word = ~word & first_bits_mask(word_bits); + const auto pos = select_64(zero_word, milestone0 - rank0 - 1); + select_writers.zeros.append((64 * word_index + pos) / + kSuperBlockSize); + milestone0 += kSelectSampleFrequency; + } + basic_block_sum += ones; + rank += ones; + rank0 += zeros; } } max_rank_ = super_block_sum + basic_block_sum; - } - - /** - * @brief Calculate select samples. - */ - void build_select() { - uint64_t milestone = kSelectSampleFrequency; - uint64_t milestone0 = kSelectSampleFrequency; - uint64_t rank = 0; - uint64_t rank0 = 0; - - size_t num_one_samples = - 1 + (max_rank_ + kSelectSampleFrequency - 1) / kSelectSampleFrequency; - size_t num_zero_samples = - 1 + (num_bits_ - max_rank_ + kSelectSampleFrequency - 1) / - kSelectSampleFrequency; - - select1_samples_.resize(num_one_samples * 64); - select0_samples_.resize(num_zero_samples * 64); - auto select1_samples = select1_samples_.As64BitInts(); - auto select0_samples = select0_samples_.As64BitInts(); - - select1_samples[0] = 0; - select0_samples[0] = 0; - - size_t num_zeros = 1, num_ones = 1; - - for (size_t i = 0; i < logical_word_count(); ++i) { - const uint64_t word = logical_word(i); - const auto ones = std::popcount(word); - const auto zeros = logical_word_bits(i) - ones; - if (rank + ones >= milestone) { - auto pos = select_64(word, milestone - rank - 1); - // TODO: try including global rank into select samples to save - // a cache miss on global rank scan - select1_samples[num_ones++] = (64 * i + pos) / kSuperBlockSize; - milestone += kSelectSampleFrequency; - } - if (rank0 + zeros >= milestone0) { - const uint64_t zero_word = - ~word & first_bits_mask(logical_word_bits(i)); - auto pos = select_64(zero_word, milestone0 - rank0 - 1); - select0_samples[num_zeros++] = (64 * i + pos) / kSuperBlockSize; - milestone0 += kSelectSampleFrequency; - } - rank += ones; - rank0 += zeros; - } + finalize_select_sample_writers(select_writers); for (size_t i = 0; i < 8; ++i) { delta_super[i] = i * kSuperBlockSize; @@ -256,10 +423,9 @@ class BitVector { * rank of the 1-bit to locate. */ uint64_t find_superblock(uint64_t rank) const { - auto select1_samples = select1_samples_.AsConst64BitInts(); auto super_block_rank = super_block_rank_.AsConst64BitInts(); - uint64_t left = select1_samples[rank / kSelectSampleFrequency]; + uint64_t left = select1_sample(rank / kSelectSampleFrequency); while (left + 7 < super_block_rank.size()) { auto len = lower_bound_8x64(&super_block_rank[left], rank); @@ -287,10 +453,9 @@ class BitVector { * rank of the 0-bit to locate. */ uint64_t find_superblock_zeros(uint64_t rank0) const { - auto select0_samples = select0_samples_.AsConst64BitInts(); auto super_block_rank = super_block_rank_.AsConst64BitInts(); - uint64_t left = select0_samples[rank0 / kSelectSampleFrequency]; + uint64_t left = select0_sample(rank0 / kSelectSampleFrequency); while (left + 7 < super_block_rank.size()) { auto len = lower_bound_delta_8x64(&super_block_rank[left], rank0, @@ -487,11 +652,11 @@ class BitVector { result.source_bitvector_bytes = (num_bits_ + 7) / 8; result.super_block_rank_bytes = super_block_rank_.AsConstBytes().size(); result.basic_block_rank_bytes = basic_block_rank_.AsConstBytes().size(); - result.select1_samples_bytes = select1_samples_.AsConstBytes().size(); - result.select0_samples_bytes = select0_samples_.AsConstBytes().size(); - result.total_bytes = - result.super_block_rank_bytes + result.basic_block_rank_bytes + - result.select1_samples_bytes + result.select0_samples_bytes; + result.select1_samples_bytes = select1_sample_count_ * sizeof(uint64_t); + result.select0_samples_bytes = select0_sample_count_ * sizeof(uint64_t); + result.total_bytes = result.super_block_rank_bytes + + result.basic_block_rank_bytes + + select_samples_.AsConstBytes().size(); return result; } @@ -523,13 +688,19 @@ class BitVector { * bit_vector Backing data, not owned. * @param num_bits Number of valid * bits in the vector. + * @param select_support Which select sample tables to build. Rank and rank0 + * remain available in all modes. + * @param one_count Optional exact number of 1-bits. When supplied, + * construction can allocate select sample storage exactly. */ - explicit BitVector(std::span bit_vector, size_t num_bits) + explicit BitVector(std::span bit_vector, + size_t num_bits, + SelectSupport select_support = SelectSupport::kBoth, + std::optional one_count = std::nullopt) : num_bits_(std::min(num_bits, bit_vector.size() * kWordSize)), padded_size_(((num_bits_ + kWordSize - 1) / kWordSize) * kWordSize), bits_(bit_vector) { - build_rank(); - build_select(); + build_rank_select(select_support, one_count); } /** @@ -537,6 +708,28 @@ class BitVector { */ size_t size() const { return num_bits_; } + /** + * @brief Whether this index stores samples for select1 queries. + */ + bool supports_select1() const { return builds_select1(select_support_); } + + /** + * @brief Whether this index stores samples for select0 queries. + */ + bool supports_select0() const { return builds_select0(select_support_); } + + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this rank/select object and its owned aligned metadata + * buffers. The external source bit words are not owned and are excluded. + */ + size_t memory_usage_bytes() const { + return sizeof(*this) + super_block_rank_.allocated_bytes() + + basic_block_rank_.allocated_bytes() + + select_samples_.allocated_bytes(); + } + /** * @brief Returns the bit at the given position. * @param pos Bit @@ -593,12 +786,15 @@ class BitVector { * size() if rank is out of range. */ uint64_t select(size_t rank) const { - if (rank > max_rank_) [[unlikely]] { - return num_bits_; - } if (rank == 0) [[unlikely]] { return 0; } + if (!supports_select1()) [[unlikely]] { + return num_bits_; + } + if (rank > max_rank_) [[unlikely]] { + return num_bits_; + } auto super_block_rank = super_block_rank_.AsConst64BitInts(); auto basic_block_rank = basic_block_rank_.AsConst16BitInts(); @@ -617,12 +813,15 @@ class BitVector { * or size() if rank0 is out of range. */ uint64_t select0(size_t rank0) const { - if (rank0 > num_bits_ - max_rank_) [[unlikely]] { - return num_bits_; - } if (rank0 == 0) [[unlikely]] { return 0; } + if (!supports_select0()) [[unlikely]] { + return num_bits_; + } + if (rank0 > num_bits_ - max_rank_) [[unlikely]] { + return num_bits_; + } auto super_block_rank = super_block_rank_.AsConst64BitInts(); auto basic_block_rank = basic_block_rank_.AsConst16BitInts(); diff --git a/include/pixie/cache_line.h b/include/pixie/cache_line.h index 94b3ce2..f027df4 100644 --- a/include/pixie/cache_line.h +++ b/include/pixie/cache_line.h @@ -6,83 +6,137 @@ #include #include +namespace pixie { + +inline constexpr std::size_t kAlignedStorageLineBytes = 64; +inline constexpr std::size_t kAlignedStorageLineBits = + kAlignedStorageLineBytes * 8; +inline constexpr std::size_t kAlignedStorageLineWords64 = + kAlignedStorageLineBytes / sizeof(std::uint64_t); +inline constexpr std::size_t kAlignedStorageLineWords16 = + kAlignedStorageLineBytes / sizeof(std::uint16_t); + /** - * @brief A simple struct to represent a aligned storage for a cache line. + * @brief A 64-byte aligned storage block. */ -struct alignas(64) CacheLine { - std::array data; +struct alignas(kAlignedStorageLineBytes) CacheLine { + std::array data{}; }; +static_assert(alignof(CacheLine) == kAlignedStorageLineBytes); +static_assert(sizeof(CacheLine) == kAlignedStorageLineBytes); +static_assert(kAlignedStorageLineBytes % sizeof(std::uint64_t) == 0); +static_assert(kAlignedStorageLineBytes % sizeof(std::uint16_t) == 0); + /** - * @brief A simple aligned storage for cache-line sized blocks. + * @brief Aligned storage for 64-byte blocks. * - * @details Provides typed views over the same underlying storage as cache - * lines, 64-bit words, or bytes. All spans are contiguous and sized to the - * total storage capacity. + * @details The constructor and resize accept a logical size in bits. Storage is + * rounded up to a full 64-byte block, and all views expose the padded capacity. */ class AlignedStorage { private: std::vector data_; + static constexpr std::size_t LinesForBits(std::size_t bits) { + return bits / kAlignedStorageLineBits + + (bits % kAlignedStorageLineBits != 0); + } + public: AlignedStorage() = default; /** - * @brief Construct storage for at least @p bits bytes, rounded up to 512 - * bits. + * @brief Construct storage for at least @p bits bits. + */ + explicit AlignedStorage(std::size_t bits) : data_(LinesForBits(bits)) {} + + /** + * @brief Resize storage to hold at least @p bits bits. + */ + void resize(std::size_t bits) { data_.resize(LinesForBits(bits)); } + + /** + * @brief Padded storage capacity in bits. + */ + std::size_t capacity_bits() const { + return data_.size() * kAlignedStorageLineBits; + } + + /** + * @brief Padded storage capacity in bytes. + */ + std::size_t capacity_bytes() const { + return data_.size() * kAlignedStorageLineBytes; + } + + /** + * @brief Bytes reserved by the underlying aligned cache-line buffer. */ - AlignedStorage(size_t bits) : data_((bits + 511) / 512) {} + std::size_t allocated_bytes() const { + return data_.capacity() * kAlignedStorageLineBytes; + } /** - * @brief Resize storage to hold at least @p bits bits, rounded up to 512 + * @brief Request that reserved storage be reduced to the current size. + */ + void shrink_to_fit() { data_.shrink_to_fit(); } + + /** + * @brief Mutable view as cache lines. */ - void resize(size_t bits) { data_.resize((bits + 511) / 512); } - /** @brief Mutable view as cache lines. */ std::span AsLines() { return data_; } - /** @brief Const view as cache lines. */ + /** + * @brief Const view as cache lines. + */ std::span AsConstLines() const { return data_; } /** * @brief Mutable view as 64-bit words. */ - std::span As64BitInts() { - return std::span(reinterpret_cast(data_.data()), - data_.size() * 8); + std::span As64BitInts() { + return std::span( + reinterpret_cast(data_.data()), + data_.size() * kAlignedStorageLineWords64); } - /** @brief Const view as 64-bit words. */ - std::span AsConst64BitInts() const { - return std::span( - reinterpret_cast(data_.data()), data_.size() * 8); + /** + * @brief Const view as 64-bit words. + */ + std::span AsConst64BitInts() const { + return std::span( + reinterpret_cast(data_.data()), + data_.size() * kAlignedStorageLineWords64); } /** * @brief Mutable view as bytes. - * @note Uses a byte pointer to the underlying storage. */ - std::span AsBytes() { - return std::span(reinterpret_cast(data_.data()), - data_.size() * 64); - } + std::span AsBytes() { return std::as_writable_bytes(AsLines()); } - /** @brief Const view as bytes. */ + /** + * @brief Const view as bytes. + */ std::span AsConstBytes() const { - return std::span( - reinterpret_cast(data_.data()), data_.size() * 64); + return std::as_bytes(AsConstLines()); } /** - * @brief Mutable view as bytes. - * @note Uses a byte pointer to the underlying storage. + * @brief Mutable view as 16-bit words. */ std::span As16BitInts() { return std::span( - reinterpret_cast(data_.data()), data_.size() * 32); + reinterpret_cast(data_.data()), + data_.size() * kAlignedStorageLineWords16); } - /** @brief Const view as bytes. */ + /** + * @brief Const view as 16-bit words. + */ std::span AsConst16BitInts() const { return std::span( reinterpret_cast(data_.data()), - data_.size() * 32); + data_.size() * kAlignedStorageLineWords16); } }; + +} // namespace pixie diff --git a/include/pixie/experimental/excess.h b/include/pixie/experimental/excess.h index ef64dde..fde9f31 100644 --- a/include/pixie/experimental/excess.h +++ b/include/pixie/experimental/excess.h @@ -2,13 +2,842 @@ #include +#include +#include #include #include #include +// clang-format off +/** + * Benchmark note: + * + * This header keeps experimental and historical excess_min/excess_positions + * variants for comparison benchmarks. Production excess code lives in + * pixie/bits.h; a benchmark win here should be treated as a candidate to port, + * not evidence that callers use this variant already. + * + * excess_min_128 method comparison, 2026-06-17: + * taskset -c 0 ./build/release/excess_positions_benchmarks + * --benchmark_filter='^(BM_ExcessMin128/(left:0/right:(128|16|32)|left:1/right:17|left:3/right:35|left:5/right:37|left:56/right:72|left:60/right:68|left:63/right:64|left:17/right:17)|BM_ExcessMin128_(ByteLUT|NibbleLUT|DeinterleavedSSE|Lane64SSE|Split64SSE)/(left:0/right:(128|16|32)|left:1/right:17|left:3/right:35|left:5/right:37|left:56/right:72|left:60/right:68|left:63/right:64|left:17/right:17)|BM_ExcessMin128_RandomRange$|BM_ExcessMin128_(ByteLUT|NibbleLUT|DeinterleavedSSE|Lane64SSE|Split64SSE)_RandomRange$)' + * --benchmark_min_time=0.5s + * --benchmark_repetitions=5 + * --benchmark_report_aggregates_only=true + * + * Tables report median CPU time across 5 repetitions. Each numeric cell is + * nanoseconds per `excess_min_128` call. Columns named `A-B` are the inclusive + * prefix-offset arguments `[left, right]`; `Random` is the benchmark's + * reproducible mixed-range workload. + * + * excess_min_128 CPU time: + * + * | method | 0-128 | 0-16 | 0-32 | 1-17 | 3-35 | 5-37 | + * | :--------------- | ----: | ----: | ----: | ----: | ----: | ----: | + * | Production | 5.17 | 3.54 | 5.88 | 7.97 | 7.58 | 8.17 | + * | ByteLUT | 17.82 | 2.97 | 5.04 | 11.51 | 11.96 | 10.85 | + * | NibbleLUT | 40.00 | 4.78 | 8.14 | 8.59 | 10.72 | 10.81 | + * | DeinterleavedSSE | 5.44 | 6.08 | 6.24 | 8.28 | 7.84 | 8.90 | + * | Lane64SSE | 6.53 | 7.26 | 7.27 | 10.25 | 10.26 | 10.30 | + * | Split64SSE | 9.67 | 6.22 | 6.30 | 9.63 | 9.52 | 9.48 | + * + * | method | 56-72 | 60-68 | 63-64 | 17-17 | Random | + * | :--------------- | ----: | ----: | ----: | ----: | -----: | + * | Production | 4.01 | 5.89 | 2.12 | 1.55 | 10.07 | + * | ByteLUT | 3.60 | 8.74 | 2.34 | 1.55 | 13.14 | + * | NibbleLUT | 5.27 | 3.52 | 2.33 | 1.55 | 17.48 | + * | DeinterleavedSSE | 6.30 | 6.38 | 2.07 | 1.55 | 9.10 | + * | Lane64SSE | 4.27 | 7.23 | 3.84 | 2.25 | 11.36 | + * | Split64SSE | 9.86 | 9.75 | 3.17 | 1.55 | 11.02 | + * + * Diagnostic run, 2026-05-30: + * taskset -c 0 build/benchmarks-diagnostic_local/excess_positions_benchmarks + * --benchmark_filter='BM_ExcessPositions512' + * --benchmark_repetitions=3 + * --benchmark_perf_counters=CYCLES,INSTRUCTIONS,CACHE-MISSES + * --benchmark_counters_tabular=true + * + * Counter table for `excess_positions_512` with a random target in [-128, 128]. + * `CPU` is nanoseconds per call. `cycles`, `instructions`, and `cache misses` + * are hardware-counter events per call. + * + * | method | CPU (ns) | cycles | instructions | cache misses | + * | :----------- | -------: | -----: | -----------: | -----------: | + * | Production | 11.4 | 50.8 | 188.9 | 0.001 | + * | LUTAVX512 | 12.8 | 56.8 | 195.5 | 0.002 | + * | BranchingLUT | 16.7 | 73.4 | 261.4 | 0.003 | + * | ExpandAVX512 | 21.0 | 93.6 | 266.6 | 0.003 | + * | Expand8 | 24.6 | 109.9 | 449.7 | 0.002 | + * | Expand | 46.8 | 207.7 | 784.8 | 0.006 | + * | ByteLUT | 49.7 | 221.4 | 754.5 | 0.008 | + * | Scalar | 374.2 | 1656.0 | 7716.6 | 0.041 | + * + * CPU time for `excess_positions_512` fixed-target rows. Numeric cells are + * nanoseconds per call. Columns are target excess values. + * + * | method | -64 | -8 | 0 | 8 | 64 | + * | :----------- | ----: | ----: | ----: | ----: | ----: | + * | Production | 11.6 | 18.0 | 18.3 | 19.1 | 12.3 | + * | LUTAVX512 | 13.4 | 17.9 | 19.2 | 21.3 | 13.2 | + * | BranchingLUT | 19.1 | 28.9 | 28.7 | 28.3 | 16.8 | + * | ExpandAVX512 | 22.7 | 36.6 | 36.3 | 36.2 | 22.5 | + * | Expand8 | 17.6 | 52.7 | 47.5 | 46.9 | 17.7 | + * | Expand | 51.0 | 85.9 | 86.1 | 85.5 | 53.9 | + * | ByteLUT | 34.7 | 77.4 | 76.9 | 79.0 | 34.3 | + * | Scalar | 367.2 | 433.5 | 466.3 | 428.1 | 364.6 | + */ +// clang-format on + namespace pixie::experimental { +namespace detail { + +constexpr int8_t nibble_delta(uint8_t x) { + return static_cast(2 * std::popcount(x) - 4); +} + +constexpr int8_t byte_delta(uint8_t x) { + return static_cast(2 * std::popcount(x) - 8); +} + +constexpr int8_t min_prefix(uint8_t x, int bits) { + int cur = 0; + int best = 0; + for (int bit = 0; bit < bits; ++bit) { + cur += ((x >> bit) & 1u) != 0 ? 1 : -1; + if (bit == 0 || cur < best) { + best = cur; + } + } + return static_cast(best); +} + +constexpr int8_t min_prefix_offset(uint8_t x, int bits) { + int cur = 0; + int best = 0; + int best_offset = 1; + for (int bit = 0; bit < bits; ++bit) { + cur += ((x >> bit) & 1u) != 0 ? 1 : -1; + if (bit == 0 || cur < best) { + best = cur; + best_offset = bit + 1; + } + } + return static_cast(best_offset); +} + +template +constexpr std::array make_lut(Fn fn) { + std::array out{}; + for (size_t i = 0; i < N; ++i) { + out[i] = fn(static_cast(i)); + } + return out; +} + +static inline constexpr std::array kNibbleDelta = + make_lut<16>([](uint8_t x) { return nibble_delta(x); }); +static inline constexpr std::array kNibbleMin = + make_lut<16>([](uint8_t x) { return min_prefix(x, 4); }); +static inline constexpr std::array kNibbleMinOffset = + make_lut<16>([](uint8_t x) { return min_prefix_offset(x, 4); }); +static inline constexpr std::array, 4> + kPartialNibbleMinOffset = [] { + std::array, 4> out{}; + for (size_t width = 1; width < out.size(); ++width) { + out[width] = make_lut<16>( + [width](uint8_t x) { return min_prefix_offset(x, width); }); + } + return out; + }(); +static inline constexpr std::array kByteDelta = + make_lut<256>([](uint8_t x) { return byte_delta(x); }); +static inline constexpr std::array kByteMin = + make_lut<256>([](uint8_t x) { return min_prefix(x, 8); }); +static inline constexpr std::array kByteMinOffset = + make_lut<256>([](uint8_t x) { return min_prefix_offset(x, 8); }); + +static inline void scan_bit(const uint64_t* s, + size_t bit, + int& current, + int& best, + size_t& best_offset) noexcept { + current += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (current < best) { + best = current; + best_offset = offset; + } +} + +} // namespace detail + +/** + * @brief Reference scalar excess_min_128 implementation. + * + * @details Workflow: + * + * prefix(left) -> scan bits left..right-1 -> first strict minimum + * + * The value at offset left is included before scanning any bits, matching the + * production inclusive prefix range [left, right]. Each scanned bit advances to + * the next prefix offset. Ties are intentionally ignored, so the first minimum + * offset is preserved. + */ +static inline ExcessResult excess_min_128_scalar_bits(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + for (size_t bit = left; bit < right; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + return {best, best_offset}; +} + +/** + * @brief Scalar 4-bit LUT excess_min_128 experiment. + * + * @details Workflow: + * + * unaligned bits -> full nibbles -> trailing bits + * | delta + * | local min + * ` first local min offset + * + * Full nibbles use lookup tables for the nibble delta, the minimum prefix value + * inside positions 1..4, and the first local bit offset that reaches that + * minimum. Boundary bits are scanned scalar so the LUT never observes bits + * outside the query range. + */ +static inline ExcessResult excess_min_128_nibble_lut(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + size_t bit = left; + for (; bit < right && (bit & 3u) != 0; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + + for (; bit + 4 <= right; bit += 4) { + const uint8_t nibble = + static_cast((s[bit >> 6] >> (bit & 63)) & 0xFu); + const int candidate = current + detail::kNibbleMin[nibble]; + if (candidate < best) { + best = candidate; + best_offset = bit + static_cast(detail::kNibbleMinOffset[nibble]); + } + current += detail::kNibbleDelta[nibble]; + } + + for (; bit < right; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + return {best, best_offset}; +} + +/** + * @brief Scalar 8-bit LUT excess_min_128 experiment. + * + * @details Workflow: + * + * unaligned bits -> full bytes -> trailing bits + * | delta + * | local min + * ` first local min offset + * + * Full bytes use lookup tables for byte delta, minimum prefix value inside + * positions 1..8, and the first local bit offset that reaches that minimum. + * This reduces loop iterations on byte-aligned ranges but pays scalar boundary + * work on unaligned ranges. + */ +static inline ExcessResult excess_min_128_byte_lut(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + size_t bit = left; + for (; bit < right && (bit & 7u) != 0; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + + for (; bit + 8 <= right; bit += 8) { + const uint8_t byte = + static_cast((s[bit >> 6] >> (bit & 63)) & 0xFFu); + const int candidate = current + detail::kByteMin[byte]; + if (candidate < best) { + best = candidate; + best_offset = bit + static_cast(detail::kByteMinOffset[byte]); + } + current += detail::kByteDelta[byte]; + } + + for (; bit < right; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + return {best, best_offset}; +} + +/** + * @brief Hybrid dispatch over scalar, byte-LUT, nibble-LUT, and production. + * + * @details Workflow: + * + * width <= 2 -> scalar bits + * width <= 64 and byte aligned -> byte LUT + * width <= 32 -> nibble LUT + * otherwise -> production excess_min_128 + * + * This variant probes whether the fastest implementation depends primarily on + * query width and boundary alignment. It keeps production behavior for wider + * ranges where the AVX2 production path usually wins. + */ +static inline ExcessResult excess_min_128_hybrid_lut(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + const size_t clamped_left = std::min(left, 128); + const size_t clamped_right = std::min(right, 128); + const size_t width = clamped_right - clamped_left; + + if (width <= 2) { + return excess_min_128_scalar_bits(s, left, right); + } + if (width <= 64 && (clamped_left & 7u) == 0 && (clamped_right & 7u) == 0) { + return excess_min_128_byte_lut(s, left, right); + } + if (width <= 32) { + return excess_min_128_nibble_lut(s, left, right); + } + return excess_min_128(s, left, right); +} + #ifdef PIXIE_AVX2_SUPPORT +// clang-format off +static inline const __m128i excess_lut_delta_128 = _mm_setr_epi8( + -4, -2, -2, 0, + -2, 0, 0, 2, + -2, 0, 0, 2, + 0, 2, 2, 4); +static inline const __m128i excess_lut_min_128 = _mm_setr_epi8( + -4, -2, -2, 0, + -2, 0, -1, 1, + -3, -1, -1, 1, + -2, 0, -1, 1); +static inline const __m128i excess_lut_nibble_index_128 = _mm_setr_epi8( + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15); +static inline const __m128i excess_lut_low_nibble_index_128 = _mm_setr_epi8( + 0, 2, 4, 6, + 8, 10, 12, 14, + 16, 18, 20, 22, + 24, 26, 28, 30); +static inline const __m128i excess_lut_high_nibble_index_128 = _mm_setr_epi8( + 1, 3, 5, 7, + 9, 11, 13, 15, + 17, 19, 21, 23, + 25, 27, 29, 31); +static inline const __m128i excess_lut_nibble_mask_128 = _mm_set1_epi8(0x0F); +// clang-format on + +namespace detail { + +static inline __m128i excess_nibbles_64_sse(uint64_t word) noexcept { + const __m128i word_vec = _mm_cvtsi64_si128(static_cast(word)); + const __m128i lo_nibbles = + _mm_and_si128(word_vec, excess_lut_nibble_mask_128); + const __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(word_vec, 4), excess_lut_nibble_mask_128); + return _mm_unpacklo_epi8(lo_nibbles, hi_nibbles); +} + +static inline __m128i excess_prefix_sum_16x_i8(__m128i v) noexcept { + __m128i x = v; + __m128i t = _mm_slli_si128(x, 1); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 2); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 4); + x = _mm_add_epi8(x, t); + t = _mm_slli_si128(x, 8); + return _mm_add_epi8(x, t); +} + +static inline int horizontal_min_i8(__m128i v) noexcept { + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 8)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 4)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 2)); + v = _mm_min_epi8(v, _mm_alignr_epi8(v, v, 1)); + return static_cast(static_cast(_mm_extract_epi8(v, 0))); +} + +static inline void scan_full_nibbles_64_sse(uint64_t word, + int lane_base_excess, + size_t lane_base_offset, + size_t first_nibble, + size_t last_nibble, + int& best, + size_t& best_offset) noexcept { + if (first_nibble >= last_nibble) { + return; + } + + const __m128i nibbles = excess_nibbles_64_sse(word); + __m128i ps = + excess_prefix_sum_16x_i8(_mm_shuffle_epi8(excess_lut_delta_128, nibbles)); + const __m128i excl_ps = _mm_alignr_epi8(ps, _mm_setzero_si128(), 15); + const __m128i candidates = _mm_add_epi8( + _mm_add_epi8(_mm_set1_epi8(static_cast(lane_base_excess)), + excl_ps), + _mm_shuffle_epi8(excess_lut_min_128, nibbles)); + + const __m128i idx = excess_lut_nibble_index_128; + const __m128i first_minus_one = + _mm_set1_epi8(static_cast(static_cast(first_nibble) - 1)); + const __m128i last = _mm_set1_epi8(static_cast(last_nibble)); + const __m128i active = _mm_and_si128(_mm_cmpgt_epi8(idx, first_minus_one), + _mm_cmpgt_epi8(last, idx)); + const __m128i masked_candidates = + _mm_blendv_epi8(_mm_set1_epi8(127), candidates, active); + + __m128i min128 = masked_candidates; + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 8)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 4)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 2)); + min128 = _mm_min_epi8(min128, _mm_alignr_epi8(min128, min128, 1)); + + const int candidate_min = + static_cast(static_cast(_mm_extract_epi8(min128, 0))); + if (candidate_min < best) { + const __m128i equal_min = _mm_cmpeq_epi8( + masked_candidates, _mm_set1_epi8(static_cast(candidate_min))); + const uint32_t equal_mask = + static_cast(_mm_movemask_epi8(equal_min)); + const uint32_t nibble_index = std::countr_zero(equal_mask); + const uint8_t nibble = + static_cast((word >> (nibble_index * 4u)) & 0xFu); + best = candidate_min; + best_offset = lane_base_offset + static_cast(nibble_index) * 4u + + static_cast(kNibbleMinOffset[nibble]); + } +} + +static inline size_t partial_nibble_min_offset(uint8_t nibble, + size_t width) noexcept { + return static_cast(kPartialNibbleMinOffset[width][nibble]); +} + +static inline ExcessResult excess_min_128_split64_sse_impl( + const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + size_t bit = left; + for (; bit < right && (bit & 3u) != 0; ++bit) { + scan_bit(s, bit, current, best, best_offset); + } + + size_t first_full_nibble = bit >> 2; + const size_t last_full_nibble = right >> 2; + while (first_full_nibble < last_full_nibble) { + const size_t word_index = first_full_nibble >> 4; + const size_t lane_first = first_full_nibble & 15u; + const size_t lane_last = + std::min(last_full_nibble - word_index * 16u, 16); + const size_t lane_base_offset = word_index * 64u; + scan_full_nibbles_64_sse( + s[word_index], prefix_excess_128(s, lane_base_offset), lane_base_offset, + lane_first, lane_last, best, best_offset); + first_full_nibble = word_index * 16u + lane_last; + } + + bit = std::max(bit, first_full_nibble * 4u); + current = prefix_excess_128(s, bit); + for (; bit < right; ++bit) { + scan_bit(s, bit, current, best, best_offset); + } + + return {best, best_offset}; +} + +} // namespace detail + +/** + * @brief Single-64-bit-lane SSE excess_min_128 experiment. + * + * @details Workflow: + * + * scalar boundary -> one 64-bit word as 16 nibbles -> scalar tail + * fallback to production if full nibbles cross words + * + * This variant tests whether short ranges benefit from avoiding the 128-bit + * cross-lane prefix work used by broader vector paths. It only handles the + * single-word full-nibble case; multi-word ranges fall back to production. + */ +static inline ExcessResult excess_min_128_lane64_sse(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + const size_t clamped_left = std::min(left, 128); + const size_t clamped_right = std::min(right, 128); + const size_t first_full_nibble = ((clamped_left + 3u) & ~size_t{3}) >> 2; + const size_t last_full_nibble = clamped_right >> 2; + if (first_full_nibble < last_full_nibble && + (first_full_nibble >> 4) != ((last_full_nibble - 1u) >> 4)) { + return excess_min_128(s, left, right); + } + return detail::excess_min_128_split64_sse_impl(s, left, right); +} + +/** + * @brief Split-64-bit-lane SSE excess_min_128 experiment. + * + * @details Workflow: + * + * scalar boundary -> word 0 full nibbles -> word 1 full nibbles -> tail + * 16-nibble SSE 16-nibble SSE + * + * Each 64-bit word is processed as an independent 16-nibble vector scan. The + * base excess for a word is recomputed from prefix_excess_128, avoiding + * vector-prefix carry propagation across the 64-bit boundary. + */ +static inline ExcessResult excess_min_128_split64_sse(const uint64_t* s, + size_t left, + size_t right) noexcept { + return detail::excess_min_128_split64_sse_impl(s, left, right); +} + +/** + * @brief Deinterleaved-nibble SSE excess_min_128 experiment. + * + * @details Workflow: + * + * scalar boundary -> 16 low byte-nibbles -> low candidates + * \-> 16 high byte-nibbles -> high candidates + * -> interleaved first-min selection + * + * The prefix scan is over byte deltas. Low-nibble candidates use the byte + * prefix before each byte; high-nibble candidates add the low-nibble delta. + * This avoids concatenating low/high nibbles or carrying prefix sums across a + * synthetic 32-nibble vector. + */ +static inline ExcessResult excess_min_128_deinterleaved_sse( + const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + int current = best; + size_t bit = left; + for (; bit < right && (bit & 3u) != 0; ++bit) { + detail::scan_bit(s, bit, current, best, best_offset); + } + + const size_t first_nibble = bit >> 2; + const size_t last_full_nibble = right >> 2; + const size_t right_partial_width = bit < right ? (right & 3u) : 0; + const size_t end_nibble = + last_full_nibble + (right_partial_width == 0 ? 0 : 1); + + if (first_nibble < end_nibble) { + const __m128i bytes = _mm_loadu_si128(reinterpret_cast(s)); + const __m128i lo_nibbles = _mm_and_si128(bytes, excess_lut_nibble_mask_128); + const __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(bytes, 4), excess_lut_nibble_mask_128); + const __m128i lo_delta = _mm_shuffle_epi8(excess_lut_delta_128, lo_nibbles); + const __m128i hi_delta = _mm_shuffle_epi8(excess_lut_delta_128, hi_nibbles); + const __m128i byte_delta = _mm_add_epi8(lo_delta, hi_delta); + const __m128i byte_prefix = detail::excess_prefix_sum_16x_i8(byte_delta); + const __m128i byte_prefix_before = _mm_slli_si128(byte_prefix, 1); + + __m128i lo_local_min = _mm_shuffle_epi8(excess_lut_min_128, lo_nibbles); + __m128i hi_local_min = _mm_shuffle_epi8(excess_lut_min_128, hi_nibbles); + + const __m128i byte_index = excess_lut_nibble_index_128; + if (right_partial_width != 0) { + const bool partial_is_high = (last_full_nibble & 1u) != 0; + const size_t partial_byte = last_full_nibble >> 1; + const __m128i partial_source = partial_is_high ? hi_nibbles : lo_nibbles; + __m128i partial_min = + _mm_shuffle_epi8(excess_lut_pos0_sse, partial_source); + if (right_partial_width >= 2) { + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos1_sse, partial_source)); + } + if (right_partial_width >= 3) { + partial_min = _mm_min_epi8( + partial_min, _mm_shuffle_epi8(excess_lut_pos2_sse, partial_source)); + } + const __m128i partial_lane = _mm_cmpeq_epi8( + byte_index, _mm_set1_epi8(static_cast(partial_byte))); + if (partial_is_high) { + hi_local_min = _mm_blendv_epi8(hi_local_min, partial_min, partial_lane); + } else { + lo_local_min = _mm_blendv_epi8(lo_local_min, partial_min, partial_lane); + } + } + + const __m128i lo_candidates = + _mm_add_epi8(byte_prefix_before, lo_local_min); + const __m128i hi_candidates = + _mm_add_epi8(_mm_add_epi8(byte_prefix_before, lo_delta), hi_local_min); + + __m128i masked_lo = lo_candidates; + __m128i masked_hi = hi_candidates; + if (first_nibble != 0 || end_nibble != 32) { + const __m128i first_minus_one = _mm_set1_epi8( + static_cast(static_cast(first_nibble) - 1)); + const __m128i last = _mm_set1_epi8(static_cast(end_nibble)); + const __m128i lo_active = _mm_and_si128( + _mm_cmpgt_epi8(excess_lut_low_nibble_index_128, first_minus_one), + _mm_cmpgt_epi8(last, excess_lut_low_nibble_index_128)); + const __m128i hi_active = _mm_and_si128( + _mm_cmpgt_epi8(excess_lut_high_nibble_index_128, first_minus_one), + _mm_cmpgt_epi8(last, excess_lut_high_nibble_index_128)); + masked_lo = _mm_blendv_epi8(_mm_set1_epi8(127), lo_candidates, lo_active); + masked_hi = _mm_blendv_epi8(_mm_set1_epi8(127), hi_candidates, hi_active); + } + const int candidate_min = + detail::horizontal_min_i8(_mm_min_epi8(masked_lo, masked_hi)); + + if (candidate_min < best) { + const __m128i min_vec = _mm_set1_epi8(static_cast(candidate_min)); + const uint32_t lo_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(masked_lo, min_vec))); + const uint32_t hi_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(masked_hi, min_vec))); + const uint32_t lo_nibble_index = + lo_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(lo_equal_mask)) * 2u; + const uint32_t hi_nibble_index = + hi_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(hi_equal_mask)) * 2u + + 1u; + const uint32_t nibble_index = std::min(lo_nibble_index, hi_nibble_index); + const uint32_t byte_offset = nibble_index >> 1u; + const uint64_t byte_word = s[byte_offset >> 3u]; + const uint8_t byte = static_cast( + (byte_word >> ((byte_offset & 7u) * 8u)) & 0xFFu); + const uint8_t nibble = (nibble_index & 1u) == 0 + ? static_cast(byte & 0xFu) + : static_cast((byte >> 4u) & 0xFu); + const size_t local_offset = + right_partial_width != 0 && nibble_index == last_full_nibble + ? detail::partial_nibble_min_offset(nibble, right_partial_width) + : static_cast(detail::kNibbleMinOffset[nibble]); + best = candidate_min; + best_offset = static_cast(nibble_index) * 4u + local_offset; + } + } + + return {best, best_offset}; +} + +/** + * @brief Deinterleaved SSE with a full-block fast path. + * + * @details Workflow: + * + * left == 0 && right == 128 -> deinterleaved full block without masks + * otherwise -> deinterleaved SSE + * + * The full-block path skips scalar boundary handling, partial-nibble handling, + * and active-lane masks. It is intended to measure whether the common + * whole-block shape is worth specializing independently from the general + * deinterleaved implementation. + */ +static inline ExcessResult excess_min_128_deinterleaved_full_sse( + const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + if (left != 0 || right != 128) { + return excess_min_128_deinterleaved_sse(s, left, right); + } + + const __m128i bytes = _mm_loadu_si128(reinterpret_cast(s)); + const __m128i lo_nibbles = _mm_and_si128(bytes, excess_lut_nibble_mask_128); + const __m128i hi_nibbles = + _mm_and_si128(_mm_srli_epi16(bytes, 4), excess_lut_nibble_mask_128); + const __m128i lo_delta = _mm_shuffle_epi8(excess_lut_delta_128, lo_nibbles); + const __m128i hi_delta = _mm_shuffle_epi8(excess_lut_delta_128, hi_nibbles); + const __m128i byte_delta = _mm_add_epi8(lo_delta, hi_delta); + const __m128i byte_prefix = detail::excess_prefix_sum_16x_i8(byte_delta); + const __m128i byte_prefix_before = _mm_slli_si128(byte_prefix, 1); + + const __m128i lo_candidates = _mm_add_epi8( + byte_prefix_before, _mm_shuffle_epi8(excess_lut_min_128, lo_nibbles)); + const __m128i hi_candidates = + _mm_add_epi8(_mm_add_epi8(byte_prefix_before, lo_delta), + _mm_shuffle_epi8(excess_lut_min_128, hi_nibbles)); + const int candidate_min = + detail::horizontal_min_i8(_mm_min_epi8(lo_candidates, hi_candidates)); + + int best = 0; + size_t best_offset = 0; + if (candidate_min < best) { + const __m128i min_vec = _mm_set1_epi8(static_cast(candidate_min)); + const uint32_t lo_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(lo_candidates, min_vec))); + const uint32_t hi_equal_mask = static_cast( + _mm_movemask_epi8(_mm_cmpeq_epi8(hi_candidates, min_vec))); + const uint32_t lo_nibble_index = + lo_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(lo_equal_mask)) * 2u; + const uint32_t hi_nibble_index = + hi_equal_mask == 0 + ? 32u + : static_cast(std::countr_zero(hi_equal_mask)) * 2u + 1u; + const uint32_t nibble_index = std::min(lo_nibble_index, hi_nibble_index); + const uint32_t byte_offset = nibble_index >> 1u; + const uint64_t byte_word = s[byte_offset >> 3u]; + const uint8_t byte = + static_cast((byte_word >> ((byte_offset & 7u) * 8u)) & 0xFFu); + const uint8_t nibble = (nibble_index & 1u) == 0 + ? static_cast(byte & 0xFu) + : static_cast((byte >> 4u) & 0xFu); + best = candidate_min; + best_offset = static_cast(nibble_index) * 4u + + static_cast(detail::kNibbleMinOffset[nibble]); + } + return {best, best_offset}; +} + +/** + * @brief Deinterleaved SSE with a narrow byte-aligned shortcut. + * + * @details Workflow: + * + * byte-aligned width <= 16 -> byte LUT + * otherwise -> deinterleaved SSE + * + * Production currently keeps a byte-LUT shortcut for byte-aligned widths up to + * 32. This variant narrows that condition to test whether the 32-bit case is + * worth its dispatch cost in mixed workloads. + */ +static inline ExcessResult excess_min_128_deinterleaved_byte16_sse( + const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + const size_t clamped_left = std::min(left, 128); + const size_t clamped_right = std::min(right, 128); + const size_t width = clamped_right - clamped_left; + if (width <= 16 && (clamped_left & 7u) == 0 && (clamped_right & 7u) == 0) { + return excess_min_128_byte_lut(s, left, right); + } + return excess_min_128_deinterleaved_sse(s, left, right); +} + +/** + * @brief Short-range dispatch experiment. + * + * @details Workflow: + * + * width <= 2 -> scalar bits + * full nibbles in 1 word -> lane64 SSE + * width <= 80 -> split64 SSE + * otherwise -> production excess_min_128 + * + * This variant tests two ideas together: avoid 128-bit lane crossing when a + * query is contained in one 64-bit word, and skip a few production iterations + * for medium ranges where split-lane scans may be cheaper. + */ +static inline ExcessResult excess_min_128_short_skip(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + const size_t clamped_left = std::min(left, 128); + const size_t clamped_right = std::min(right, 128); + const size_t width = clamped_right - clamped_left; + if (width <= 2) { + return excess_min_128_scalar_bits(s, left, right); + } + + const size_t first_full_nibble = ((clamped_left + 3u) & ~size_t{3}) >> 2; + const size_t last_full_nibble = clamped_right >> 2; + if (first_full_nibble < last_full_nibble && + (first_full_nibble >> 4) == ((last_full_nibble - 1u) >> 4)) { + return excess_min_128_lane64_sse(s, left, right); + } + if (width <= 80) { + return excess_min_128_split64_sse(s, left, right); + } + return excess_min_128(s, left, right); +} + // clang-format off static inline const __m256i excess_branch_lut_em4 = _mm256_setr_epi8( 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -133,6 +962,88 @@ static inline int8_t excess_last_prefix_32x_i8(__m256i pref) noexcept { return (int8_t)_mm_extract_epi8(hi, 15); } +/** + * @brief AVX2 expand-to-i16 excess_min_128 experiment. + * + * @details Workflow: + * + * 16 input bits -> 16 x i16 +/-1 -> vector prefix sum -> store -> scalar min + * + * The implementation scans eight 16-bit chunks. For chunks overlapping the + * query, bits are expanded to signed +/-1 i16 lanes and prefix summed with the + * running carry. The vector result is stored to memory, then relevant lanes are + * checked scalar for the first strict minimum. + */ +static inline ExcessResult excess_min_128_expand16_avx2(const uint64_t* s, + size_t left, + size_t right) noexcept { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int best = prefix_excess_128(s, left); + size_t best_offset = left; + if (left == right) { + return {best, best_offset}; + } + + const __m256i masks = excess_bit_masks_16x(); + const __m256i zero = _mm256_setzero_si256(); + const __m256i pos = _mm256_set1_epi16(1); + const __m256i neg = _mm256_set1_epi16(-1); + + int carry = 0; + alignas(32) int16_t prefix_values[16]; + for (size_t chunk = 0; chunk < 8; ++chunk) { + const size_t chunk_bit = chunk * 16; + const uint16_t bits = + chunk < 4 + ? static_cast((s[0] >> (chunk * 16)) & 0xFFFFu) + : static_cast((s[1] >> ((chunk - 4) * 16)) & 0xFFFFu); + const int delta = 2 * static_cast(std::popcount(bits)) - 16; + + if (chunk_bit + 1 <= right && chunk_bit + 16 >= left) { + const __m256i selected = _mm256_and_si256( + _mm256_set1_epi16(static_cast(bits)), masks); + const __m256i is_zero = _mm256_cmpeq_epi16(selected, zero); + const __m256i steps = _mm256_blendv_epi8(pos, neg, is_zero); + const __m256i pref = + _mm256_add_epi16(excess_prefix_sum_16x_i16(steps), + _mm256_set1_epi16(static_cast(carry))); + _mm256_store_si256(reinterpret_cast<__m256i*>(prefix_values), pref); + + for (size_t lane = 0; lane < 16; ++lane) { + const size_t offset = chunk_bit + lane + 1; + if (offset < left || offset > right) { + continue; + } + const int value = prefix_values[lane]; + if (value < best) { + best = value; + best_offset = offset; + } + } + } + carry += delta; + } + + return {best, best_offset}; +} + +/** + * @brief Historical AVX2 branching-LUT excess_positions_512 variant. + * + * @details Workflow: + * + * 128-bit block -> 32 nibbles -> prefix sums -> branch by target-relative + * -> LUT masks -> packed output bits + * + * Each 128-bit block is converted to nibbles, prefix-summed, and filtered by + * reachability. A family of per-target LUTs produces within-nibble match masks, + * which are packed back to the output words. + */ static inline void excess_positions_512_branching_lut(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -249,6 +1160,13 @@ static inline void excess_positions_512_branching_lut(const uint64_t* s, } } #else +/** + * @brief Scalar fallback for the branching-LUT positions variant. + * + * @details Used when AVX2 is not enabled. Delegates to production + * excess_positions_512 so callers can benchmark the same symbol across build + * configurations. + */ static inline void excess_positions_512_branching_lut(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -378,6 +1296,19 @@ static inline uint64_t excess_repeat_byte(int value) noexcept { static_cast(static_cast(value)); } +/** + * @brief AVX-512 nibble-LUT excess_positions_512 experiment. + * + * @details Workflow: + * + * 256 input bits -> 64 nibbles -> two 128-bit logical halves + * -> nibble prefix/LUT matches -> packed output masks + * + * The implementation processes four words at a time. It computes reachability + * for each 128-bit half, converts bytes to nibbles, builds exclusive prefix + * sums, compares nibble-local positions against the target, and packs matches + * back to four output words. + */ static inline void excess_positions_512_lut_avx512(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -463,6 +1394,13 @@ static inline void excess_positions_512_lut_avx512(const uint64_t* s, } } #else +/** + * @brief Fallback for the AVX-512 LUT positions variant. + * + * @details Used when AVX-512 is not enabled. Delegates to production + * excess_positions_512 so callers can benchmark the same symbol across build + * configurations. + */ static inline void excess_positions_512_lut_avx512(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -470,6 +1408,18 @@ static inline void excess_positions_512_lut_avx512(const uint64_t* s, } #endif +/** + * @brief Expand-to-i16 excess_positions_512 experiment. + * + * @details Workflow: + * + * 16 input bits -> 16 x i16 +/-1 -> vector prefix sum -> compare target + * -> pext mask -> output word + * + * With AVX2, this variant scans 16-bit chunks, expands them to i16 prefix + * lanes, compares absolute prefix values against the target, and compresses the + * comparison mask into output bits. Without AVX2 it uses a scalar scan. + */ static inline void excess_positions_512_expand(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -541,6 +1491,18 @@ static inline void excess_positions_512_expand(const uint64_t* s, #endif } +/** + * @brief AVX2 expand-to-i8 excess_positions_512 experiment. + * + * @details Workflow: + * + * 32 input bits -> 32 x i8 +/-1 -> byte prefix sum -> compare target + * -> movemask -> output word + * + * This variant handles 32-bit chunks. It first checks whether the target is + * reachable within the chunk, then expands bits to byte lanes and uses the + * vector comparison mask directly as output bits. + */ static inline void excess_positions_512_expand8(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -601,6 +1563,18 @@ static inline void excess_positions_512_expand8(const uint64_t* s, #endif } +/** + * @brief AVX-512 expand-to-i8 excess_positions_512 experiment. + * + * @details Workflow: + * + * 64 input bits -> 64 x i8 +/-1 -> byte prefix sum -> k-mask output + * + * This variant processes one 64-bit word per vector. If the target is + * unreachable in the word, it advances by popcount delta only. Otherwise it + * expands the word to byte lanes, prefix-sums the lanes, compares against the + * target, and stores the resulting AVX-512 mask as the output word. + */ static inline void excess_positions_512_expand_avx512(const uint64_t* s, int target_x, uint64_t* out) noexcept { @@ -681,6 +1655,19 @@ struct ExcessByteLut { inline constexpr ExcessByteLut kExcessByteLut; +/** + * @brief Scalar byte-LUT excess_positions_512 experiment. + * + * @details Workflow: + * + * byte -> relative target in [-8, 8] -> LUT match mask + * -> byte delta -> next byte base excess + * + * The table stores, for each byte, all bit positions that reach each local + * target and the byte delta. The scan walks 64 bytes, emits a mask when the + * relative target is in range, and then advances the running excess by the + * byte delta. + */ static inline void excess_positions_512_byte_lut(const uint64_t* s, int target_x, uint64_t* out) noexcept { diff --git a/include/pixie/experimental/rmm_btree.h b/include/pixie/experimental/rmm_btree.h index 1d83a70..981637b 100644 --- a/include/pixie/experimental/rmm_btree.h +++ b/include/pixie/experimental/rmm_btree.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -104,6 +105,14 @@ class RmMBTree : public RmMBase> { RmMBTree& operator=(const RmMBTree&) = default; RmMBTree& operator=(RmMBTree&&) noexcept = default; + /** + * @brief Minimum position and value returned by one range-min traversal. + */ + struct RangeMinQueryResult { + std::size_t position = npos; + int value = 0; + }; + /** * @brief Construct an RmM btree over an external bit-vector span. * @details The tree stores a non-owning view of @p words and builds its @@ -120,6 +129,22 @@ class RmMBTree : public RmMBase> { build(words, bit_count); } + /** + * @brief Construct with explicit rank/select support options. + * + * @details This keeps the normal RmMBTree API full-featured by default while + * allowing adapters that only need rank/select0 to skip select1 samples. + * @param one_count Optional exact number of 1-bits for exact select-sample + * allocation. + */ + RmMBTree(std::span words, + std::size_t bit_count, + BitVector::SelectSupport select_support, + std::optional one_count = std::nullopt, + std::size_t = kBlockBits) { + build(words, bit_count, select_support, one_count); + } + std::size_t size_impl() const { return bit_count_; } std::size_t rank1_impl(std::size_t end_position) const { @@ -276,6 +301,21 @@ class RmMBTree : public RmMBase> { return range_extreme_query_val(range_begin, range_end, true); } + /** + * @brief Return the first minimum position and value in one traversal. + * @details This is a concrete-type helper for adapters that need to compare + * a boundary prefix against the interior range minimum. Invalid ranges return + * `{npos, 0}`. + */ + RangeMinQueryResult range_min_query_result(std::size_t range_begin, + std::size_t range_end) const { + if (range_begin > range_end || range_end >= bit_count_) { + return {}; + } + const auto result = range_extreme_query(range_begin, range_end, true); + return {result.position, static_cast(result.value)}; + } + std::size_t range_max_query_pos_impl(std::size_t range_begin, std::size_t range_end) const { if (range_begin > range_end || range_end >= bit_count_) { @@ -292,6 +332,28 @@ class RmMBTree : public RmMBase> { return range_extreme_query_val(range_begin, range_end, false); } + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this rmM object, rank/select support, and summary buffers. + * The external bit words indexed by this object are not owned and are + * excluded. + */ + std::size_t memory_usage_bytes() const { + std::size_t bytes = sizeof(*this); + bytes += pixie::optional_nested_owned_memory_bytes(rank_index_); + bytes += pixie::vector_capacity_bytes(level_counts_); + bytes += pixie::vector_capacity_bytes(low_levels_); + bytes += pixie::vector_capacity_bytes(high_levels_); + for (const std::vector& level : low_levels_) { + bytes += pixie::vector_capacity_bytes(level); + } + for (const std::vector& level : high_levels_) { + bytes += pixie::vector_capacity_bytes(level); + } + return bytes; + } + std::size_t mincount_impl(std::size_t range_begin, std::size_t range_end) const { if (range_begin > range_end || range_end >= bit_count_) { @@ -460,6 +522,16 @@ class RmMBTree : public RmMBase> { * @p bit_count. */ void build(std::span words, std::size_t bit_count) { + build(words, bit_count, BitVector::SelectSupport::kBoth, std::nullopt); + } + + /** + * @brief Build with explicit rank/select support options. + */ + void build(std::span words, + std::size_t bit_count, + BitVector::SelectSupport select_support, + std::optional one_count = std::nullopt) { const std::size_t required_words = (bit_count + 63) / 64; if (words.size() < required_words) { throw std::invalid_argument( @@ -468,7 +540,7 @@ class RmMBTree : public RmMBase> { bits_ = words; bit_count_ = bit_count; - rank_index_.emplace(words, bit_count); + rank_index_.emplace(words, bit_count, select_support, one_count); block_count_ = (bit_count_ + kBlockBits - 1) / kBlockBits; std::vector block_summaries(block_count_); @@ -586,6 +658,13 @@ class RmMBTree : public RmMBase> { * @return Relative summary of `[begin, begin + length)`. */ Summary summarize_bits(std::size_t begin, std::size_t length) const { + if (length == kBlockBits && (begin % kBlockBits) == 0) { + const std::size_t block_index = begin / kBlockBits; + if (full_block_has_words(block_index)) { + return summarize_full_block(block_index); + } + } + Summary summary; summary.size_bits = length; if (length == 0) { @@ -614,6 +693,52 @@ class RmMBTree : public RmMBase> { return summary; } + /** + * @brief Summarize one complete 512-bit block with 128-bit excess kernels. + * @details Full Cartesian-BP construction blocks are aligned and contain all + * eight backing words. This fast path reuses the existing SIMD/minimum + * primitives from `bits.h` instead of reading every bit individually. + */ + Summary summarize_full_block(std::size_t block_index) const { + const std::uint64_t* block = bits_.data() + block_index * kBlockWords; + Summary summary; + for (std::size_t chunk = 0; chunk < kSearchChunkCount; ++chunk) { + summary = append(summary, + summarize_128_chunk(block + chunk * kSearchChunkWords)); + } + return summary; + } + + /** + * @brief Summarize one full 128-bit chunk relative to its first bit. + * @details Minima are found directly; maxima are minima of the bit-inverted + * chunk with the sign flipped. Minimum multiplicity is computed by the + * existing SIMD target-position kernel. + */ + static Summary summarize_128_chunk(const std::uint64_t* chunk) { + Summary summary; + summary.size_bits = kSearchChunkBits; + summary.ones = std::popcount(chunk[0]) + std::popcount(chunk[1]); + summary.block_excess = 2 * static_cast(summary.ones) - + static_cast(kSearchChunkBits); + + const ExcessResult minimum = excess_min_128(chunk, 1, kSearchChunkBits); + summary.min_excess = minimum.min_excess; + + const std::array inverted = {~chunk[0], + ~chunk[1]}; + const ExcessResult inverted_min = + excess_min_128(inverted.data(), 1, kSearchChunkBits); + summary.max_excess = -static_cast(inverted_min.min_excess); + + std::uint64_t minimum_positions[kSearchChunkWords]; + excess_positions_128(chunk, static_cast(summary.min_excess), + minimum_positions); + summary.min_count = std::popcount(minimum_positions[0]) + + std::popcount(minimum_positions[1]); + return summary; + } + /** * @brief Concatenate two relative summaries. * @details Produces the summary for `left || right`, translating right-side @@ -1413,8 +1538,8 @@ class RmMBTree : public RmMBase> { } struct NodeRef { - std::size_t level = 0; - std::size_t index = 0; + std::size_t level; + std::size_t index; }; static constexpr std::size_t kMaxCoverNodes = 512; @@ -1428,6 +1553,12 @@ class RmMBTree : public RmMBase> { std::size_t max_position = npos; }; + struct MinScanResult { + std::int64_t block_excess = 0; + std::int64_t min_value = std::numeric_limits::max(); + std::size_t min_position = npos; + }; + struct RangeExtremeResult { std::size_t position = npos; std::int64_t value = 0; @@ -1438,23 +1569,6 @@ class RmMBTree : public RmMBase> { std::uint64_t count = 0; }; - struct Cover { - std::array nodes{}; - std::size_t size = 0; - - /** - * @brief Append a cover node if the fixed-capacity buffer has room. - * @details Cover construction has a conservative fixed upper bound. Extra - * pushes are ignored instead of growing storage. - * @param node Node reference to append. - */ - void push(NodeRef node) { - if (size < nodes.size()) { - nodes[size++] = node; - } - } - }; - /** * @brief Return the position of the first range minimum or maximum. * @details Wrapper around `range_extreme_query` that extracts only the @@ -1483,7 +1597,78 @@ class RmMBTree : public RmMBase> { std::size_t range_end, bool find_min) const { return static_cast( - range_extreme_query(range_begin, range_end, find_min).value); + range_extreme_query_value(range_begin, range_end, find_min)); + } + + /** + * @brief Return only the minimum or maximum relative excess in a range. + * @details Uses the same decomposition as `range_extreme_query`, but skips + * the final descent needed to recover a bit position. + * @param range_begin Inclusive range start. + * @param range_end Inclusive range end. + * @param find_min True for minimum, false for maximum. + * @return Extreme relative excess value. + */ + std::int64_t range_extreme_query_value(std::size_t range_begin, + std::size_t range_end, + bool find_min) const { + std::int64_t value = 0; + std::int64_t best = find_min ? std::numeric_limits::max() + : std::numeric_limits::min(); + + auto consider_value = [&](std::int64_t candidate) { + if ((find_min && candidate < best) || (!find_min && candidate > best)) { + best = candidate; + } + }; + + const std::size_t range_end_exclusive = range_end + 1; + const std::size_t first_full_block = + (range_begin + kBlockBits - 1) / kBlockBits; + const std::size_t full_begin = + std::min(range_end_exclusive, first_full_block * kBlockBits); + if (range_begin < full_begin) { + if (find_min) { + const MinScanResult scan = scan_min_range(range_begin, full_begin); + consider_value(scan.min_value); + value += scan.block_excess; + } else { + const ScanResult scan = scan_range(range_begin, full_begin); + consider_value(scan.max_value); + value += scan.block_excess; + } + } + + const std::size_t last_full_block_exclusive = + range_end_exclusive / kBlockBits; + const std::size_t middle_begin = full_begin; + const std::size_t middle_end = + std::max(middle_begin, last_full_block_exclusive * kBlockBits); + if (middle_begin < middle_end) { + for_each_cover_node(middle_begin, middle_end, [&](NodeRef node) { + const Summary summary = summary_at(node.level, node.index); + consider_value(value + + (find_min ? summary.min_excess : summary.max_excess)); + value += summary.block_excess; + return true; + }); + } + + if (middle_end < range_end_exclusive) { + if (find_min) { + const MinScanResult scan = + scan_min_range(middle_end, range_end_exclusive); + consider_value(value + scan.min_value); + } else { + const ScanResult scan = scan_range(middle_end, range_end_exclusive); + consider_value(value + scan.max_value); + } + } + if (best == std::numeric_limits::max() || + best == std::numeric_limits::min()) { + return 0; + } + return best; } /** @@ -1521,10 +1706,15 @@ class RmMBTree : public RmMBase> { const std::size_t full_begin = std::min(range_end_exclusive, first_full_block * kBlockBits); if (range_begin < full_begin) { - const ScanResult scan = scan_range(range_begin, full_begin); - consider_point(find_min ? scan.min_value : scan.max_value, - find_min ? scan.min_position : scan.max_position); - value += scan.block_excess; + if (find_min) { + const MinScanResult scan = scan_min_range(range_begin, full_begin); + consider_point(scan.min_value, scan.min_position); + value += scan.block_excess; + } else { + const ScanResult scan = scan_range(range_begin, full_begin); + consider_point(scan.max_value, scan.max_position); + value += scan.block_excess; + } } const std::size_t last_full_block_exclusive = @@ -1533,11 +1723,8 @@ class RmMBTree : public RmMBase> { const std::size_t middle_end = std::max(middle_begin, last_full_block_exclusive * kBlockBits); if (middle_begin < middle_end) { - Cover cover; - collect_cover(middle_begin, middle_end, cover); - for (std::size_t i = 0; i < cover.size; ++i) { - const NodeRef& node = cover.nodes[i]; - Summary summary = summary_at(node.level, node.index); + for_each_cover_node(middle_begin, middle_end, [&](NodeRef node) { + const Summary summary = summary_at(node.level, node.index); const std::int64_t candidate = value + (find_min ? summary.min_excess : summary.max_excess); if ((find_min && candidate < best) || (!find_min && candidate > best)) { @@ -1547,15 +1734,19 @@ class RmMBTree : public RmMBase> { best_is_node = true; } value += summary.block_excess; - } + return true; + }); } if (middle_end < range_end_exclusive) { - const ScanResult scan = scan_range(middle_end, range_end_exclusive); - const std::int64_t candidate = - value + (find_min ? scan.min_value : scan.max_value); - consider_point(candidate, - find_min ? scan.min_position : scan.max_position); + if (find_min) { + const MinScanResult scan = + scan_min_range(middle_end, range_end_exclusive); + consider_point(value + scan.min_value, scan.min_position); + } else { + const ScanResult scan = scan_range(middle_end, range_end_exclusive); + consider_point(value + scan.max_value, scan.max_position); + } } if (best_is_node) { @@ -1610,14 +1801,12 @@ class RmMBTree : public RmMBase> { const std::size_t middle_end = std::max(middle_begin, last_full_block_exclusive * kBlockBits); if (middle_begin < middle_end) { - Cover cover; - collect_cover(middle_begin, middle_end, cover); - for (std::size_t i = 0; i < cover.size; ++i) { - const NodeRef& node = cover.nodes[i]; - Summary summary = summary_at(node.level, node.index); + for_each_cover_node(middle_begin, middle_end, [&](NodeRef node) { + const Summary summary = summary_at(node.level, node.index); consider(value + summary.min_excess, summary.min_count); value += summary.block_excess; - } + return true; + }); } if (middle_end < range_end_exclusive) { @@ -1665,20 +1854,25 @@ class RmMBTree : public RmMBase> { const std::size_t middle_end = std::max(middle_begin, last_full_block_exclusive * kBlockBits); if (middle_begin < middle_end) { - Cover cover; - collect_cover(middle_begin, middle_end, cover); - for (std::size_t i = 0; i < cover.size; ++i) { - const NodeRef& node = cover.nodes[i]; - Summary summary = summary_at(node.level, node.index); + bool found_node = false; + std::size_t selected = npos; + for_each_cover_node(middle_begin, middle_end, [&](NodeRef node) { + const Summary summary = summary_at(node.level, node.index); const std::int64_t candidate = value + summary.min_excess; if (candidate == target) { if (rank <= summary.min_count) { - return descend_qth_min(node.level, node.index, target - value, - rank); + selected = + descend_qth_min(node.level, node.index, target - value, rank); + found_node = true; + return false; } rank -= summary.min_count; } value += summary.block_excess; + return true; + }); + if (found_node) { + return selected; } } @@ -1693,49 +1887,74 @@ class RmMBTree : public RmMBase> { } /** - * @brief Decompose an aligned block interval into summary nodes. - * @details Produces a left-to-right cover of `[begin, end)` using the largest - * summary nodes available. Both boundaries must be aligned to block size. - * @param begin Inclusive bit start, aligned to `kBlockBits`. - * @param end Exclusive bit end, aligned to `kBlockBits`. - * @param out Destination fixed-size cover buffer. + * @brief Visit an aligned block interval as summary nodes. + * @details Produces the same left-to-right cover as the previous materialized + * cover helper, but streams nodes into @p callback and only keeps the right + * boundary stack needed to preserve bit order. The callback should return + * false to stop early. */ - void collect_cover(std::size_t begin, std::size_t end, Cover& out) const { + template + bool for_each_cover_node(std::size_t begin, + std::size_t end, + Callback&& callback) const { if (begin >= end || total_levels() == 0 || (begin % kBlockBits) != 0 || (end % kBlockBits) != 0) { - return; + return true; } - Cover right_cover; + std::array right_nodes; + std::size_t right_size = 0; + std::size_t emitted = 0; std::size_t level = 0; std::size_t left = begin / kBlockBits; std::size_t right = end / kBlockBits; + auto emit = [&](NodeRef node) { + if (emitted >= kMaxCoverNodes) { + return true; + } + ++emitted; + return callback(node); + }; + + auto push_right = [&](NodeRef node) { + if (right_size < right_nodes.size()) { + right_nodes[right_size++] = node; + } + }; + while (left < right) { if (!has_parent_level(level)) { for (std::size_t index = left; index < right; ++index) { - out.push({level, index}); + if (!emit({level, index})) { + return false; + } } break; } const std::size_t fanout = fanout_to_parent(level); while (left < right && (left % fanout) != 0) { - out.push({level, left}); + if (!emit({level, left})) { + return false; + } ++left; } while (left < right && (right % fanout) != 0) { --right; - right_cover.push({level, right}); + push_right({level, right}); } left /= fanout; right /= fanout; ++level; } - while (right_cover.size > 0) { - out.push(right_cover.nodes[--right_cover.size]); + while (right_size > 0) { + if (!emit(right_nodes[--right_size])) { + return false; + } } + return true; } /** @@ -1927,6 +2146,35 @@ class RmMBTree : public RmMBase> { return result; } + /** + * @brief Byte-accelerated minimum-only scan of an arbitrary bit interval. + * @details Used by range-min position/value queries that do not need maximum + * fields or minimum multiplicity. + */ + MinScanResult scan_min_range(std::size_t begin, std::size_t end) const { + MinScanResult result; + const auto& lut = byte_lut(); + while (begin < end && (begin & 7) != 0) { + append_scanned_min_bit(result, begin); + ++begin; + } + while (begin + 8 <= end) { + const ByteAgg& byte = lut[get_byte(begin)]; + const std::int64_t min_candidate = result.block_excess + byte.min_excess; + if (min_candidate < result.min_value) { + result.min_value = min_candidate; + result.min_position = begin + byte.pos_first_min; + } + result.block_excess += byte.block_excess; + begin += 8; + } + while (begin < end) { + append_scanned_min_bit(result, begin); + ++begin; + } + return result; + } + /** * @brief Append one bit to an incremental range scan. * @details Updates total excess, min/max values, first min/max positions, and @@ -1949,6 +2197,18 @@ class RmMBTree : public RmMBase> { } } + /** + * @brief Append one bit to an incremental minimum-only range scan. + */ + void append_scanned_min_bit(MinScanResult& result, + std::size_t position) const { + result.block_excess += bit(position) ? 1 : -1; + if (result.block_excess < result.min_value) { + result.min_value = result.block_excess; + result.min_position = position; + } + } + /** * @brief Return the byte starting at a byte-aligned bit position. * @details Reads eight bits from the backing span by shifting the containing diff --git a/include/pixie/memory_usage.h b/include/pixie/memory_usage.h new file mode 100644 index 0000000..9be316d --- /dev/null +++ b/include/pixie/memory_usage.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include + +namespace pixie { + +/** + * @brief Return bytes reserved by a vector's heap buffer. + * + * @details This intentionally uses capacity rather than size because the goal + * is to report practical owned memory after construction. The vector control + * block itself is counted by the owning object's `sizeof(*this)`. + */ +template +std::size_t vector_capacity_bytes(const std::vector& values) { + return values.capacity() * sizeof(T); +} + +/** + * @brief Concept for types that expose total owned memory usage in bytes. + */ +template +concept HasMemoryUsageBytes = requires(const T& value) { + { value.memory_usage_bytes() } -> std::convertible_to; +}; + +/** + * @brief Return heap bytes owned below an inline nested object. + * + * @details `outer.sizeof(*this)` already counts the inline nested object + * storage. This helper subtracts that inline storage from the nested object's + * total memory usage and leaves only buffers owned below it. + */ +template +std::size_t nested_owned_memory_bytes(const T& value) { + const std::size_t total = value.memory_usage_bytes(); + return total > sizeof(T) ? total - sizeof(T) : 0; +} + +/** + * @brief Return heap bytes owned below an engaged optional nested object. + */ +template +std::size_t optional_nested_owned_memory_bytes(const std::optional& value) { + return value.has_value() ? nested_owned_memory_bytes(*value) : 0; +} + +} // namespace pixie diff --git a/include/pixie/rmq.h b/include/pixie/rmq.h new file mode 100644 index 0000000..4e16f14 --- /dev/null +++ b/include/pixie/rmq.h @@ -0,0 +1,148 @@ +#pragma once + +// clang-format off +/* + * RMQ benchmark snapshot, 2026-06-15. + * + * Baseline command: + * ./build/release/bench_rmq + * --benchmark_filter='^(rmq_sparse_table|rmq_segment_tree|rmq_hybrid_btree|rmq_cartesian_rmm|rmq_cartesian_hybrid_btree|rmq_sdsl_sct|rmq_build_sparse_table|rmq_build_segment_tree|rmq_build_hybrid_btree|rmq_build_cartesian_rmm|rmq_build_cartesian_hybrid_btree|rmq_build_sdsl_sct)/' + * --benchmark_min_time=0.20s + * --benchmark_repetitions=10 + * --benchmark_report_aggregates_only=true + * + * Focused CartesianHybridBTree refresh for N=2^28 and N=2^30: + * ./build/release/bench_rmq + * --benchmark_filter='^(rmq_build_cartesian_hybrid_btree/(268435456|1073741824)|rmq_cartesian_hybrid_btree/(268435456|1073741824)/)' + * --benchmark_repetitions=10 + * --benchmark_report_aggregates_only=true + * + * The focused refresh used current benchmark registrations: 0.2s warmup and + * 2.0s minimum time. + * + * Tables report CPU mean across 10 repetitions. "max width" is the benchmark's + * maximum sampled query width. Sparse-table rows are intentionally not + * registered above 2^22. Rows above 2^26 are focused large-RMQ measurements; + * unavailable columns are marked with "-". + * + * Query CPU time. + * + * | N | max width | sparse table (ns) | segment tree (ns) | hybrid btree (ns) | cartesian rmm (ns) | cartesian hybrid (ns) | sdsl sct (ns) | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^10 | 64 | 17.2 | 51.3 | 65.8 | 174.2 | 96.1 | 136.9 | + * | 2^10 | 2^10 | 24.0 | 76.4 | 54.0 | 576.2 | 145.0 | 245.9 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^14 | 64 | 19.0 | 54.1 | 44.4 | 207.0 | 108.4 | 166.1 | + * | 2^14 | 4096 | 20.1 | 103.8 | 63.6 | 777.2 | 105.7 | 512.3 | + * | 2^14 | 2^14 | 18.1 | 114.9 | 46.1 | 976.5 | 66.1 | 612.8 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^18 | 64 | 36.3 | 78.1 | 51.8 | 201.2 | 116.1 | 187.0 | + * | 2^18 | 4096 | 35.7 | 148.7 | 79.7 | 874.1 | 220.7 | 655.6 | + * | 2^18 | 2^18 | 27.8 | 196.8 | 23.3 | 1255.2 | 37.9 | 838.7 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^22 | 64 | 89.3 | 170.5 | 128.9 | 220.6 | 125.9 | 204.4 | + * | 2^22 | 4096 | 73.2 | 310.2 | 131.8 | 882.9 | 239.3 | 750.4 | + * | 2^22 | 2^18 | 61.9 | 461.6 | 33.2 | 1361.6 | 53.0 | 1107.3 | + * | 2^22 | 2^22 | 57.3 | 507.1 | 17.9 | 1460.8 | 20.5 | 962.6 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^24 | 64 | - | 193.9 | 169.9 | 336.3 | 184.1 | 274.9 | + * | 2^24 | 4096 | - | 365.6 | 155.9 | 1023.5 | 291.8 | 931.9 | + * | 2^24 | 2^18 | - | 497.2 | 43.1 | 1493.6 | 74.1 | 1276.3 | + * | 2^24 | 2^22 | - | 603.4 | 20.8 | 1659.0 | 22.2 | 1287.3 | + * | 2^24 | 2^24 | - | 574.9 | 17.5 | 1637.9 | 20.0 | 1251.4 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^26 | 64 | - | 250.4 | 251.0 | 359.3 | 289.3 | 807.8 | + * | 2^26 | 4096 | - | 450.0 | 243.5 | 1243.7 | 465.4 | 1389.8 | + * | 2^26 | 2^18 | - | 678.1 | 77.0 | 1685.7 | 121.2 | 2113.1 | + * | 2^26 | 2^22 | - | 697.8 | 23.6 | 2016.6 | 31.2 | 1990.0 | + * | 2^26 | 2^26 | - | 698.8 | 17.5 | 1844.8 | 19.0 | 1832.5 | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^28 | 64 | - | - | 300.0 | 471.1 | 451.8 | - | + * | 2^28 | 4096 | - | - | 372.6 | 1421.0 | 803.9 | - | + * | 2^28 | 2^18 | - | - | 224.9 | 2084.5 | 416.6 | - | + * | 2^28 | 2^22 | - | - | 56.9 | 2095.4 | 77.2 | - | + * | 2^28 | 2^26 | - | - | 21.2 | 2301.6 | 31.7 | - | + * | 2^28 | 2^28 | - | - | 21.3 | 2515.1 | 19.0 | - | + * | -----: | --------: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^30 | 64 | - | - | 334.2 | 715.4 | 656.2 | - | + * | 2^30 | 4096 | - | - | 408.5 | 2061.2 | 1006.4 | - | + * | 2^30 | 2^18 | - | - | 557.7 | 2575.8 | 927.3 | - | + * | 2^30 | 2^22 | - | - | 145.5 | 2703.3 | 258.8 | - | + * | 2^30 | 2^26 | - | - | 45.5 | 2826.7 | 82.3 | - | + * | 2^30 | 2^30 | - | - | 30.4 | 3110.2 | 28.5 | - | + * + * Build CPU time. Sparse-table build rows are intentionally not registered + * above 2^22. + * + * | N | sparse table (ms) | segment tree (ms) | hybrid btree (ms) | cartesian rmm (ms) | cartesian hybrid (ms) | sdsl sct (ms) | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^10 | 0.007 | 0.001 | 0.002 | 0.014 | 0.013 | 0.007 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^14 | 1.232 | 0.014 | 0.028 | 0.137 | 0.115 | 0.218 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^18 | 30.883 | 0.279 | 0.453 | 3.372 | 2.980 | 2.556 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^22 | 715.861 | 65.124 | 8.960 | 55.726 | 49.400 | 41.505 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^24 | - | 276.132 | 35.264 | 225.624 | 200.655 | 170.930 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^26 | - | 1281.908 | 138.719 | 891.819 | 779.080 | 663.442 | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^28 | - | - | 586.827 | 3730.628 | 3347.897 | - | + * | -----: | ----------------: | ----------------: | ----------------: | -----------------: | --------------------: | ------------: | + * | 2^30 | - | - | 2324.537 | 15002.269 | 16070.000 | - | + * + * Owned auxiliary memory. Benchmarks use 64-bit signed integer values and + * 64-bit indexes. The external input values are not owned by RMQ indexes and + * are excluded. CartesianRmM does not currently expose an owned-memory counter. + * + * | N | sparse table (MiB) | segment tree (MiB) | hybrid btree (MiB) | cartesian rmm (MiB) | cartesian hybrid (MiB) | sdsl sct (MiB) | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^10 | 0.071 | 0.016 | 0.001 | - | 0.014 | 0.001 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^14 | 1.626 | 0.250 | 0.003 | - | 0.018 | 0.005 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^18 | 34.001 | 4.000 | 0.036 | - | 0.082 | 0.079 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^22 | 672.001 | 64.000 | 0.606 | - | 1.141 | 1.262 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^24 | - | 256.000 | 2.483 | - | 4.585 | 5.051 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^26 | - | 1024.000 | 10.182 | - | 18.551 | 20.224 | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^28 | - | - | 35.102 | - | 68.411 | - | + * | -----: | -----------------: | -----------------: | -----------------: | ------------------: | ---------------------: | -------------: | + * | 2^30 | - | - | 134.783 | - | 267.979 | - | + * + * Owned auxiliary memory normalized by indexed value count (bits/value). + * + * | N | sparse table | segment tree | hybrid btree | cartesian rmm | cartesian hybrid | sdsl sct | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^10 | 579.188 | 128.438 | 4.500 | - | 112.938 | 7.312 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^14 | 832.262 | 128.027 | 1.293 | - | 8.977 | 2.770 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^18 | 1088.020 | 128.002 | 1.162 | - | 2.629 | 2.534 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^22 | 1344.002 | 128.000 | 1.211 | - | 2.281 | 2.524 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^24 | - | 128.000 | 1.242 | - | 2.293 | 2.526 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^26 | - | 128.000 | 1.273 | - | 2.319 | 2.528 | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^28 | - | - | 1.097 | - | 2.138 | - | + * | -----: | -----------: | -----------: | -----------: | ------------: | ---------------: | -------: | + * | 2^30 | - | - | 1.053 | - | 2.094 | - | + */ +// clang-format on + +#include +#include +#include +#include +#include +#include + +#ifdef SDSL_SUPPORT +#include +#endif diff --git a/include/pixie/rmq/cartesian_hybrid_btree.h b/include/pixie/rmq/cartesian_hybrid_btree.h new file mode 100644 index 0000000..5584a1b --- /dev/null +++ b/include/pixie/rmq/cartesian_hybrid_btree.h @@ -0,0 +1,1810 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +namespace detail { + +/** + * @brief HybridBTree-style RMQ backend for ±1 depth sequences. + * + * @details The input words encode adjacent depth deltas: bit 1 means +1 and + * bit 0 means -1. The indexed sequence has `depth_count` prefix positions, so + * the bit sequence has `depth_count - 1` deltas. Public ranges are half-open + * over depth positions, and ties return the smaller depth position. + * + * This backend mirrors the high-level query shape of `HybridBTree`: depth + * positions are split into configurable-size leaves, leaves are grouped into a + * B-tree, and every internal node stores a local Cartesian/BP selector over the + * minima of its immediate children. Middle levels are fixed at 192 child slots: + * their selector uses 384 BP bits and reserves the remaining 128 bits for the + * embedded subtree minimum position/depth. A configurable number of top levels + * additionally keep sparse tables over child-minimum slots. Unlike value + * `HybridBTree`, leaves do not store another local Cartesian selector or + * cached minima. Leaf minima are recomputed from the original BP delta bits + * with the 128-bit excess primitives from `bits.h`, using rank support to + * recover the absolute base depth at the leaf start. + * + * @tparam Index Unsigned integer type used for stored positions. + * @tparam LeafSize Number of depth positions per low-level leaf; must be a + * multiple of 512. + * @tparam UseHighSparseLayout Whether top levels use sparse tables instead of + * local BP selectors. + * @tparam HighSparseLayoutLevels Number of top tree levels using that sparse + * layout when enabled. + */ +template +class HybridBTreePlusMinusOne { + public: + static_assert(std::is_unsigned_v, + "HybridBTreePlusMinusOne index type must be unsigned"); + static_assert(LeafSize != 0 && LeafSize % 512 == 0, + "HybridBTreePlusMinusOne leaf size must be a positive " + "multiple of 512"); + static_assert(!UseHighSparseLayout || HighSparseLayoutLevels > 0, + "HybridBTreePlusMinusOne high sparse layout must cover " + "at least one level when enabled"); + + static constexpr std::size_t npos = std::numeric_limits::max(); + static constexpr Index invalid_index = std::numeric_limits::max(); + static constexpr std::size_t kLeafSize = LeafSize; + static constexpr std::size_t kHighLevelFanout = 256; + static constexpr std::size_t kMiddleFanout = 192; + static constexpr bool kUseHighSparseLayout = UseHighSparseLayout; + static constexpr std::size_t kHighSparseLayoutLevels = HighSparseLayoutLevels; + + /** + * @brief Construct an empty ±1 RMQ index. + */ + HybridBTreePlusMinusOne() = default; + + /** + * @brief Build a ±1 RMQ index over external packed delta bits. + * + * @param bits Little-endian packed delta bits. + * @param depth_count Number of indexed depth positions. + * @throws std::length_error if @p Index cannot represent all positions. + * @throws std::invalid_argument if @p bits is shorter than required. + */ + HybridBTreePlusMinusOne(std::span bits, + std::size_t depth_count) { + build(bits, depth_count); + } + + /** + * @brief Build a ±1 RMQ index over external packed bits and rank support. + * + * @details The supplied rank index is non-owning and must outlive this RMQ + * object. This is used by CartesianHybridBTree to reuse its BP rank/select + * index. + */ + HybridBTreePlusMinusOne(std::span bits, + std::size_t depth_count, + const BitVector& rank_index) { + build(bits, depth_count, rank_index); + } + + /** + * @brief Rebuild this index over external packed delta bits. + */ + void build(std::span bits, std::size_t depth_count) { + input_bits_ = bits; + depth_count_ = depth_count; + external_rank_index_ = nullptr; + build(); + } + + /** + * @brief Rebuild this index using non-owning rank support for the same bits. + */ + void build(std::span bits, + std::size_t depth_count, + const BitVector& rank_index) { + input_bits_ = bits; + depth_count_ = depth_count; + external_rank_index_ = &rank_index; + build(); + } + + /** + * @brief Return the number of indexed depth positions. + */ + std::size_t size() const { return depth_count_; } + + /** + * @brief Whether the indexed depth sequence is empty. + */ + bool empty() const { return depth_count_ == 0; } + + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this ±1 RMQ object and all selector/metadata buffers. The + * external packed delta bits are not owned and are excluded. + */ + std::size_t memory_usage_bytes() const { + std::size_t bytes = sizeof(*this); + if (external_rank_index_ == nullptr) { + bytes += pixie::optional_nested_owned_memory_bytes(owned_rank_index_); + } + bytes += pixie::vector_capacity_bytes(internal_selectors_); + bytes += pixie::vector_capacity_bytes(internal_min_positions_); + bytes += pixie::vector_capacity_bytes(internal_min_depths_); + bytes += pixie::vector_capacity_bytes(high_child_metadata_); + bytes += pixie::vector_capacity_bytes(high_sparse_min_slots_); + bytes += pixie::vector_capacity_bytes(internal_level_offsets_); + bytes += pixie::vector_capacity_bytes(min_summary_level_offsets_); + bytes += pixie::vector_capacity_bytes(high_level_offsets_); + bytes += pixie::vector_capacity_bytes(level_sizes_); + bytes += pixie::vector_capacity_bytes(level_position_spans_); + bytes += pixie::vector_capacity_bytes(level_fanouts_); + return bytes; + } + + /** + * @brief Return the first minimum depth position in [@p left, @p right). + * + * @details Empty or invalid ranges return `npos`. Equal depths return the + * smaller position. + */ + std::size_t arg_min(std::size_t left, std::size_t right) const { + if (left >= right || right > depth_count_ || level_sizes_.empty()) { + return npos; + } + if (left + 1 == right) { + return left; + } + + const std::size_t root_level = level_count() - 1; + if (left == 0 && right == depth_count_) { + return subtree_min_candidate(root_level, 0).position; + } + + const std::size_t left_leaf = leaf_for_position(left); + const std::size_t right_leaf = leaf_for_position(right - 1); + if (left_leaf == right_leaf) { + return leaf_range_min(left_leaf, left, right).position; + } + + const auto [level, node] = covering_node(left_leaf, right_leaf); + return query_node(level, node, left, right).position; + } + + /** + * @brief Return the one-based rank-th zero delta bit position. + * + * @details This is the select operation needed by the Cartesian wrapper to + * map value endpoints to close-parenthesis positions. The returned position + * is a delta-bit position in `[0, size() - 1)`, or `npos` when @p rank is out + * of range. + */ + std::size_t select0(std::size_t rank) const { + if (rank == 0 || depth_count_ == 0 || rank_index_or_null() == nullptr) { + return npos; + } + const std::size_t delta_count = depth_count_ - 1; + const std::size_t position = rank_index().select0(rank); + return position < delta_count ? position : npos; + } + + private: + static constexpr std::size_t kLeafWords = LeafSize / 64; + static constexpr std::size_t kLeafChunks = LeafSize / 128; + static constexpr std::size_t kSelectorEntries = 256; + static constexpr std::size_t kSelectorBits = 2 * kSelectorEntries; + static constexpr std::size_t kSelectorWords = kSelectorBits / 64; + static constexpr std::size_t kEmbeddedSummaryWords = 2; + static constexpr std::size_t kEmbeddedSummaryBits = + 64 * kEmbeddedSummaryWords; + static constexpr std::size_t kEmbeddedSummaryMaxEntries = + (kSelectorBits - kEmbeddedSummaryBits) / 2; + static constexpr std::size_t kEmbeddedSummaryPositionWord = + kSelectorWords - 2; + static constexpr std::size_t kEmbeddedSummaryDepthWord = kSelectorWords - 1; + static constexpr std::size_t kHighSparseTableLevels = + static_cast(std::bit_width(kHighLevelFanout)); + static constexpr std::size_t kHighSparseSlotsPerNode = + kHighSparseTableLevels * kHighLevelFanout; + static constexpr bool kInvalidIndexEqualsNpos = + static_cast(invalid_index) == npos; + static_assert(kEmbeddedSummaryMaxEntries == 192); + static_assert(kMiddleFanout == kEmbeddedSummaryMaxEntries); + static_assert(sizeof(std::size_t) <= sizeof(std::uint64_t)); + + struct DepthCandidate { + std::size_t position = npos; + std::int64_t depth = std::numeric_limits::max(); + }; + + struct HighChildMetadata { + std::size_t position_begin = 0; + std::size_t position_end = 0; + Index min_position = invalid_index; + std::int64_t min_depth = std::numeric_limits::max(); + }; + + class alignas(64) Bp512Selector { + public: + /** + * @brief Construct an empty packed BP selector. + */ + Bp512Selector() = default; + + /** + * @brief Build a stable local Cartesian-tree BP selector over entries. + * + * @details The comparator must return whether the left slot is strictly + * better than the right slot. Equal minima stay stable through the same + * right-to-left construction rule used by the value Cartesian tree. + */ + template + void build(std::size_t entry_count, EntryLess entry_less) { + if (entry_count > kSelectorEntries) { + throw std::length_error( + "HybridBTreePlusMinusOne local selector too large"); + } + + bp_bits_.fill(0); + if (entry_count == 0) { + return; + } + + std::array stack{}; + std::size_t stack_size = 0; + std::size_t write_position = 2 * entry_count; + + for (std::size_t i = entry_count; i-- > 0;) { + while (stack_size != 0 && !entry_less(stack[stack_size - 1], i)) { + --stack_size; + prepend_bp_bit(write_position, true); + } + stack[stack_size++] = static_cast(i); + prepend_bp_bit(write_position, false); + } + + while (write_position != 0) { + prepend_bp_bit(write_position, true); + } + } + + /** + * @brief Return the first minimum slot in a local entry range. + */ + std::size_t arg_min(std::size_t slot_left, + std::size_t slot_right, + std::size_t entry_count) const { + if (slot_left >= slot_right || slot_right > entry_count || + entry_count > kSelectorEntries) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t bit_count = 2 * entry_count; + const std::size_t first_close = close_position(slot_left); + const std::size_t last_close = close_position(slot_right - 1); + if (first_close > last_close || last_close >= bit_count) { + return npos; + } + + const std::size_t shifted_min = + depth_arg_min(first_close + 1, last_close + 2, bit_count); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + + const std::size_t zero_rank = rank0_at(shifted_min, bit_count); + if (zero_rank == 0) { + return npos; + } + const std::size_t entry = zero_rank - 1; + return entry < entry_count ? entry : npos; + } + + /** + * @brief Store a node's subtree minimum in the unused selector tail. + * + * @details This is valid for nodes with at most 192 entries. Their BP + * sequence uses at most the first 384 bits, and all local selector queries + * are bounded by `2 * entry_count`, so the final two words are available + * for node metadata. + */ + void set_embedded_min_summary(std::size_t position, std::int64_t depth) { + bp_bits_[kEmbeddedSummaryPositionWord] = + static_cast(position); + bp_bits_[kEmbeddedSummaryDepthWord] = std::bit_cast(depth); + } + + /** + * @brief Return the embedded subtree-minimum position. + */ + std::size_t embedded_min_position() const { + return static_cast(bp_bits_[kEmbeddedSummaryPositionWord]); + } + + /** + * @brief Return the embedded subtree-minimum depth. + */ + std::int64_t embedded_min_depth() const { + return std::bit_cast(bp_bits_[kEmbeddedSummaryDepthWord]); + } + + private: + /** + * @brief Prepend one BP bit while building the sequence right-to-left. + */ + void prepend_bp_bit(std::size_t& write_position, bool bit) { + --write_position; + if (bit) { + bp_bits_[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); + } + } + + /** + * @brief Return the BP close position for a local entry slot. + */ + std::size_t close_position(std::size_t slot) const { + return select0_512(bp_bits_.data(), slot); + } + + /** + * @brief Count zero bits before @p position in the raw BP sequence. + */ + std::size_t rank0_at(std::size_t position, std::size_t bit_count) const { + position = std::min(position, bit_count); + return position - rank_512(bp_bits_.data(), position); + } + + /** + * @brief Return open-minus-close excess before a raw BP position. + */ + int prefix_excess(std::size_t position) const { + position = std::min(position, kSelectorBits); + const std::size_t ones = rank_512(bp_bits_.data(), position); + return static_cast(2 * ones) - static_cast(position); + } + + /** + * @brief Return the minimum BP-depth position in a selector depth range. + */ + std::size_t depth_arg_min(std::size_t left, + std::size_t right, + std::size_t bit_count) const { + const std::size_t depth_count = bit_count + 1; + if (left >= right || right > depth_count) { + return npos; + } + + std::size_t position = left; + int best_depth = prefix_excess(position); + std::size_t best_position = position; + + while (position < right) { + const std::size_t chunk_begin = (position / 128) * 128; + const std::size_t local_left = position - chunk_begin; + const std::size_t local_right = + std::min(right - 1, chunk_begin + 128) - chunk_begin; + + int candidate_depth; + std::size_t candidate_position; + if (chunk_begin >= bit_count) { + candidate_depth = prefix_excess(bit_count); + candidate_position = bit_count; + } else { + const std::size_t word = chunk_begin >> 6; + const ExcessResult result = + excess_min_128(bp_bits_.data() + word, local_left, local_right); + candidate_depth = prefix_excess(chunk_begin) + result.min_excess; + candidate_position = chunk_begin + result.offset; + } + + if (candidate_depth < best_depth) { + best_depth = candidate_depth; + best_position = candidate_position; + } + + position = chunk_begin + local_right + 1; + } + + return best_position; + } + + std::array bp_bits_{}; + }; + + static_assert(sizeof(Bp512Selector) == 64); + + /** + * @brief Return whether a stored position is one of the missing sentinels. + */ + bool missing_position(std::size_t position) const { + if constexpr (kInvalidIndexEqualsNpos) { + return position == npos; + } else { + return position == npos || + position == static_cast(invalid_index); + } + } + + /** + * @brief Choose the smaller-depth candidate, breaking ties by position. + */ + DepthCandidate better_candidate(DepthCandidate left, + DepthCandidate right) const { + if (missing_position(left.position)) { + return right; + } + if (missing_position(right.position)) { + return left; + } + if (right.depth < left.depth) { + return right; + } + if (left.depth < right.depth) { + return left; + } + return right.position < left.position ? right : left; + } + + /** + * @brief Return whether @p left is strictly better than @p right. + */ + bool strictly_better_candidate(DepthCandidate left, + DepthCandidate right) const { + if (missing_position(left.position)) { + return false; + } + if (missing_position(right.position)) { + return true; + } + if (left.depth != right.depth) { + return left.depth < right.depth; + } + return left.position < right.position; + } + + /** + * @brief Build rank support, internal selectors, and top sparse tables. + */ + void build() { + owned_rank_index_.reset(); + internal_selectors_.clear(); + internal_min_positions_.clear(); + internal_min_depths_.clear(); + high_child_metadata_.clear(); + high_sparse_min_slots_.clear(); + internal_level_offsets_.clear(); + min_summary_level_offsets_.clear(); + high_level_offsets_.clear(); + level_sizes_.clear(); + level_position_spans_.clear(); + level_fanouts_.clear(); + high_level_begin_ = std::numeric_limits::max(); + + if (depth_count_ == 0) { + return; + } + if (depth_count_ > static_cast(invalid_index)) { + throw std::length_error( + "HybridBTreePlusMinusOne index type is too small"); + } + if (depth_count_ > + static_cast(std::numeric_limits::max())) { + throw std::length_error( + "HybridBTreePlusMinusOne depth range is too large"); + } + if (input_bits_.size() < (depth_count_ - 1 + 63) / 64) { + throw std::invalid_argument( + "HybridBTreePlusMinusOne bit span is too small"); + } + + const std::size_t delta_count = depth_count_ - 1; + if (external_rank_index_ == nullptr) { + owned_rank_index_.emplace(input_bits_, delta_count, + BitVector::SelectSupport::kSelect0); + } else if (external_rank_index_->size() < delta_count) { + throw std::invalid_argument( + "HybridBTreePlusMinusOne external rank index is too small"); + } + + initialize_layout((depth_count_ + LeafSize - 1) / LeafSize); + for (std::size_t level = 1; level < level_count(); ++level) { + for (std::size_t node = 0; node < level_sizes_[level]; ++node) { + build_internal_node(level, node); + } + } + } + + /** + * @brief Compute B-tree level sizes, fanouts, and flat storage offsets. + */ + void initialize_layout(std::size_t leaf_count) { + level_sizes_.push_back(leaf_count); + level_position_spans_.push_back(LeafSize); + level_fanouts_.push_back(0); + + std::size_t current_count = leaf_count; + std::size_t current_span = LeafSize; + while (current_count > kHighLevelFanout * kHighLevelFanout) { + level_fanouts_.push_back(kMiddleFanout); + current_count = ceil_div(current_count, kMiddleFanout); + current_span = saturating_product(current_span, kMiddleFanout); + level_sizes_.push_back(current_count); + level_position_spans_.push_back(current_span); + } + while (current_count > 1) { + level_fanouts_.push_back(kHighLevelFanout); + current_count = ceil_div(current_count, kHighLevelFanout); + current_span = saturating_product(current_span, kHighLevelFanout); + level_sizes_.push_back(current_count); + level_position_spans_.push_back(current_span); + } + internal_level_offsets_.assign(level_count(), 0); + min_summary_level_offsets_.assign(level_count(), npos); + high_level_offsets_.assign(level_count(), 0); + if (level_count() <= 1) { + return; + } + + std::size_t internal_count = 0; + for (std::size_t level = 1; level < level_count(); ++level) { + internal_level_offsets_[level] = internal_count; + internal_count += level_sizes_[level]; + } + internal_selectors_.resize(internal_count); + + if constexpr (UseHighSparseLayout) { + const std::size_t root_level = level_count() - 1; + std::size_t high_layout_levels = 0; + for (std::size_t level = root_level; + level > 0 && high_layout_levels < HighSparseLayoutLevels && + fanout_at_level(level) == kHighLevelFanout; + --level) { + ++high_layout_levels; + } + high_level_begin_ = high_layout_levels == 0 + ? level_count() + : root_level + 1 - high_layout_levels; + + std::size_t high_node_count = 0; + for (std::size_t level = high_level_begin_; level < level_count(); + ++level) { + high_level_offsets_[level] = high_node_count; + high_node_count += level_sizes_[level]; + } + high_child_metadata_.resize(high_node_count * kHighLevelFanout); + high_sparse_min_slots_.resize(high_node_count * kHighSparseSlotsPerNode); + } else { + high_level_begin_ = level_count(); + } + + std::size_t side_summary_count = 0; + for (std::size_t level = 1; level < level_count(); ++level) { + if (!level_embeds_min_summary(level)) { + min_summary_level_offsets_[level] = side_summary_count; + side_summary_count += level_sizes_[level]; + } + } + internal_min_positions_.resize(side_summary_count, invalid_index); + internal_min_depths_.resize(side_summary_count, + std::numeric_limits::max()); + } + + /** + * @brief Build one internal node selector and cached minimum. + */ + void build_internal_node(std::size_t level, std::size_t node) { + const std::size_t count = entry_count(level, node); + const std::size_t first_child = node * fanout_at_level(level); + const bool high_level = is_high_level(level); + const std::size_t high_flat = high_level ? high_flat_index(level, node) : 0; + + std::array child_minima{}; + for (std::size_t slot = 0; slot < count; ++slot) { + child_minima[slot] = subtree_min_candidate(level - 1, first_child + slot); + } + + if (high_level) { + for (std::size_t slot = 0; slot < count; ++slot) { + const std::size_t child = first_child + slot; + const DepthCandidate child_min = child_minima[slot]; + HighChildMetadata& metadata = + mutable_high_child_metadata_at(high_flat, slot); + metadata.position_begin = node_position_begin(level - 1, child); + metadata.position_end = node_position_end(level - 1, child); + metadata.min_position = static_cast(child_min.position); + metadata.min_depth = child_min.depth; + } + build_high_sparse_min_slots(level, node, count); + } + + Bp512Selector& selector = mutable_selector_at(level, node); + selector.build(count, [&](std::size_t left, std::size_t right) { + return strictly_better_candidate(child_minima[left], child_minima[right]); + }); + + const std::size_t slot = selector.arg_min(0, count, count); + const DepthCandidate minimum = child_minima[slot]; + if (level_embeds_min_summary(level)) { + selector.set_embedded_min_summary(minimum.position, minimum.depth); + } else { + const std::size_t flat = min_summary_flat_index(level, node); + internal_min_positions_[flat] = static_cast(minimum.position); + internal_min_depths_[flat] = minimum.depth; + } + } + + /** + * @brief Return `ceil(value / divisor)` for positive divisors. + */ + static std::size_t ceil_div(std::size_t value, std::size_t divisor) { + return (value + divisor - 1) / divisor; + } + + /** + * @brief Multiply two extents, saturating at `std::size_t` maximum. + */ + static std::size_t saturating_product(std::size_t left, std::size_t right) { + if (left != 0 && right > std::numeric_limits::max() / left) { + return std::numeric_limits::max(); + } + return left * right; + } + + /** + * @brief Read an input word or return zero past the available span. + */ + std::uint64_t word_or_zero(std::size_t word) const { + return word < input_bits_.size() ? input_bits_[word] : 0; + } + + /** + * @brief Load all delta words aligned to a leaf boundary. + */ + std::array leaf_bits(std::size_t leaf) const { + std::array bits{}; + const std::size_t first_word = leaf * kLeafWords; + for (std::size_t word = 0; word < kLeafWords; ++word) { + bits[word] = word_or_zero(first_word + word); + } + return bits; + } + + /** + * @brief Return the active BP rank/select support, if one exists. + */ + const BitVector* rank_index_or_null() const { + return external_rank_index_ != nullptr + ? external_rank_index_ + : (owned_rank_index_ ? &*owned_rank_index_ : nullptr); + } + + /** + * @brief Return the active BP rank/select support. + */ + const BitVector& rank_index() const { return *rank_index_or_null(); } + + /** + * @brief Return the absolute open-minus-close depth at a BP depth position. + */ + std::int64_t depth_at_position(std::size_t position) const { + const std::size_t delta_count = depth_count_ == 0 ? 0 : depth_count_ - 1; + position = std::min(position, delta_count); + const std::uint64_t ones = rank_index().rank(position); + return static_cast(ones) - + static_cast(position - ones); + } + + /** + * @brief Return the first minimum candidate inside one leaf local range. + */ + DepthCandidate scan_leaf_range_with_base(std::size_t leaf, + std::size_t left_offset, + std::size_t right_offset, + std::int64_t base_depth) const { + const std::size_t begin = node_position_begin(0, leaf); + const std::size_t count = entry_count(0, leaf); + if (count == 0 || left_offset > right_offset || left_offset >= count) { + return {}; + } + right_offset = std::min(right_offset, count - 1); + + const auto bits = leaf_bits(leaf); + DepthCandidate answer; + std::int64_t chunk_base_excess = 0; + for (std::size_t chunk = 0; chunk < kLeafChunks; ++chunk) { + const std::size_t chunk_begin = chunk * 128; + if (chunk_begin >= count || chunk_begin > right_offset) { + break; + } + + const std::size_t chunk_end = + std::min(count - 1, chunk_begin + 127); + if (left_offset > chunk_end) { + chunk_base_excess += prefix_excess_128(bits.data() + 2 * chunk, 128); + continue; + } + + const std::size_t local_left = + std::max(left_offset, chunk_begin) - chunk_begin; + const std::size_t local_right = + std::min(right_offset, chunk_end) - chunk_begin; + const ExcessResult result = + excess_min_128(bits.data() + 2 * chunk, local_left, local_right); + const std::size_t offset = chunk_begin + result.offset; + if (result.offset != npos && offset < count) { + answer = better_candidate( + answer, {begin + offset, + base_depth + chunk_base_excess + result.min_excess}); + } + + chunk_base_excess += prefix_excess_128(bits.data() + 2 * chunk, 128); + } + return answer; + } + + /** + * @brief Return the minimum candidate in a possibly partial leaf. + */ + DepthCandidate leaf_range_min(std::size_t leaf, + std::size_t left, + std::size_t right) const { + if (left >= right) { + return {}; + } + + const std::size_t begin = node_position_begin(0, leaf); + const std::size_t slot_left = left - begin; + const std::size_t slot_right = right - begin; + return scan_leaf_range_with_base(leaf, slot_left, slot_right - 1, + depth_at_position(begin)); + } + + /** + * @brief Find the lowest tree node that contains both endpoint leaves. + */ + std::pair covering_node( + std::size_t left_leaf, + std::size_t right_leaf) const { + std::size_t level = 0; + std::size_t left_node = left_leaf; + std::size_t right_node = right_leaf; + while (left_node != right_node) { + ++level; + const std::size_t fanout = fanout_at_level(level); + left_node /= fanout; + right_node /= fanout; + } + return {level, left_node}; + } + + /** + * @brief Return the leaf index containing a depth position. + */ + std::size_t leaf_for_position(std::size_t position) const { + return position / LeafSize; + } + + /** + * @brief Return the child index at @p child_level containing a position. + */ + std::size_t child_for_position(std::size_t child_level, + std::size_t position) const { + return position / level_position_spans_[child_level]; + } + + /** + * @brief Query a child-slot range of an internal node. + */ + DepthCandidate query_child_slots(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t left, + std::size_t right) const { + if (slot_left >= slot_right) { + return {}; + } + + const std::size_t count = entry_count(level, node); + const std::size_t slot = + slot_left + 1 == slot_right + ? slot_left + : selector_arg_min(level, node, slot_left, slot_right, count); + if (slot == npos) { + return {}; + } + + const std::size_t child_level = level - 1; + const std::size_t first_child = node * fanout_at_level(level); + const std::size_t child = first_child + slot; + const HighChildMetadata* high_children = + is_high_level(level) ? high_child_metadata_begin(level, node) : nullptr; + const DepthCandidate child_min = + high_children != nullptr ? high_child_min_candidate(level, node, slot) + : subtree_min_candidate(child_level, child); + const std::size_t child_begin = + high_children != nullptr ? high_children[slot].position_begin + : node_position_begin(child_level, child); + const std::size_t child_end = high_children != nullptr + ? high_children[slot].position_end + : node_position_end(child_level, child); + if ((left <= child_begin && child_end <= right) || + contains_position(left, right, child_min.position)) { + return child_min; + } + + const std::size_t last_slot = slot_right - 1; + const std::size_t left_child_begin = + high_children != nullptr + ? high_children[slot_left].position_begin + : node_position_begin(child_level, first_child + slot_left); + const std::size_t left_child_end = + high_children != nullptr + ? high_children[slot_left].position_end + : node_position_end(child_level, first_child + slot_left); + DepthCandidate answer = query_node(child_level, first_child + slot_left, + std::max(left, left_child_begin), + std::min(right, left_child_end)); + + if (slot_left != last_slot) { + const std::size_t right_child_begin = + high_children != nullptr + ? high_children[last_slot].position_begin + : node_position_begin(child_level, first_child + last_slot); + const std::size_t right_child_end = + high_children != nullptr + ? high_children[last_slot].position_end + : node_position_end(child_level, first_child + last_slot); + answer = better_candidate(answer, + query_node(child_level, first_child + last_slot, + std::max(left, right_child_begin), + std::min(right, right_child_end))); + } + + if (slot_left + 1 < last_slot) { + answer = better_candidate( + answer, + full_child_slot_range_min(level, node, slot_left + 1, last_slot)); + } + + return answer; + } + + /** + * @brief Return the best candidate among fully covered child slots. + */ + DepthCandidate full_child_slot_range_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right) const { + if (slot_left >= slot_right) { + return {}; + } + + const std::size_t slot = + slot_left + 1 == slot_right + ? slot_left + : selector_arg_min(level, node, slot_left, slot_right, + entry_count(level, node)); + if (slot == npos) { + return {}; + } + return child_min_candidate(level, node, slot); + } + + /** + * @brief Query a tree node for the minimum candidate in a depth range. + */ + DepthCandidate query_node(std::size_t level, + std::size_t node, + std::size_t left, + std::size_t right) const { + if (left >= right) { + return {}; + } + + const std::size_t begin = node_position_begin(level, node); + const std::size_t end = node_position_end(level, node); + if (left <= begin && end <= right) { + return subtree_min_candidate(level, node); + } + if (level == 0) { + return leaf_range_min(node, left, right); + } + + const std::size_t child_level = level - 1; + const std::size_t left_child = child_for_position(child_level, left); + const std::size_t right_child = child_for_position(child_level, right - 1); + const std::size_t first_child = node * fanout_at_level(level); + const std::size_t left_slot = left_child - first_child; + const std::size_t right_slot = right_child - first_child + 1; + return query_child_slots(level, node, left_slot, right_slot, left, right); + } + + /** + * @brief Return whether a position lies inside a half-open range. + */ + bool contains_position(std::size_t left, + std::size_t right, + std::size_t position) const { + return !missing_position(position) && left <= position && position < right; + } + + /** + * @brief Return the number of B-tree levels, including leaves. + */ + std::size_t level_count() const { return level_sizes_.size(); } + + /** + * @brief Return the number of entries in a leaf or child slots in a node. + */ + std::size_t entry_count(std::size_t level, std::size_t node) const { + if (level == 0) { + const std::size_t begin = node_position_begin(0, node); + return std::min(LeafSize, depth_count_ - begin); + } + const std::size_t first_child = node * fanout_at_level(level); + return std::min(fanout_at_level(level), + level_sizes_[level - 1] - first_child); + } + + /** + * @brief Return the first depth position covered by a node. + */ + std::size_t node_position_begin(std::size_t level, std::size_t node) const { + return node * level_position_spans_[level]; + } + + /** + * @brief Return one past the last depth position covered by a node. + */ + std::size_t node_position_end(std::size_t level, std::size_t node) const { + return std::min(depth_count_, node_position_begin(level, node) + + level_position_spans_[level]); + } + + /** + * @brief Return a node's cached subtree-minimum candidate. + */ + DepthCandidate subtree_min_candidate(std::size_t level, + std::size_t node) const { + if (level == 0) { + return leaf_range_min(node, node_position_begin(0, node), + node_position_end(0, node)); + } + if (level_embeds_min_summary(level)) { + const Bp512Selector& selector = selector_at(level, node); + return {selector.embedded_min_position(), selector.embedded_min_depth()}; + } + const std::size_t flat = min_summary_flat_index(level, node); + return {static_cast(internal_min_positions_[flat]), + internal_min_depths_[flat]}; + } + + /** + * @brief Return a child slot's subtree-minimum candidate. + */ + DepthCandidate child_min_candidate(std::size_t level, + std::size_t node, + std::size_t slot) const { + if (is_high_level(level)) { + return high_child_min_candidate(level, node, slot); + } + return subtree_min_candidate(level - 1, + node * fanout_at_level(level) + slot); + } + + /** + * @brief Return a high-node child candidate using cached metadata. + */ + DepthCandidate high_child_min_candidate(std::size_t level, + std::size_t node, + std::size_t slot) const { + const HighChildMetadata& metadata = + high_child_metadata_at(high_flat_index(level, node), slot); + return {static_cast(metadata.min_position), + metadata.min_depth}; + } + + /** + * @brief Return an immutable internal-node BP selector. + */ + const Bp512Selector& selector_at(std::size_t level, std::size_t node) const { + return internal_selectors_[internal_flat_index(level, node)]; + } + + /** + * @brief Return a mutable internal-node BP selector while building. + */ + Bp512Selector& mutable_selector_at(std::size_t level, std::size_t node) { + return internal_selectors_[internal_flat_index(level, node)]; + } + + /** + * @brief Run the appropriate local selector for an internal node. + */ + std::size_t selector_arg_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t count) const { + if constexpr (UseHighSparseLayout) { + if (is_high_level(level)) { + return high_sparse_arg_min(level, node, slot_left, slot_right, count); + } + } + return selector_at(level, node).arg_min(slot_left, slot_right, count); + } + + /** + * @brief Return whether a level uses the high-node layout. + */ + bool is_high_level(std::size_t level) const { + if constexpr (!UseHighSparseLayout) { + (void)level; + return false; + } + return level > 0 && level >= high_level_begin_ && level < level_count(); + } + + /** + * @brief Map an internal node to its flat storage index. + */ + std::size_t internal_flat_index(std::size_t level, std::size_t node) const { + return internal_level_offsets_[level] + node; + } + + /** + * @brief Return whether a level embeds subtree minima in selector tail bits. + */ + bool level_embeds_min_summary(std::size_t level) const { + return level > 0 && !is_high_level(level) && + fanout_at_level(level) <= kEmbeddedSummaryMaxEntries; + } + + /** + * @brief Map a non-embedded internal node to side summary storage. + */ + std::size_t min_summary_flat_index(std::size_t level, + std::size_t node) const { + return min_summary_level_offsets_[level] + node; + } + + /** + * @brief Return the fanout used to group children at a level. + */ + std::size_t fanout_at_level(std::size_t level) const { + return level_fanouts_[level]; + } + + /** + * @brief Map a high-level node to its flat high-node storage index. + */ + std::size_t high_flat_index(std::size_t level, std::size_t node) const { + return high_level_offsets_[level] + node; + } + + /** + * @brief Return the first child-metadata record for a high node. + */ + const HighChildMetadata* high_child_metadata_begin(std::size_t level, + std::size_t node) const { + return high_child_metadata_.data() + + high_flat_index(level, node) * kHighLevelFanout; + } + + /** + * @brief Return high-child metadata by flat high-node index and slot. + */ + const HighChildMetadata& high_child_metadata_at(std::size_t high_flat, + std::size_t slot) const { + return high_child_metadata_[high_flat * kHighLevelFanout + slot]; + } + + /** + * @brief Return mutable high-child metadata while building. + */ + HighChildMetadata& mutable_high_child_metadata_at(std::size_t high_flat, + std::size_t slot) { + return high_child_metadata_[high_flat * kHighLevelFanout + slot]; + } + + /** + * @brief Return mutable sparse-slot storage for one high node. + */ + std::uint8_t* mutable_high_sparse_min_slots_begin(std::size_t high_flat) { + return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; + } + + /** + * @brief Return sparse-slot storage for one high node. + */ + const std::uint8_t* high_sparse_min_slots_begin(std::size_t high_flat) const { + return high_sparse_min_slots_.data() + high_flat * kHighSparseSlotsPerNode; + } + + /** + * @brief Choose the better high-node child slot. + */ + std::size_t better_high_child_slot(std::size_t level, + std::size_t node, + std::size_t left_slot, + std::size_t right_slot) const { + const DepthCandidate left = + high_child_min_candidate(level, node, left_slot); + const DepthCandidate right = + high_child_min_candidate(level, node, right_slot); + return better_candidate(left, right).position == right.position ? right_slot + : left_slot; + } + + /** + * @brief Build sparse tables over high-node child minima. + */ + void build_high_sparse_min_slots(std::size_t level, + std::size_t node, + std::size_t count) { + const std::size_t high_flat = high_flat_index(level, node); + std::uint8_t* table = mutable_high_sparse_min_slots_begin(high_flat); + for (std::size_t slot = 0; slot < count; ++slot) { + table[slot] = static_cast(slot); + } + + for (std::size_t table_level = 1; table_level < kHighSparseTableLevels; + ++table_level) { + const std::size_t span = std::size_t{1} << table_level; + if (span > count) { + break; + } + const std::size_t half_span = span >> 1; + const std::uint8_t* previous = + table + (table_level - 1) * kHighLevelFanout; + std::uint8_t* current = table + table_level * kHighLevelFanout; + for (std::size_t slot = 0; slot + span <= count; ++slot) { + current[slot] = static_cast(better_high_child_slot( + level, node, previous[slot], previous[slot + half_span])); + } + } + } + + /** + * @brief Return the best high-node child slot in a slot range. + */ + std::size_t high_sparse_arg_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t count) const { + if (slot_left >= slot_right || slot_right > count) { + return npos; + } + const std::size_t length = slot_right - slot_left; + if (length == 1) { + return slot_left; + } + + const std::size_t high_flat = high_flat_index(level, node); + const std::size_t table_level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << table_level; + const std::uint8_t* table = + high_sparse_min_slots_begin(high_flat) + table_level * kHighLevelFanout; + return better_high_child_slot(level, node, table[slot_left], + table[slot_right - span]); + } + + std::span input_bits_; + std::size_t depth_count_ = 0; + std::optional owned_rank_index_; + const BitVector* external_rank_index_ = nullptr; + std::vector internal_selectors_; + std::vector internal_min_positions_; + std::vector internal_min_depths_; + std::vector high_child_metadata_; + std::vector high_sparse_min_slots_; + std::vector internal_level_offsets_; + std::vector min_summary_level_offsets_; + std::vector high_level_offsets_; + std::vector level_sizes_; + std::vector level_position_spans_; + std::vector level_fanouts_; + std::size_t high_level_begin_ = std::numeric_limits::max(); +}; + +} // namespace detail + +/** + * @brief Cartesian-tree value RMQ using HybridBTree-style LCA. + * + * @details This class follows the same public value-RMQ specification as the + * other value RMQ backends. It builds a stable Ferrada-Navarro BP + * Cartesian-tree encoding, uses `BitVector` for close-parenthesis rank/select, + * and delegates the BP-depth minimum query to + * `detail::HybridBTreePlusMinusOne`. The BP-depth backend keeps a configurable + * low-level leaf size, fixed 192-entry middle nodes with embedded minima, and + * fixed 256-entry high nodes. A single coarse value-level sparse table is + * checked first; it uses at least 4096-value blocks and grows the block width + * when needed so the top layer has at most 2^14 blocks. Wide queries whose + * padded block-cover minimum lies inside the requested range return from this + * top table without touching the global BP rank/select path. BP construction + * uses a succinct monotone bit-stack, preserving the same stable Cartesian-tree + * shape without an n-entry index stack. + * + * This implementation is included from `pixie/rmq.h` as the compact + * Cartesian-tree reduction backed by a HybridBTree-shaped ±1 RMQ index. + */ +template , + class Index = std::size_t, + std::size_t LeafSize = 512> +class CartesianHybridBTree + : public RmqBase, T> { + public: + static_assert(std::is_unsigned_v, + "CartesianHybridBTree index type must be unsigned"); + static_assert(LeafSize != 0 && LeafSize % 512 == 0, + "CartesianHybridBTree leaf size must be a positive " + "multiple of 512"); + + using Self = CartesianHybridBTree; + + static constexpr std::size_t npos = RmqBase::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + static constexpr std::size_t kMinTopSparseBlockSize = 4096; + static constexpr std::size_t kMaxTopSparseBlocks = std::size_t{1} << 14; + + /** + * @brief Construct an empty Cartesian-tree RMQ index. + */ + CartesianHybridBTree() = default; + + /** + * @brief Build a Cartesian-tree RMQ index over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values stay stable: the smaller index remains the first minimum. + */ + explicit CartesianHybridBTree(std::span values, + Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + /** + * @brief Copy an RMQ index and rebuild internal non-owning views. + */ + CartesianHybridBTree(const CartesianHybridBTree& other) + : values_(other.values_), + compare_(other.compare_), + bp_bits_(other.bp_bits_), + bp_bit_count_(other.bp_bit_count_), + top_sparse_candidates_(other.top_sparse_candidates_), + top_block_size_(other.top_block_size_), + top_block_count_(other.top_block_count_), + top_sparse_levels_(other.top_sparse_levels_) { + reset_bp_indexes(); + } + + /** + * @brief Copy-assign an RMQ index and rebuild internal non-owning views. + */ + CartesianHybridBTree& operator=(const CartesianHybridBTree& other) { + if (this == &other) { + return *this; + } + values_ = other.values_; + compare_ = other.compare_; + bp_bits_ = other.bp_bits_; + bp_bit_count_ = other.bp_bit_count_; + top_sparse_candidates_ = other.top_sparse_candidates_; + top_block_size_ = other.top_block_size_; + top_block_count_ = other.top_block_count_; + top_sparse_levels_ = other.top_sparse_levels_; + reset_bp_indexes(); + return *this; + } + + /** + * @brief Move an RMQ index and rebuild internal non-owning views. + */ + CartesianHybridBTree(CartesianHybridBTree&& other) noexcept + : values_(other.values_), + compare_(std::move(other.compare_)), + bp_bits_(std::move(other.bp_bits_)), + bp_bit_count_(other.bp_bit_count_), + top_sparse_candidates_(std::move(other.top_sparse_candidates_)), + top_block_size_(other.top_block_size_), + top_block_count_(other.top_block_count_), + top_sparse_levels_(other.top_sparse_levels_) { + other.values_ = std::span(); + other.bp_bit_count_ = 0; + other.top_block_size_ = kMinTopSparseBlockSize; + other.top_block_count_ = 0; + other.top_sparse_levels_ = 0; + reset_bp_indexes(); + } + + /** + * @brief Move-assign an RMQ index and rebuild internal non-owning views. + */ + CartesianHybridBTree& operator=(CartesianHybridBTree&& other) noexcept { + if (this == &other) { + return *this; + } + values_ = other.values_; + compare_ = std::move(other.compare_); + bp_bits_ = std::move(other.bp_bits_); + bp_bit_count_ = other.bp_bit_count_; + top_sparse_candidates_ = std::move(other.top_sparse_candidates_); + top_block_size_ = other.top_block_size_; + top_block_count_ = other.top_block_count_; + top_sparse_levels_ = other.top_sparse_levels_; + other.values_ = std::span(); + other.bp_bit_count_ = 0; + other.top_block_size_ = kMinTopSparseBlockSize; + other.top_block_count_ = 0; + other.top_sparse_levels_ = 0; + reset_bp_indexes(); + return *this; + } + + /** + * @brief Return the number of indexed values. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size()) { + return npos; + } + if (right - left <= top_block_size_) { + return cartesian_arg_min(left, right); + } + const std::size_t top_answer = top_sparse_arg_min(left, right); + if (top_answer != npos) { + return top_answer; + } + return cartesian_arg_min(left, right); + } + + /** + * @brief Return the number of BP bits in the Cartesian-tree RMQ encoding. + */ + std::size_t bp_bit_count() const { return bp_bit_count_; } + + /** + * @brief Return the packed BP words used by the RMQ encoding. + */ + std::span bp_words() const { + return bp_storage_words().first(bp_word_count()); + } + + /** + * @brief Return the top sparse-table block width chosen for a value count. + */ + static std::size_t top_sparse_block_size_for(std::size_t value_count) { + if (value_count == 0) { + return kMinTopSparseBlockSize; + } + return std::max(kMinTopSparseBlockSize, + ceil_div(value_count, kMaxTopSparseBlocks)); + } + + /** + * @brief Return the number of top sparse-table blocks for a value count. + */ + static std::size_t top_sparse_block_count_for(std::size_t value_count) { + if (value_count == 0) { + return 0; + } + return ceil_div(value_count, top_sparse_block_size_for(value_count)); + } + + /** + * @brief Return the current top sparse-table block width. + */ + std::size_t top_sparse_block_size() const { return top_block_size_; } + + /** + * @brief Return the current number of top sparse-table blocks. + */ + std::size_t top_sparse_block_count() const { return top_block_count_; } + + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this value-RMQ object, packed Cartesian BP words, the top + * sparse overlay, and nested BP rank/select and ±1 RMQ indexes. The external + * input values are not owned and are excluded. + */ + std::size_t memory_usage_bytes_impl() const { + return sizeof(*this) + bp_bits_.allocated_bytes() + + pixie::vector_capacity_bytes(top_sparse_candidates_) + + pixie::optional_nested_owned_memory_bytes(bp_index_) + + pixie::nested_owned_memory_bytes(bp_depth_rmq_); + } + + private: + using BpDepthRmq = detail::HybridBTreePlusMinusOne; + + struct TopCandidate { + Index position = invalid_index; + }; + static_assert(sizeof(TopCandidate) == sizeof(Index)); + + /** + * @brief Return the first minimum position through the Cartesian BP + * reduction. + */ + std::size_t cartesian_arg_min(std::size_t left, std::size_t right) const { + const std::size_t first_close = select_close_position(left + 1); + const std::size_t last_close = select_close_position(right); + if (first_close == npos || last_close == npos || first_close > last_close) { + return npos; + } + + const std::size_t shifted_min = + bp_depth_rmq_.arg_min(first_close + 1, last_close + 2); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + const BitVector& bp_index = *bp_index_; + const std::size_t answer = bp_index.rank0(shifted_min) - 1; + return answer < values_.size() ? answer : npos; + } + + /** + * @brief Rebuild the BP Cartesian-tree representation and support indexes. + */ + void build() { + bp_bits_.resize(0); + bp_bit_count_ = 0; + top_sparse_candidates_.clear(); + top_block_size_ = kMinTopSparseBlockSize; + top_block_count_ = 0; + top_sparse_levels_ = 0; + reset_bp_indexes(); + + if (values_.empty()) { + return; + } + if (values_.size() > (static_cast(invalid_index) - 1) / 2) { + throw std::length_error( + "CartesianHybridBTree RMQ index type is too small"); + } + + bp_bit_count_ = 2 * values_.size(); + bp_bits_.resize(padded_bp_bit_capacity()); + std::ranges::fill(bp_storage_words(), std::uint64_t{0}); + build_bp_bits(); + build_top_sparse_table(); + reset_bp_indexes(); + } + + /** + * @brief Build the Ferrada-Navarro BP bits with a monotone stack. + */ + void build_bp_bits() { + utils::SuccinctIncreasingStack stack(values_.size()); + std::size_t write_position = bp_bit_count_; + + for (std::size_t i = values_.size(); i-- > 0;) { + while (!stack.empty() && + !compare_(values_[stack_index(stack.top())], values_[i])) { + stack.pop(); + prepend_bp_bit(write_position, true); + } + stack.push(stack_key(i)); + prepend_bp_bit(write_position, false); + } + + while (write_position != 0) { + prepend_bp_bit(write_position, true); + } + } + + /** + * @brief Convert a value position to the increasing key used by the + * construction stack. + */ + std::size_t stack_key(std::size_t value_index) const { + return values_.size() - 1 - value_index; + } + + /** + * @brief Convert a construction-stack key back to the original value + * position. + */ + std::size_t stack_index(std::size_t key) const { + return values_.size() - 1 - key; + } + + /** + * @brief Prepend one BP bit into the right-to-left construction buffer. + */ + void prepend_bp_bit(std::size_t& write_position, bool bit) { + --write_position; + if (bit) { + bp_storage_words()[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); + } + } + + /** + * @brief Rebuild indexes that store non-owning spans into `bp_bits_`. + */ + void reset_bp_indexes() { + bp_index_.reset(); + bp_depth_rmq_ = BpDepthRmq(); + if (bp_bit_count_ == 0) { + return; + } + const std::span words = bp_words(); + const std::span padded_words = bp_storage_words(); + // TODO: try incorporating rank/select information into the tree. + bp_index_.emplace(words, bp_bit_count_, BitVector::SelectSupport::kSelect0, + values_.size()); + bp_depth_rmq_ = BpDepthRmq(padded_words, bp_bit_count_ + 1, *bp_index_); + } + + /** + * @brief Return the exact number of 64-bit words in the BP encoding. + */ + std::size_t bp_word_count() const { return ceil_div(bp_bit_count_, 64); } + + /** + * @brief Return BP storage capacity padded to full low-level depth leaves. + */ + std::size_t padded_bp_bit_capacity() const { + if (bp_bit_count_ == 0) { + return 0; + } + const std::size_t depth_count = bp_bit_count_ + 1; + return ceil_div(depth_count, LeafSize) * LeafSize; + } + + /** + * @brief Return mutable padded BP storage as 64-bit words. + */ + std::span bp_storage_words() { return bp_bits_.As64BitInts(); } + + /** + * @brief Return padded BP storage as 64-bit words. + */ + std::span bp_storage_words() const { + return bp_bits_.AsConst64BitInts(); + } + + /** + * @brief Build a single top sparse table over original-value block minima. + */ + void build_top_sparse_table() { + top_sparse_candidates_.clear(); + top_block_size_ = top_sparse_block_size_for(values_.size()); + top_block_count_ = ceil_div(values_.size(), top_block_size_); + top_sparse_levels_ = + top_block_count_ == 0 ? 0 : std::bit_width(top_block_count_); + if (top_block_count_ == 0) { + return; + } + + top_sparse_candidates_.assign(top_sparse_levels_ * top_block_count_, + TopCandidate{}); + for (std::size_t block = 0; block < top_block_count_; ++block) { + const std::size_t begin = block * top_block_size_; + const std::size_t end = std::min(values_.size(), begin + top_block_size_); + std::size_t minimum = begin; + for (std::size_t position = begin + 1; position < end; ++position) { + if (strictly_better_value_position(position, minimum)) { + minimum = position; + } + } + top_sparse_candidates_[block] = make_top_candidate(minimum); + } + + for (std::size_t level = 1; level < top_sparse_levels_; ++level) { + const std::size_t span = std::size_t{1} << level; + const std::size_t half_span = span >> 1; + TopCandidate* current = + top_sparse_candidates_.data() + level * top_block_count_; + const TopCandidate* previous = + top_sparse_candidates_.data() + (level - 1) * top_block_count_; + for (std::size_t block = 0; block + span <= top_block_count_; ++block) { + current[block] = + better_top_candidate(previous[block], previous[block + half_span]); + } + } + } + + /** + * @brief Return `ceil(value / divisor)` for positive divisors. + */ + static std::size_t ceil_div(std::size_t value, std::size_t divisor) { + return value == 0 ? 0 : 1 + (value - 1) / divisor; + } + + /** + * @brief Wrap an original value position as a top sparse-table candidate. + */ + TopCandidate make_top_candidate(std::size_t position) const { + if (position >= values_.size()) { + return {}; + } + return {static_cast(position)}; + } + + /** + * @brief Return whether @p position is a valid stored original-value index. + */ + bool valid_value_position(std::size_t position) const { + return position != npos && + position != static_cast(invalid_index) && + position < values_.size(); + } + + /** + * @brief Return whether value position @p left is strictly better than @p + * right. + */ + bool strictly_better_value_position(std::size_t left, + std::size_t right) const { + if (!valid_value_position(left)) { + return false; + } + if (!valid_value_position(right)) { + return true; + } + if (compare_(values_[left], values_[right])) { + return true; + } + if (compare_(values_[right], values_[left])) { + return false; + } + return left < right; + } + + /** + * @brief Choose the better original-value candidate, preserving first ties. + */ + TopCandidate better_top_candidate(TopCandidate left, + TopCandidate right) const { + const std::size_t left_position = static_cast(left.position); + const std::size_t right_position = static_cast(right.position); + return strictly_better_value_position(right_position, left_position) ? right + : left; + } + + /** + * @brief Return the sparse-table candidate over a top-block range. + */ + TopCandidate top_sparse_block_arg_min(std::size_t block_left, + std::size_t block_right) const { + if (block_left >= block_right || block_right > top_block_count_ || + top_sparse_levels_ == 0) { + return {}; + } + const std::size_t length = block_right - block_left; + const std::size_t level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << level; + const TopCandidate* table = + top_sparse_candidates_.data() + level * top_block_count_; + return better_top_candidate(table[block_left], table[block_right - span]); + } + + /** + * @brief Return whether a candidate lies inside a half-open value range. + */ + bool top_candidate_inside(TopCandidate candidate, + std::size_t left, + std::size_t right) const { + const std::size_t position = static_cast(candidate.position); + return valid_value_position(position) && left <= position && + position < right; + } + + /** + * @brief Return the top-overlay answer, or `npos` when the BP path should + * run. + */ + std::size_t top_sparse_arg_min(std::size_t left, std::size_t right) const { + if (top_block_count_ <= 1) { + return npos; + } + + const std::size_t padded_block_left = left / top_block_size_; + const std::size_t padded_block_right = (right - 1) / top_block_size_ + 1; + if (padded_block_left + 1 >= padded_block_right) { + return npos; + } + + const TopCandidate padded = + top_sparse_block_arg_min(padded_block_left, padded_block_right); + if (top_candidate_inside(padded, left, right)) { + return static_cast(padded.position); + } + + const std::size_t first_full_block = + (left + top_block_size_ - 1) / top_block_size_; + const std::size_t full_block_right = right / top_block_size_; + if (first_full_block >= full_block_right) { + return npos; + } + + TopCandidate answer = + top_sparse_block_arg_min(first_full_block, full_block_right); + + const std::size_t left_border_end = first_full_block * top_block_size_; + if (left < left_border_end) { + answer = better_top_candidate( + answer, make_top_candidate(cartesian_arg_min(left, left_border_end))); + } + + const std::size_t right_border_begin = full_block_right * top_block_size_; + if (right_border_begin < right) { + answer = better_top_candidate( + answer, + make_top_candidate(cartesian_arg_min(right_border_begin, right))); + } + + return valid_value_position(static_cast(answer.position)) + ? static_cast(answer.position) + : npos; + } + + /** + * @brief Return the one-based rank-th Cartesian close parenthesis. + */ + std::size_t select_close_position(std::size_t rank) const { + if (rank == 0 || rank > values_.size() || !bp_index_) { + return npos; + } + const std::size_t position = bp_index_->select0(rank); + return position < bp_bit_count_ ? position : npos; + } + + std::span values_; + Compare compare_; + pixie::AlignedStorage bp_bits_; + std::size_t bp_bit_count_ = 0; + std::vector top_sparse_candidates_; + std::size_t top_block_size_ = kMinTopSparseBlockSize; + std::size_t top_block_count_ = 0; + std::size_t top_sparse_levels_ = 0; + std::optional bp_index_; + BpDepthRmq bp_depth_rmq_; +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/cartesian_rmm.h b/include/pixie/rmq/cartesian_rmm.h new file mode 100644 index 0000000..e5900a3 --- /dev/null +++ b/include/pixie/rmq/cartesian_rmm.h @@ -0,0 +1,459 @@ +#pragma once + +/** + * rmM-backed Cartesian RMQ. + * + * The benchmark harness registers this retained variant as + * `rmq_cartesian_rmm` and `rmq_build_cartesian_rmm`. + * It is included from `pixie/rmq.h` with the rest of the retained RMQ backends. + * + * Diagnostic snapshot, 2026-06-13: + * + * Focused value-RMQ rows, N=2^22 values, CPU mean across 5 repetitions. + * Command shape: + * taskset -c 0 ./build/release/bench_rmq + * --benchmark_filter='^rmq_(cartesian_rmm|sdsl_sct)/4194304/' + * + * | max width | CartesianRmM (ns) | SdslSct (ns) | + * | --------: | ----------------: | -----------: | + * | 64 | 121.0 | 211.0 | + * | 4096 | 310.0 | 757.0 | + * | 2^18 | 529.0 | 1057.0 | + * | 2^22 | 598.0 | 979.0 | + * + * Raw BP rmM rows at 2^23 bits, Q=32768, CPU mean across 5 repetitions. + * Command shape: + * ./build/release/bench_rmm_{btree,sdsl} + * --ops=range_min_query_pos,range_min_query_val + * --explicit_sizes=8388608 --Q=32768 + * + * | backend | range_min_pos (ns) | range_min_val (ns) | + * | ------------------- | -----------------: | -----------------: | + * | RmMBTree | 602.0 | 540.0 | + * | SdslRmMTree | 727.0 | 742.0 | + * + * The 2026-06-13 optimization pass streams cover nodes instead of materializing + * a zero-initialized per-query cover, adds a combined min position/value query + * for the Cartesian adapter, and uses a minimum-only boundary scanner for + * range-min position/value queries. Current perf samples for raw + * range_min_query_pos are concentrated in for_each_cover_node(), + * scan_min_range(), and summary_at(). + * + * Historical build snapshot after select0-only BP rank/select construction, + * 2026-06-13, before the RmMBTree full-block summary fast path. + * Command shape: + * taskset -c 0 ./build/release/bench_rmq + * --benchmark_filter='^rmq_build_(cartesian_rmm|cartesian_hybrid_btree|sdsl_sct)/(4194304|67108864)$' + * --benchmark_repetitions=5 + * + * CPU mean, milliseconds. + * + * | N | CartesianRmM | CartesianHybrid | SdslSct | + * | ---: | -----------: | --------------: | ------: | + * | 2^22 | 54.407 | 49.361 | 41.317 | + * | 2^26 | 915.712 | 795.562 | 666.048 | + * + * Historical construction perf profile snapshot, 2026-06-13, before the + * RmMBTree full-block summary fast path. + * Profiling build: RelWithDebInfo, -O3, debug info, frame pointers. Rows are + * N=2^26 build benchmarks sampled with perf at 999 Hz. + * + * CartesianHybridBTree: 867 ms CPU in the profiled run. BP construction is the + * dominant cost: the succinct monotone-stack operations plus BP writes account + * for roughly two thirds of samples. Top sparse-table construction is about 6%. + * CartesianRmM: 1013 ms CPU in the profiled run. BP construction was still the + * dominant cost, while rmM summary construction added a visible second cost: + * summarize_bits()/bit() lines accounted for about 14%. Benchmark dataset + * generation was about 5% in both rows. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +namespace detail { + +/** + * @brief Depth RMQ adapter over the experimental rmM btree. + * + * @details The input words encode adjacent depth deltas: bit 1 means +1 and + * bit 0 means -1. The indexed depth sequence has `depth_count` prefix + * positions, so the bit sequence has `depth_count - 1` bits. Queries return the + * first position of the minimum depth in a half-open depth-position range. + * + * `pixie::experimental::RmMBTree::range_min_query_pos()` works on inclusive + * bit ranges and reports extrema after consuming each bit. This adapter adds + * the missing left prefix boundary comparison, preserving first-minimum ties. + */ +template +class RmMPlusMinusOne { + public: + static_assert(std::is_unsigned_v, + "RmMPlusMinusOne index type must be unsigned"); + + static constexpr std::size_t npos = std::numeric_limits::max(); + static constexpr Index invalid_index = std::numeric_limits::max(); + static constexpr std::size_t kBlockSize = + pixie::experimental::RmMBTree::kBlockBits; + + RmMPlusMinusOne() = default; + + /** + * @brief Build an rmM-backed ±1 RMQ over external packed delta bits. + * + * @param bits Little-endian packed delta bits. + * @param depth_count Number of indexed depth positions. + * @throws std::length_error if @p Index cannot represent all positions. + * @throws std::invalid_argument if @p bits is shorter than required. + */ + RmMPlusMinusOne(std::span bits, + std::size_t depth_count) { + build(bits, depth_count); + } + + /** + * @brief Build with an exact count of +1 delta bits. + * + * @details Cartesian BP construction knows this count exactly, which lets the + * nested BitVector allocate select samples without a second scan. + */ + RmMPlusMinusOne(std::span bits, + std::size_t depth_count, + std::size_t one_count) { + build(bits, depth_count, one_count); + } + + /** + * @brief Rebuild this adapter over external packed delta bits. + */ + void build(std::span bits, std::size_t depth_count) { + build(bits, depth_count, std::nullopt); + } + + /** + * @brief Rebuild this adapter with an optional exact count of +1 delta bits. + */ + void build(std::span bits, + std::size_t depth_count, + std::optional one_count) { + words_ = bits; + depth_count_ = depth_count; + rmm_ = RmM(); + + if (depth_count_ == 0) { + return; + } + if (depth_count_ > static_cast(invalid_index)) { + throw std::length_error("RMQ rmM btree index type is too small"); + } + rmm_ = RmM(words_, depth_count_ - 1, BitVector::SelectSupport::kSelect0, + one_count); + } + + /** + * @brief Return the number of indexed depth positions. + */ + std::size_t size() const { return depth_count_; } + + /** + * @brief Whether the indexed depth sequence is empty. + */ + bool empty() const { return depth_count_ == 0; } + + /** + * @brief Return the first minimum depth position in [@p left, @p right). + * + * @details Empty or invalid ranges return `npos`. Equal minima return the + * smaller depth position. + */ + std::size_t arg_min(std::size_t left, std::size_t right) const { + if (left >= right || right > depth_count_) { + return npos; + } + if (right == left + 1) { + return left; + } + + const std::size_t bit_left = left; + const std::size_t bit_right = right - 2; + const auto minimum_after_left = + rmm_.range_min_query_result(bit_left, bit_right); + if (minimum_after_left.value >= 0) { + return left; + } + + const std::size_t bit_position = minimum_after_left.position; + return bit_position == RmM::npos ? npos : bit_position + 1; + } + + /** + * @brief Return the one-based rank-th closing parenthesis position. + */ + std::size_t select0(std::size_t rank) const { return rmm_.select0(rank); } + + /** + * @brief Count closing parentheses in the prefix [0, @p end_position). + */ + std::size_t rank0(std::size_t end_position) const { + return rmm_.rank0(end_position); + } + + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this ±1 RMQ adapter and the nested rmM support. The + * external packed BP words are not owned and are excluded. + */ + std::size_t memory_usage_bytes() const { + return sizeof(*this) + pixie::nested_owned_memory_bytes(rmm_); + } + + private: + using RmM = pixie::experimental::RmMBTree; + + std::span words_; + std::size_t depth_count_ = 0; + RmM rmm_; +}; + +} // namespace detail + +/** + * @brief Ferrada-Navarro Cartesian-tree RMQ using rmM support. + * + * @details This class follows the same public value-RMQ specification as the + * other value RMQ backends, but replaces the usual balanced-parentheses + * rank/select and depth-RMQ support with `detail::RmMPlusMinusOne`, which in + * turn uses the experimental range min-max btree over the BP excess sequence. + * BP construction uses a succinct monotone bit-stack, preserving the same + * stable Cartesian-tree shape without an n-entry index stack. + * + * This implementation is included from `pixie/rmq.h` as the rmM-backed + * Cartesian-tree reduction. + */ +template , + class Index = std::size_t, + std::size_t HighCacheLines = 4, + std::size_t LowFanout = 32> +class CartesianRmM + : public RmqBase, + T> { + public: + static_assert(std::is_unsigned_v, + "CartesianRmM index type must be unsigned"); + + static constexpr std::size_t npos = + RmqBase, + T>::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + + CartesianRmM() = default; + + /** + * @brief Build a Cartesian-tree RMQ index over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values stay stable: the smaller index remains the first minimum. + */ + explicit CartesianRmM(std::span values, Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + CartesianRmM(const CartesianRmM& other) + : values_(other.values_), + compare_(other.compare_), + bp_bits_(other.bp_bits_), + bp_bit_count_(other.bp_bit_count_) { + reset_bp_index(); + } + + CartesianRmM& operator=(const CartesianRmM& other) { + if (this == &other) { + return *this; + } + values_ = other.values_; + compare_ = other.compare_; + bp_bits_ = other.bp_bits_; + bp_bit_count_ = other.bp_bit_count_; + reset_bp_index(); + return *this; + } + + CartesianRmM(CartesianRmM&& other) noexcept + : values_(other.values_), + compare_(std::move(other.compare_)), + bp_bits_(std::move(other.bp_bits_)), + bp_bit_count_(other.bp_bit_count_) { + other.values_ = std::span(); + other.bp_bit_count_ = 0; + reset_bp_index(); + } + + CartesianRmM& operator=(CartesianRmM&& other) noexcept { + if (this == &other) { + return *this; + } + values_ = other.values_; + compare_ = std::move(other.compare_); + bp_bits_ = std::move(other.bp_bits_); + bp_bit_count_ = other.bp_bit_count_; + other.values_ = std::span(); + other.bp_bit_count_ = 0; + reset_bp_index(); + return *this; + } + + /** + * @brief Return the number of indexed values. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size()) { + return npos; + } + const std::size_t first_close = bp_support_.select0(left + 1); + const std::size_t last_close = bp_support_.select0(right); + if (first_close == BpSupport::npos || last_close == BpSupport::npos || + first_close > last_close) { + return npos; + } + + const std::size_t shifted_min = + bp_support_.arg_min(first_close + 1, last_close + 2); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + const std::size_t answer = bp_support_.rank0(shifted_min) - 1; + return answer < values_.size() ? answer : npos; + } + + /** + * @brief Return the number of BP bits in the Cartesian-tree RMQ encoding. + */ + std::size_t bp_bit_count() const { return bp_bit_count_; } + + /** + * @brief Return the packed BP words used by the RMQ encoding. + */ + std::span bp_words() const { return bp_bits_; } + + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this value-RMQ object, packed Cartesian BP words, and + * nested BP rmM support. The external input values are not owned and are + * excluded. + */ + std::size_t memory_usage_bytes_impl() const { + return sizeof(*this) + pixie::vector_capacity_bytes(bp_bits_) + + pixie::nested_owned_memory_bytes(bp_support_); + } + + private: + using BpSupport = detail::RmMPlusMinusOne; + + void build() { + bp_bits_.clear(); + bp_bit_count_ = 0; + bp_support_ = BpSupport(); + + if (values_.empty()) { + return; + } + if (values_.size() > (static_cast(invalid_index) - 1) / 2) { + throw std::length_error("Cartesian rmM RMQ index type is too small"); + } + + bp_bit_count_ = 2 * values_.size(); + bp_bits_.assign((bp_bit_count_ + 63) / 64, 0); + build_bp_bits(); + reset_bp_index(); + } + + void build_bp_bits() { + utils::SuccinctIncreasingStack stack(values_.size()); + std::size_t write_position = bp_bit_count_; + + for (std::size_t i = values_.size(); i-- > 0;) { + while (!stack.empty() && + !compare_(values_[stack_index(stack.top())], values_[i])) { + stack.pop(); + prepend_bp_bit(write_position, true); + } + stack.push(stack_key(i)); + prepend_bp_bit(write_position, false); + } + + while (write_position != 0) { + prepend_bp_bit(write_position, true); + } + } + + /** + * @brief Convert a value position to the increasing key used by the + * construction stack. + */ + std::size_t stack_key(std::size_t value_index) const { + return values_.size() - 1 - value_index; + } + + /** + * @brief Convert a construction-stack key back to the original value + * position. + */ + std::size_t stack_index(std::size_t key) const { + return values_.size() - 1 - key; + } + + void prepend_bp_bit(std::size_t& write_position, bool bit) { + --write_position; + if (bit) { + bp_bits_[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); + } + } + + void reset_bp_index() { + bp_support_ = BpSupport(); + if (bp_bit_count_ == 0) { + return; + } + bp_support_ = BpSupport(std::span(bp_bits_), + bp_bit_count_ + 1, values_.size()); + } + + std::span values_; + Compare compare_; + std::vector bp_bits_; + std::size_t bp_bit_count_ = 0; + BpSupport bp_support_; +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/hybrid_btree.h b/include/pixie/rmq/hybrid_btree.h new file mode 100644 index 0000000..9c5f479 --- /dev/null +++ b/include/pixie/rmq/hybrid_btree.h @@ -0,0 +1,1653 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief Low-level selector implementation used by HybridBTree leaves. + */ +enum class HybridBTreeLeafSelector { + PrefixSuffix, + BP, +}; + +/** + * @brief Hybrid B-tree RMQ with compact per-level selectors. + * + * @details This is a static, non-owning value-RMQ index over an external value + * array. Values are split into leaves, leaves are grouped into a B-tree, and + * each node stores a selector over the minima of its immediate children. Query + * ranges use the library-wide half-open convention `[left, right)`, invalid + * ranges return `npos`, and equal values resolve to the smaller original + * position. + * + * The default low level uses 496-value prefix/suffix mask leaves. A leaf stores + * one bit for each prefix/suffix record position plus a 16-bit local-minimum + * offset in one 64-byte object. Prefix or suffix ranges can be answered from + * this mask; interior partial ranges fall back to a linear scan of the original + * values. The same implementation also supports 248-value mask leaves and + * 252-value BP leaves for controlled experiments. + * + * Internal nodes use 192-way fanout. Their selector is a 512-bit local + * Cartesian-tree balanced-parentheses encoding over child minima: 384 BP bits, + * a 64-bit absolute subtree-minimum position, and 64 bits of zero-rank prefix + * metadata. Node minimum values are cached in a side vector so comparing + * candidates does not require repeatedly descending to leaves. + * + * Wide queries first try a single coarse sparse table over original-value block + * minima. The sparse table uses at least 4096-value blocks and grows the block + * width when needed so the top layer has at most 2^14 blocks. If the padded + * block-cover candidate lies inside the query, it is returned immediately; on + * a miss, the sparse table answers the fully covered middle block range and + * the B-tree handles the two border ranges. + * + * @code + * value array, n entries + * | + * +-- L0: leaves + * | ceil(n / 496) nodes by default + * | each leaf covers <=496 values and stores one 64-byte selector + * | + * +-- T: value-block sparse overlay + * | block width >=4096 values, at most 2^14 blocks + * | stores one original-value minimum position per sparse-table cell + * | + * +-- L1..Lm: internal levels + * | fanout 192 + * | each node stores one 512-bit BP selector over child minima + * + * Examples with default 496-value leaves: + * + * n = 2^24: + * 33,826 leaves -> 177 internal nodes -> 1 root + * + * n = 2^26: + * 135,301 leaves -> 705 internal nodes -> 4 internal nodes -> 1 root + * @endcode + * + * Querying starts at the lowest node that covers both endpoint leaves. At each + * internal node, the selector is asked for the best child over the intersecting + * child-slot range. If that child is fully covered, or its stored subtree + * minimum lies inside the query, that candidate is final for the node. + * Otherwise only the affected border child is corrected recursively and + * compared with the middle full-child range. All comparisons use `(value, + * position)` semantics so traversal order cannot change tie-breaking. + * + * @tparam T Value type in the indexed array. + * @tparam Compare Strict weak ordering used to choose minima. + * @tparam Index Unsigned integer type used for stored positions. + * @tparam LeafSize Number of original values per leaf. This backend currently + * supports 252 with BP leaves and 248 or 496 with mask leaves. + * @tparam Fanout Template parameter kept fixed at 256 for the current layout; + * internal tree nodes use fixed 192-way fanout. + * @tparam LeafSelectorKind Compile-time low-level selector kind. + */ +template , + class Index = std::size_t, + std::size_t LeafSize = 496, + std::size_t Fanout = 256, + HybridBTreeLeafSelector LeafSelectorKind = + HybridBTreeLeafSelector::PrefixSuffix> +class HybridBTree + : public RmqBase< + HybridBTree, + T> { + public: + static_assert(std::is_unsigned_v, + "HybridBTree index type must be unsigned"); + static constexpr bool kBpLeafSelector = + LeafSelectorKind == HybridBTreeLeafSelector::BP; + static constexpr bool kMaskLeafSelector = + LeafSelectorKind == HybridBTreeLeafSelector::PrefixSuffix; + static_assert((kBpLeafSelector && LeafSize == 252) || + (kMaskLeafSelector && (LeafSize == 248 || LeafSize == 496)), + "HybridBTree requires 252-value BP leaves " + "or 248/496-value prefix/suffix mask leaves"); + static_assert(Fanout == 256, + "HybridBTree currently requires a 256 fanout template " + "argument"); + static_assert(kBpLeafSelector || kMaskLeafSelector, + "unsupported HybridBTree low-level selector kind"); + + using Self = + HybridBTree; + + static constexpr std::size_t npos = RmqBase::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + static constexpr std::size_t kLeafSize = LeafSize; + static constexpr std::size_t kFanout = Fanout; + static constexpr std::size_t kMiddleFanout = 192; + static constexpr std::size_t kMinTopSparseBlockSize = 4096; + static constexpr std::size_t kMaxTopSparseBlocks = std::size_t{1} << 14; + + /** + * @brief Construct an empty RMQ index. + */ + HybridBTree() = default; + + /** + * @brief Copy an RMQ index while preserving its non-owning value span. + */ + HybridBTree(const HybridBTree&) = default; + + /** + * @brief Move an RMQ index while preserving selector and cache storage. + */ + HybridBTree(HybridBTree&&) noexcept = default; + + /** + * @brief Copy-assign an RMQ index and its cached metadata. + */ + HybridBTree& operator=(const HybridBTree&) = default; + + /** + * @brief Move-assign an RMQ index and its cached metadata. + */ + HybridBTree& operator=(HybridBTree&&) noexcept = default; + + /** + * @brief Build a hybrid B-tree RMQ index over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values keep the smaller original position as the answer. + * + * @param values Values to index. + * @param compare Ordering used to choose minima. + * @throws std::length_error if @p Index cannot represent all positions. + */ + explicit HybridBTree(std::span values, Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + /** + * @brief Return the number of indexed values. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + * + * @details Empty or invalid ranges return `npos`. Ties are reduced by + * comparing `(value, position)`, so traversal order cannot change first-min + * semantics. + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size() || level_sizes_.empty()) { + return npos; + } + + if (right - left <= top_block_size_) { + return tree_arg_min(left, right); + } + const std::size_t top_answer = top_sparse_arg_min(left, right); + return top_answer != npos ? top_answer : tree_arg_min(left, right); + } + + /** + * @brief Return the top sparse-table block width chosen for a value count. + */ + static std::size_t top_sparse_block_size_for(std::size_t value_count) { + if (value_count == 0) { + return kMinTopSparseBlockSize; + } + return std::max(kMinTopSparseBlockSize, + 1 + (value_count - 1) / kMaxTopSparseBlocks); + } + + /** + * @brief Return the number of top sparse-table blocks for a value count. + */ + static std::size_t top_sparse_block_count_for(std::size_t value_count) { + if (value_count == 0) { + return 0; + } + const std::size_t block_size = top_sparse_block_size_for(value_count); + return 1 + (value_count - 1) / block_size; + } + + /** + * @brief Return the current top sparse-table block width. + */ + std::size_t top_sparse_block_size() const { return top_block_size_; } + + /** + * @brief Return the current number of top sparse-table blocks. + */ + std::size_t top_sparse_block_count() const { return top_block_count_; } + + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this RMQ object and all selector/metadata buffers. The + * external input values are not owned and are excluded. + */ + std::size_t memory_usage_bytes_impl() const { + std::size_t bytes = sizeof(*this); + bytes += pixie::vector_capacity_bytes(leaf_selectors_); + bytes += pixie::vector_capacity_bytes(medium_selectors_); + bytes += pixie::vector_capacity_bytes(medium_min_values_); + bytes += pixie::vector_capacity_bytes(top_sparse_candidates_); + bytes += pixie::vector_capacity_bytes(medium_level_offsets_); + bytes += pixie::vector_capacity_bytes(level_sizes_); + bytes += pixie::vector_capacity_bytes(level_value_spans_); + bytes += pixie::vector_capacity_bytes(level_fanouts_); + return bytes; + } + + private: + static constexpr std::size_t kSelectorEntries = 256; + static constexpr std::size_t kSelectorBits = 2 * kSelectorEntries; + static constexpr std::size_t kSelectorWords = kSelectorBits / 64; + static constexpr std::size_t kEmbeddedOffsetEntries = 252; + static constexpr std::size_t kEmbeddedPositionEntries = 192; + static constexpr std::size_t kEmbeddedOffsetBit = 2 * kEmbeddedOffsetEntries; + static constexpr std::size_t kEmbeddedPositionBit = + 2 * kEmbeddedPositionEntries; + static constexpr std::size_t kMiddleZeroPrefixWord = 7; + static constexpr std::size_t kLeafLinearScanThreshold = 64; + static constexpr std::size_t kLeafAvx2ScanThreshold = 16; + static constexpr std::uint64_t kEmbeddedOffsetMask = + std::uint64_t{0xFF} << (kEmbeddedOffsetBit & 63); + static constexpr bool kInvalidIndexEqualsNpos = + static_cast(invalid_index) == npos; + + static_assert(kEmbeddedOffsetBit + 8 == kSelectorBits); + static_assert(kEmbeddedPositionBit + 128 == kSelectorBits); + static_assert(!kBpLeafSelector || LeafSize <= kEmbeddedOffsetEntries); + static_assert(kMiddleFanout <= kEmbeddedPositionEntries); + + struct MinCandidate { + std::size_t position = npos; + const T* value = nullptr; + }; + + struct TopCandidate { + Index position = invalid_index; + }; + static_assert(sizeof(TopCandidate) == sizeof(Index)); + + class alignas(64) Bp512Selector { + public: + /** + * @brief Construct an empty packed BP selector. + */ + Bp512Selector() = default; + + /** + * @brief Build a stable local Cartesian-tree BP selector. + * + * @details The selector indexes @p entry_count logical entries. The + * supplied comparator returns whether the left entry is strictly better + * than the right entry. Equal entries keep the smaller slot by using the + * same stable rule as the value-level RMQ. + */ + template + void build(std::size_t entry_count, EntryLess entry_less) { + if (entry_count > kSelectorEntries) { + throw std::length_error("HybridBTree local selector too large"); + } + + bp_bits_.fill(0); + if (entry_count == 0) { + return; + } + + std::array stack{}; + std::size_t stack_size = 0; + std::size_t write_position = 2 * entry_count; + + for (std::size_t i = entry_count; i-- > 0;) { + while (stack_size != 0 && !entry_less(stack[stack_size - 1], i)) { + --stack_size; + prepend_bp_bit(write_position, true); + } + stack[stack_size++] = static_cast(i); + prepend_bp_bit(write_position, false); + } + + while (write_position != 0) { + prepend_bp_bit(write_position, true); + } + } + + /** + * @brief Return the first minimum slot in a local entry range. + * + * @details This path is used by BP leaves and high nodes whose BP words do + * not share storage with zero-rank prefix metadata. + */ + std::size_t arg_min(std::size_t slot_left, + std::size_t slot_right, + std::size_t entry_count) const { + if (slot_left >= slot_right || slot_right > entry_count || + entry_count > kSelectorEntries) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t bit_count = 2 * entry_count; + const std::size_t first_close = close_position(slot_left); + const std::size_t last_close = close_position(slot_right - 1); + if (first_close > last_close || last_close >= bit_count) { + return npos; + } + + const std::size_t shifted_min = + depth_arg_min(first_close + 1, last_close + 2, bit_count); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + + const std::size_t zero_rank = rank0_at(shifted_min, bit_count); + if (zero_rank == 0) { + return npos; + } + const std::size_t entry = zero_rank - 1; + return entry < entry_count ? entry : npos; + } + + /** + * @brief Return the first minimum slot using packed zero-rank metadata. + * + * @details Middle nodes overwrite the last selector word with zero-rank + * prefixes, so rank/select operations must consult the packed metadata + * instead of treating all eight words as raw BP bits. + */ + std::size_t arg_min_with_zero_prefix(std::size_t slot_left, + std::size_t slot_right, + std::size_t entry_count) const { + if (slot_left >= slot_right || slot_right > entry_count || + entry_count > kEmbeddedPositionEntries) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t bit_count = 2 * entry_count; + const std::size_t first_close = + close_position_with_zero_prefix(slot_left, bit_count); + const std::size_t last_close = + close_position_with_zero_prefix(slot_right - 1, bit_count); + if (first_close > last_close || last_close >= bit_count) { + return npos; + } + + const std::size_t shifted_min = depth_arg_min_with_zero_prefix( + first_close + 1, last_close + 2, bit_count); + if (shifted_min == npos || shifted_min == 0) { + return npos; + } + + const std::size_t zero_rank = + rank0_at_with_zero_prefix(shifted_min, bit_count); + if (zero_rank == 0) { + return npos; + } + const std::size_t entry = zero_rank - 1; + return entry < entry_count ? entry : npos; + } + + /** + * @brief Return the BP close position for a local entry slot. + */ + std::size_t close_position(std::size_t slot) const { + return select0_512(bp_bits_.data(), slot); + } + + /** + * @brief Count zero bits before @p position in the raw BP sequence. + */ + std::size_t rank0_at(std::size_t position, std::size_t bit_count) const { + position = std::min(position, bit_count); + return position - rank_512(bp_bits_.data(), position); + } + + /** + * @brief Pack a leaf-local minimum offset into the spare BP bits. + */ + void set_embedded_min_offset(std::size_t offset) { + bp_bits_[kEmbeddedOffsetBit >> 6] = + (bp_bits_[kEmbeddedOffsetBit >> 6] & ~kEmbeddedOffsetMask) | + ((static_cast(offset) & 0xFFu) + << (kEmbeddedOffsetBit & 63)); + } + + /** + * @brief Return the leaf-local minimum offset stored in the selector. + */ + std::uint8_t embedded_min_offset() const { + return static_cast( + (bp_bits_[kEmbeddedOffsetBit >> 6] & kEmbeddedOffsetMask) >> + (kEmbeddedOffsetBit & 63)); + } + + /** + * @brief Store a node's absolute subtree minimum position. + */ + void set_embedded_min_position(std::size_t position) { + bp_bits_[kEmbeddedPositionBit >> 6] = + static_cast(position); + } + + /** + * @brief Return the absolute subtree minimum position stored in a node. + */ + std::size_t embedded_min_position() const { + return static_cast(bp_bits_[kEmbeddedPositionBit >> 6]); + } + + /** + * @brief Pack per-word zero-count prefixes for middle-node selectors. + */ + void build_zero_prefix_metadata(std::size_t bit_count) { + const std::size_t word_count = (bit_count + 63) / 64; + std::uint64_t packed = 0; + std::size_t zeros = 0; + for (std::size_t word = 0; word <= word_count; ++word) { + packed |= (static_cast(zeros) & 0xFFu) << (8 * word); + if (word == word_count) { + break; + } + const std::size_t word_begin = word * 64; + const std::size_t word_bits = + std::min(64, bit_count - word_begin); + const std::uint64_t bits = bp_bits_[word] & first_bits_mask(word_bits); + zeros += word_bits - std::popcount(bits); + } + bp_bits_[kMiddleZeroPrefixWord] = packed; + } + + private: + /** + * @brief Prepend one BP bit while building the sequence right-to-left. + */ + std::size_t prepend_bp_bit(std::size_t& write_position, bool bit) { + --write_position; + if (bit) { + bp_bits_[write_position >> 6] |= std::uint64_t{1} + << (write_position & 63); + } + return write_position; + } + + /** + * @brief Return open-minus-close excess before a raw BP position. + */ + int prefix_excess(std::size_t position) const { + position = std::min(position, kSelectorBits); + const std::size_t ones = rank_512(bp_bits_.data(), position); + return static_cast(2 * ones) - static_cast(position); + } + + /** + * @brief Return the packed zero-count prefix at a 64-bit word boundary. + */ + std::size_t zero_prefix_at_word(std::size_t word) const { + return static_cast(bp_bits_[kMiddleZeroPrefixWord] >> + (8 * word)); + } + + /** + * @brief Count close parentheses before @p position using packed prefixes. + */ + std::size_t rank0_at_with_zero_prefix(std::size_t position, + std::size_t bit_count) const { + position = std::min(position, bit_count); + const std::size_t full_words = position >> 6; + std::size_t zeros = zero_prefix_at_word(full_words); + const std::size_t tail_bits = position & 63; + if (tail_bits != 0) { + const std::uint64_t tail = + bp_bits_[full_words] & first_bits_mask(tail_bits); + zeros += tail_bits - std::popcount(tail); + } + return zeros; + } + + /** + * @brief Return BP excess when the last word stores zero-rank metadata. + */ + int prefix_excess_with_zero_prefix(std::size_t position, + std::size_t bit_count) const { + position = std::min(position, bit_count); + return static_cast(position) - + 2 * static_cast( + rank0_at_with_zero_prefix(position, bit_count)); + } + + /** + * @brief Select a close parenthesis using packed per-word zero counts. + */ + std::size_t close_position_with_zero_prefix(std::size_t slot, + std::size_t bit_count) const { + const std::size_t word_count = (bit_count + 63) / 64; + for (std::size_t word = 0; word < word_count; ++word) { + const std::size_t next_zero_prefix = zero_prefix_at_word(word + 1); + if (next_zero_prefix <= slot) { + continue; + } + const std::size_t local_rank = slot - zero_prefix_at_word(word); + const std::size_t word_begin = word * 64; + const std::size_t word_bits = + std::min(64, bit_count - word_begin); + const std::uint64_t zeros = + (~bp_bits_[word]) & first_bits_mask(word_bits); + return word_begin + select_64(zeros, local_rank); + } + return npos; + } + + /** + * @brief Return the position of the minimum BP depth in a depth range. + */ + std::size_t depth_arg_min(std::size_t left, + std::size_t right, + std::size_t bit_count) const { + const std::size_t depth_count = bit_count + 1; + if (left >= right || right > depth_count) { + return npos; + } + + std::size_t position = left; + int best_depth = prefix_excess(position); + std::size_t best_position = position; + + while (position < right) { + const std::size_t chunk_begin = (position / 128) * 128; + const std::size_t local_left = position - chunk_begin; + const std::size_t local_right = + std::min(right - 1, chunk_begin + 128) - chunk_begin; + + int candidate_depth; + std::size_t candidate_position; + if (chunk_begin >= bit_count) { + candidate_depth = prefix_excess(chunk_begin); + candidate_position = chunk_begin; + } else { + const std::size_t word = chunk_begin >> 6; + const ExcessResult result = + excess_min_128(bp_bits_.data() + word, local_left, local_right); + candidate_depth = prefix_excess(chunk_begin) + result.min_excess; + candidate_position = chunk_begin + result.offset; + } + + if (candidate_depth < best_depth) { + best_depth = candidate_depth; + best_position = candidate_position; + } + + position = chunk_begin + local_right + 1; + } + + return best_position; + } + + /** + * @brief Return the minimum-depth position for middle-node packed BP data. + */ + std::size_t depth_arg_min_with_zero_prefix(std::size_t left, + std::size_t right, + std::size_t bit_count) const { + const std::size_t depth_count = bit_count + 1; + if (left >= right || right > depth_count) { + return npos; + } + + std::size_t position = left; + int best_depth = prefix_excess_with_zero_prefix(position, bit_count); + std::size_t best_position = position; + + while (position < right) { + const std::size_t chunk_begin = (position / 128) * 128; + const std::size_t local_left = position - chunk_begin; + const std::size_t local_right = + std::min(right - 1, chunk_begin + 128) - chunk_begin; + + int candidate_depth; + std::size_t candidate_position; + if (chunk_begin >= bit_count) { + candidate_depth = + prefix_excess_with_zero_prefix(chunk_begin, bit_count); + candidate_position = chunk_begin; + } else { + const std::size_t word = chunk_begin >> 6; + const ExcessResult result = + excess_min_128(bp_bits_.data() + word, local_left, local_right); + candidate_depth = + prefix_excess_with_zero_prefix(chunk_begin, bit_count) + + result.min_excess; + candidate_position = chunk_begin + result.offset; + } + + if (candidate_depth < best_depth) { + best_depth = candidate_depth; + best_position = candidate_position; + } + + position = chunk_begin + local_right + 1; + } + + return best_position; + } + + std::array bp_bits_{}; + }; + + static_assert(sizeof(Bp512Selector) == 64); + + class alignas(LeafSize == 496 ? 64 : 32) PrefixSuffixMaskLeafSelector { + public: + /** + * @brief Construct an empty prefix/suffix mask selector. + */ + PrefixSuffixMaskLeafSelector() = default; + + /** + * @brief Build prefix/suffix record masks and store the local minimum. + * + * @details A bit is set for every prefix-minimum record and every + * suffix-minimum record. Equal suffix values set the smaller slot so + * first-minimum tie-breaking is preserved. + */ + template + void build(std::size_t entry_count, EntryLess entry_less) { + if (entry_count > kMaskEntries) { + throw std::length_error( + "HybridBTree prefix/suffix leaf selector too large"); + } + + words_.fill(0); + if (entry_count == 0) { + set_embedded_min_offset(0); + return; + } + + std::size_t prefix_best = 0; + set_mask_bit(0); + for (std::size_t slot = 1; slot < entry_count; ++slot) { + if (entry_less(slot, prefix_best)) { + prefix_best = slot; + set_mask_bit(slot); + } + } + + std::size_t suffix_best = entry_count - 1; + set_mask_bit(suffix_best); + for (std::size_t slot = entry_count - 1; slot > 0;) { + --slot; + if (!entry_less(suffix_best, slot)) { + suffix_best = slot; + set_mask_bit(slot); + } + } + + set_embedded_min_offset(prefix_best); + } + + /** + * @brief Return a local minimum slot when the mask can answer directly. + * + * @details The mask answers full-leaf ranges, prefixes before the global + * leaf minimum, and suffixes after the global leaf minimum. Interior ranges + * that cannot be represented by prefix/suffix records return `npos` so the + * caller can scan original values. + */ + std::size_t arg_min(std::size_t slot_left, + std::size_t slot_right, + std::size_t entry_count) const { + if (slot_left >= slot_right || slot_right > entry_count || + entry_count > kMaskEntries) { + return npos; + } + if (slot_left + 1 == slot_right) { + return slot_left; + } + + const std::size_t min_offset = embedded_min_offset(); + if (slot_left <= min_offset && min_offset < slot_right) { + return min_offset; + } + if (slot_left == 0 && slot_right <= min_offset) { + return previous_set_bit_before(slot_right); + } + if (slot_right == entry_count && min_offset < slot_left) { + return next_set_bit_at_or_after(slot_left); + } + return npos; + } + + /** + * @brief Pack the local minimum offset into the high bits of the mask. + */ + void set_embedded_min_offset(std::size_t offset) { + words_[kOffsetWord] = + (words_[kOffsetWord] & kMaskWordMask) | + ((static_cast(offset) & kOffsetMask) << kOffsetShift); + } + + /** + * @brief Return the local minimum offset packed into the mask words. + */ + std::size_t embedded_min_offset() const { + return static_cast((words_[kOffsetWord] >> kOffsetShift) & + kOffsetMask); + } + + private: + static constexpr std::size_t kMaskEntries = LeafSize; + static constexpr std::size_t kOffsetBits = LeafSize == 496 ? 16 : 8; + static constexpr std::size_t kPackedBits = kMaskEntries + kOffsetBits; + static constexpr std::size_t kWordCount = (kPackedBits + 63) / 64; + static constexpr std::size_t kOffsetWord = kMaskEntries / 64; + static constexpr std::size_t kOffsetShift = kMaskEntries & 63; + + /** + * @brief Return a mask with the lowest @p bits set. + */ + static constexpr std::uint64_t low_bits_mask(std::size_t bits) { + if (bits == 0) { + return 0; + } + if (bits >= 64) { + return std::numeric_limits::max(); + } + return (std::uint64_t{1} << bits) - 1; + } + + static constexpr std::uint64_t kOffsetMask = low_bits_mask(kOffsetBits); + static constexpr std::uint64_t kMaskWordMask = low_bits_mask(kOffsetShift); + + static_assert(!kMaskLeafSelector || kPackedBits % 64 == 0); + static_assert(!kMaskLeafSelector || kOffsetShift + kOffsetBits == 64); + + /** + * @brief Set the prefix/suffix-record bit for a local leaf slot. + */ + void set_mask_bit(std::size_t slot) { + words_[slot >> 6] |= std::uint64_t{1} << (slot & 63); + } + + /** + * @brief Return a mask word with embedded offset bits hidden. + */ + std::uint64_t mask_word(std::size_t word) const { + return word == kOffsetWord ? words_[word] & kMaskWordMask : words_[word]; + } + + /** + * @brief Return the last set record bit strictly before @p limit. + */ + std::size_t previous_set_bit_before(std::size_t limit) const { + if (limit == 0) { + return npos; + } + + std::size_t word = (limit - 1) >> 6; + std::uint64_t bits = + mask_word(word) & first_bits_mask(((limit - 1) & 63) + 1); + while (true) { + if (bits != 0) { + return word * 64 + 63 - std::countl_zero(bits); + } + if (word == 0) { + break; + } + --word; + bits = mask_word(word); + } + return npos; + } + + /** + * @brief Return the first set record bit at or after @p slot. + */ + std::size_t next_set_bit_at_or_after(std::size_t slot) const { + std::size_t word = slot >> 6; + std::uint64_t bits = mask_word(word) & ~first_bits_mask(slot & 63); + while (word < kWordCount) { + if (bits != 0) { + const std::size_t result = word * 64 + std::countr_zero(bits); + return result < kMaskEntries ? result : npos; + } + ++word; + bits = word < kWordCount ? mask_word(word) : 0; + } + return npos; + } + + std::array words_{}; + }; + + static_assert(!kMaskLeafSelector || sizeof(PrefixSuffixMaskLeafSelector) == + (LeafSize == 496 ? 64 : 32)); + static_assert(!kMaskLeafSelector || alignof(PrefixSuffixMaskLeafSelector) == + (LeafSize == 496 ? 64 : 32)); + + using LeafSelector = std::conditional_t; + + /** + * @brief Return whether a stored position is one of the missing sentinels. + */ + bool missing_position(std::size_t position) const { + if constexpr (kInvalidIndexEqualsNpos) { + return position == npos; + } else { + return position == npos || + position == static_cast(invalid_index); + } + } + + /** + * @brief Wrap an original value position as a comparable candidate. + */ + MinCandidate value_candidate(std::size_t position) const { + if (missing_position(position) || position >= values_.size()) { + return {}; + } + return {position, values_.data() + position}; + } + + /** + * @brief Return a node's cached subtree-minimum candidate. + */ + MinCandidate subtree_min_candidate(std::size_t level, + std::size_t node) const { + const std::size_t position = subtree_min_position(level, node); + if (missing_position(position)) { + return {}; + } + return {position, &subtree_min_value(level, node)}; + } + + /** + * @brief Return a child slot's subtree-minimum candidate. + */ + MinCandidate subtree_child_min_candidate(std::size_t level, + std::size_t node, + std::size_t slot) const { + const std::size_t child_level = level - 1; + return subtree_min_candidate(child_level, + node * fanout_at_level(level) + slot); + } + + /** + * @brief Return whether @p left is strictly better than @p right. + */ + bool strictly_better_candidate(MinCandidate left, MinCandidate right) const { + if (missing_position(left.position)) { + return false; + } + if (missing_position(right.position)) { + return true; + } + return compare_(*left.value, *right.value); + } + + /** + * @brief Choose the better candidate using value, then smaller position. + */ + MinCandidate better_candidate(MinCandidate left, MinCandidate right) const { + if (missing_position(left.position)) { + return right; + } + if (missing_position(right.position)) { + return left; + } + if (compare_(*right.value, *left.value)) { + return right; + } + if (compare_(*left.value, *right.value)) { + return left; + } + return right.position < left.position ? right : left; + } + + /** + * @brief Compare two regular child slots while building a local selector. + */ + bool strictly_better_subtree_child_slot(std::size_t level, + std::size_t node, + std::size_t left_slot, + std::size_t right_slot) const { + return strictly_better_candidate( + subtree_child_min_candidate(level, node, left_slot), + subtree_child_min_candidate(level, node, right_slot)); + } + + /** + * @brief Build all levels, selectors, and cached minimum metadata. + */ + void build() { + leaf_selectors_.clear(); + medium_selectors_.clear(); + medium_min_values_.clear(); + top_sparse_candidates_.clear(); + top_block_size_ = kMinTopSparseBlockSize; + top_block_count_ = 0; + top_sparse_levels_ = 0; + medium_level_offsets_.clear(); + level_sizes_.clear(); + level_value_spans_.clear(); + level_fanouts_.clear(); + if (values_.empty()) { + return; + } + if (values_.size() > static_cast(invalid_index)) { + throw std::length_error("HybridBTree index type is too small"); + } + + initialize_layout((values_.size() + LeafSize - 1) / LeafSize); + for (std::size_t leaf = 0; leaf < level_sizes_[0]; ++leaf) { + build_leaf(leaf); + } + + for (std::size_t level = 1; level < level_count(); ++level) { + for (std::size_t node = 0; node < level_sizes_[level]; ++node) { + build_internal_node(level, node); + } + } + build_top_sparse_table(); + } + + /** + * @brief Compute level sizes, fanouts, flat offsets, and cache storage. + * + * @details Level zero always stores leaf selectors. Every internal level uses + * the fixed 192-way middle-node layout. + */ + void initialize_layout(std::size_t leaf_count) { + level_sizes_.push_back(leaf_count); + level_value_spans_.push_back(LeafSize); + level_fanouts_.push_back(0); + + std::size_t current_count = leaf_count; + std::size_t current_span = LeafSize; + while (current_count > 1) { + level_fanouts_.push_back(kMiddleFanout); + current_count = ceil_div(current_count, kMiddleFanout); + current_span = saturating_product(current_span, kMiddleFanout); + level_sizes_.push_back(current_count); + level_value_spans_.push_back(current_span); + } + + leaf_selectors_.resize(level_sizes_[0]); + + medium_level_offsets_.assign(level_count(), 0); + if (level_count() <= 1) { + return; + } + + std::size_t medium_node_count = 0; + for (std::size_t level = 1; level < level_count(); ++level) { + medium_level_offsets_[level] = medium_node_count; + medium_node_count += level_sizes_[level]; + } + medium_selectors_.resize(medium_node_count); + medium_min_values_.reserve(medium_node_count); + } + + /** + * @brief Build one leaf selector over original values. + */ + void build_leaf(std::size_t leaf) { + LeafSelector& selector = leaf_selectors_[leaf]; + const std::size_t begin = node_value_begin(0, leaf); + const std::size_t count = entry_count(0, leaf); + selector.build(count, [&](std::size_t left, std::size_t right) { + return compare_(values_[begin + left], values_[begin + right]); + }); + + if constexpr (kBpLeafSelector) { + const std::size_t slot = selector.arg_min(0, count, count); + selector.set_embedded_min_offset(slot); + } + } + + /** + * @brief Build one internal node selector and its cached minima. + */ + void build_internal_node(std::size_t level, std::size_t node) { + Bp512Selector& selector = mutable_selector_at(level, node); + const std::size_t count = entry_count(level, node); + const std::size_t first_child = node * fanout_at_level(level); + + selector.build(count, [&](std::size_t left, std::size_t right) { + return strictly_better_subtree_child_slot(level, node, left, right); + }); + + const std::size_t slot = selector.arg_min(0, count, count); + const std::size_t min_position = + subtree_min_position(level - 1, first_child + slot); + selector.set_embedded_min_position(min_position); + medium_min_values_.push_back(values_[min_position]); + selector.build_zero_prefix_metadata(2 * count); + } + + /** + * @brief Multiply two extents, saturating at `std::size_t` maximum. + */ + static std::size_t saturating_product(std::size_t left, std::size_t right) { + if (left != 0 && right > std::numeric_limits::max() / left) { + return std::numeric_limits::max(); + } + return left * right; + } + + /** + * @brief Return `ceil(value / divisor)` for positive divisors. + */ + static std::size_t ceil_div(std::size_t value, std::size_t divisor) { + return value == 0 ? 0 : 1 + (value - 1) / divisor; + } + + /** + * @brief Return the first minimum position through the B-tree fallback. + */ + std::size_t tree_arg_min(std::size_t left, std::size_t right) const { + const std::size_t root_level = level_count() - 1; + if (left == 0 && right == values_.size()) { + return subtree_min_position(root_level, 0); + } + + const std::size_t left_leaf = leaf_for_value(left); + const std::size_t right_leaf = leaf_for_value(right - 1); + if (left_leaf == right_leaf) { + return leaf_range_min(left_leaf, left, right); + } + + const auto [level, node_index] = covering_node(left_leaf, right_leaf); + return query_node(level, node_index, left, right).position; + } + + /** + * @brief Return the first minimum position for a possibly partial leaf. + * + * @details Full leaves use the embedded minimum. Mask leaves answer prefix + * and suffix ranges through their record bits and fall back to scanning only + * for unsupported interior ranges. + */ + std::size_t leaf_range_min(std::size_t leaf, + std::size_t left, + std::size_t right) const { + if (left >= right) { + return npos; + } + + const std::size_t begin = node_value_begin(0, leaf); + const std::size_t end = node_value_end(0, leaf); + if (left <= begin && end <= right) { + return subtree_min_position(0, leaf); + } + const std::size_t slot_left = left - begin; + const std::size_t slot_right = right - begin; + + if constexpr (kMaskLeafSelector) { + const std::size_t slot = leaf_selectors_[leaf].arg_min( + slot_left, slot_right, entry_count(0, leaf)); + if (slot != npos) { + return begin + slot; + } + return linear_range_min(left, right); + } else { + if (right - left <= kLeafLinearScanThreshold) { + return linear_range_min(left, right); + } + + const std::size_t slot = leaf_selectors_[leaf].arg_min( + slot_left, slot_right, entry_count(0, leaf)); + if (slot == npos) { + return npos; + } + return begin + slot; + } + } + + /** + * @brief Scan original values to answer a range inside a leaf. + */ + std::size_t linear_range_min(std::size_t left, std::size_t right) const { + if (left >= right) { + return npos; + } +#ifdef PIXIE_AVX2_SUPPORT + if constexpr (std::is_same_v && + std::is_same_v>) { + if (right - left >= kLeafAvx2ScanThreshold) { + return linear_range_min_i64_avx2(left, right); + } + } +#endif + std::size_t best = left; + for (std::size_t position = left + 1; position < right; ++position) { + if (compare_(values_[position], values_[best])) { + best = position; + } + } + return best; + } + +#ifdef PIXIE_AVX2_SUPPORT + /** + * @brief AVX2 scan for signed 64-bit minimum ranges. + * + * @details Vector lanes track the best value and first position seen so far. + * Equal values keep the earlier position by only replacing lanes on strict + * value improvement. + */ + std::size_t linear_range_min_i64_avx2(std::size_t left, + std::size_t right) const { + const std::int64_t* data = values_.data(); + std::size_t position = left; + + __m256i best_values = + _mm256_loadu_si256(reinterpret_cast(data + position)); + __m256i best_positions = _mm256_set_epi64x( + static_cast(position + 3), + static_cast(position + 2), + static_cast(position + 1), static_cast(position)); + position += 4; + + for (; position + 4 <= right; position += 4) { + const __m256i values = + _mm256_loadu_si256(reinterpret_cast(data + position)); + const __m256i positions = + _mm256_set_epi64x(static_cast(position + 3), + static_cast(position + 2), + static_cast(position + 1), + static_cast(position)); + const __m256i take_new = _mm256_cmpgt_epi64(best_values, values); + best_values = _mm256_blendv_epi8(best_values, values, take_new); + best_positions = _mm256_blendv_epi8(best_positions, positions, take_new); + } + + alignas(32) std::int64_t value_lanes[4]; + alignas(32) std::uint64_t position_lanes[4]; + _mm256_store_si256(reinterpret_cast<__m256i*>(value_lanes), best_values); + _mm256_store_si256(reinterpret_cast<__m256i*>(position_lanes), + best_positions); + + std::int64_t best_value = value_lanes[0]; + std::size_t best_position = static_cast(position_lanes[0]); + for (std::size_t lane = 1; lane < 4; ++lane) { + const std::size_t lane_position = + static_cast(position_lanes[lane]); + if (value_lanes[lane] < best_value || + (value_lanes[lane] == best_value && lane_position < best_position)) { + best_value = value_lanes[lane]; + best_position = lane_position; + } + } + + for (; position < right; ++position) { + if (data[position] < best_value) { + best_value = data[position]; + best_position = position; + } + } + return best_position; + } +#endif + + /** + * @brief Find the lowest tree node that contains both endpoint leaves. + */ + std::pair covering_node( + std::size_t left_leaf, + std::size_t right_leaf) const { + std::size_t level = 0; + std::size_t left_node = left_leaf; + std::size_t right_node = right_leaf; + while (left_node != right_node) { + ++level; + const std::size_t fanout = fanout_at_level(level); + left_node /= fanout; + right_node /= fanout; + } + return {level, left_node}; + } + + /** + * @brief Return the leaf index containing an original value position. + */ + std::size_t leaf_for_value(std::size_t position) const { + return position / LeafSize; + } + + /** + * @brief Return the child index at @p child_level containing a value. + */ + std::size_t child_for_value(std::size_t child_level, + std::size_t position) const { + return position / level_value_spans_[child_level]; + } + + /** + * @brief Query a child-slot range of an internal node. + * + * @details The node selector first picks the best child in the requested slot + * range. If that child cannot be accepted directly because only a border part + * is queried, the two border children are queried recursively and the middle + * full-child range is answered by the current node selector. + */ + MinCandidate query_child_slots(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t left, + std::size_t right) const { + if (slot_left >= slot_right) { + return {}; + } + + const std::size_t count = entry_count(level, node); + const std::size_t slot = + slot_left + 1 == slot_right + ? slot_left + : selector_arg_min(level, node, slot_left, slot_right, count); + if (slot == npos) { + return {}; + } + + const std::size_t child_level = level - 1; + const std::size_t first_child = node * fanout_at_level(level); + const std::size_t child = first_child + slot; + const MinCandidate child_min = subtree_min_candidate(child_level, child); + const std::size_t child_begin = node_value_begin(child_level, child); + const std::size_t child_end = node_value_end(child_level, child); + if ((left <= child_begin && child_end <= right) || + contains_position(left, right, child_min.position)) { + return child_min; + } + + const std::size_t last_slot = slot_right - 1; + const std::size_t left_child_begin = + node_value_begin(child_level, first_child + slot_left); + const std::size_t left_child_end = + node_value_end(child_level, first_child + slot_left); + MinCandidate answer = query_node(child_level, first_child + slot_left, + std::max(left, left_child_begin), + std::min(right, left_child_end)); + + if (slot_left != last_slot) { + const std::size_t right_child_begin = + node_value_begin(child_level, first_child + last_slot); + const std::size_t right_child_end = + node_value_end(child_level, first_child + last_slot); + answer = better_candidate(answer, + query_node(child_level, first_child + last_slot, + std::max(left, right_child_begin), + std::min(right, right_child_end))); + } + + if (slot_left + 1 < last_slot) { + answer = better_candidate( + answer, + full_child_slot_range_min(level, node, slot_left + 1, last_slot)); + } + + return answer; + } + + /** + * @brief Return the best candidate among fully covered child slots. + */ + MinCandidate full_child_slot_range_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right) const { + if (slot_left >= slot_right) { + return {}; + } + + const std::size_t slot = + slot_left + 1 == slot_right + ? slot_left + : selector_arg_min(level, node, slot_left, slot_right, + entry_count(level, node)); + if (slot == npos) { + return {}; + } + + return subtree_child_min_candidate(level, node, slot); + } + + /** + * @brief Query a tree node for the minimum candidate in a value range. + */ + MinCandidate query_node(std::size_t level, + std::size_t node, + std::size_t left, + std::size_t right) const { + if (left >= right) { + return {}; + } + const std::size_t begin = node_value_begin(level, node); + const std::size_t end = node_value_end(level, node); + if (left <= begin && end <= right) { + return subtree_min_candidate(level, node); + } + if (level == 0) { + return value_candidate(leaf_range_min(node, left, right)); + } + + const std::size_t child_level = level - 1; + const std::size_t left_child = child_for_value(child_level, left); + const std::size_t right_child = child_for_value(child_level, right - 1); + const std::size_t first_child = node * fanout_at_level(level); + const std::size_t left_slot = left_child - first_child; + const std::size_t right_slot = right_child - first_child + 1; + return query_child_slots(level, node, left_slot, right_slot, left, right); + } + + /** + * @brief Return whether a position lies inside a half-open range. + */ + bool contains_position(std::size_t left, + std::size_t right, + std::size_t position) const { + return !missing_position(position) && left <= position && position < right; + } + + /** + * @brief Return the number of B-tree levels, including leaves. + */ + std::size_t level_count() const { return level_sizes_.size(); } + + /** + * @brief Return the number of entries in a leaf or child slots in a node. + */ + std::size_t entry_count(std::size_t level, std::size_t node) const { + if (level == 0) { + const std::size_t begin = node_value_begin(0, node); + return std::min(LeafSize, values_.size() - begin); + } + const std::size_t first_child = node * fanout_at_level(level); + return std::min(fanout_at_level(level), + level_sizes_[level - 1] - first_child); + } + + /** + * @brief Return the first original value position covered by a node. + */ + std::size_t node_value_begin(std::size_t level, std::size_t node) const { + return node * level_value_spans_[level]; + } + + /** + * @brief Return one past the last original value position covered by a node. + */ + std::size_t node_value_end(std::size_t level, std::size_t node) const { + return std::min(values_.size(), + node_value_begin(level, node) + level_value_spans_[level]); + } + + /** + * @brief Return a node's absolute subtree-minimum position. + */ + std::size_t subtree_min_position(std::size_t level, std::size_t node) const { + if (level == 0) { + return node_value_begin(0, node) + + leaf_selectors_[node].embedded_min_offset(); + } + return selector_at(level, node).embedded_min_position(); + } + + /** + * @brief Return a cached reference to a node's subtree-minimum value. + */ + const T& subtree_min_value(std::size_t level, std::size_t node) const { + if (level == 0) { + return values_[subtree_min_position(0, node)]; + } + return medium_min_values_[medium_flat_index(level, node)]; + } + + /** + * @brief Return an immutable internal-node BP selector. + */ + const Bp512Selector& selector_at(std::size_t level, std::size_t node) const { + return medium_selectors_[medium_flat_index(level, node)]; + } + + /** + * @brief Return a mutable internal-node BP selector while building. + */ + Bp512Selector& mutable_selector_at(std::size_t level, std::size_t node) { + return medium_selectors_[medium_flat_index(level, node)]; + } + + /** + * @brief Run the appropriate local selector for an internal node. + */ + std::size_t selector_arg_min(std::size_t level, + std::size_t node, + std::size_t slot_left, + std::size_t slot_right, + std::size_t count) const { + const Bp512Selector& selector = selector_at(level, node); + return selector.arg_min_with_zero_prefix(slot_left, slot_right, count); + } + + /** + * @brief Map a medium-level node to its flat storage index. + */ + std::size_t medium_flat_index(std::size_t level, std::size_t node) const { + return medium_level_offsets_[level] + node; + } + + /** + * @brief Return the fanout used to group children at a level. + */ + std::size_t fanout_at_level(std::size_t level) const { + return level_fanouts_[level]; + } + + /** + * @brief Build a single top sparse table over original-value block minima. + */ + void build_top_sparse_table() { + top_sparse_candidates_.clear(); + top_block_size_ = top_sparse_block_size_for(values_.size()); + top_block_count_ = top_sparse_block_count_for(values_.size()); + top_sparse_levels_ = + top_block_count_ == 0 ? 0 : std::bit_width(top_block_count_); + if (top_block_count_ == 0) { + return; + } + + top_sparse_candidates_.assign(top_sparse_levels_ * top_block_count_, + TopCandidate{}); + for (std::size_t block = 0; block < top_block_count_; ++block) { + const std::size_t begin = block * top_block_size_; + const std::size_t end = std::min(values_.size(), begin + top_block_size_); + std::size_t minimum = begin; + for (std::size_t position = begin + 1; position < end; ++position) { + if (strictly_better_value_position(position, minimum)) { + minimum = position; + } + } + top_sparse_candidates_[block] = make_top_candidate(minimum); + } + + for (std::size_t level = 1; level < top_sparse_levels_; ++level) { + const std::size_t span = std::size_t{1} << level; + const std::size_t half_span = span >> 1; + TopCandidate* current = + top_sparse_candidates_.data() + level * top_block_count_; + const TopCandidate* previous = + top_sparse_candidates_.data() + (level - 1) * top_block_count_; + for (std::size_t block = 0; block + span <= top_block_count_; ++block) { + current[block] = + better_top_candidate(previous[block], previous[block + half_span]); + } + } + } + + /** + * @brief Wrap an original value position as a top sparse-table candidate. + */ + TopCandidate make_top_candidate(std::size_t position) const { + if (!valid_value_position(position)) { + return {}; + } + return {static_cast(position)}; + } + + /** + * @brief Return whether @p position is a valid original-value index. + */ + bool valid_value_position(std::size_t position) const { + return !missing_position(position) && position < values_.size(); + } + + /** + * @brief Return whether value position @p left is strictly better than @p + * right. + */ + bool strictly_better_value_position(std::size_t left, + std::size_t right) const { + if (!valid_value_position(left)) { + return false; + } + if (!valid_value_position(right)) { + return true; + } + if (compare_(values_[left], values_[right])) { + return true; + } + if (compare_(values_[right], values_[left])) { + return false; + } + return left < right; + } + + /** + * @brief Choose the better original-value candidate, preserving first ties. + */ + TopCandidate better_top_candidate(TopCandidate left, + TopCandidate right) const { + const std::size_t left_position = static_cast(left.position); + const std::size_t right_position = static_cast(right.position); + return strictly_better_value_position(right_position, left_position) ? right + : left; + } + + /** + * @brief Return the sparse-table candidate over a top-block range. + */ + TopCandidate top_sparse_block_arg_min(std::size_t block_left, + std::size_t block_right) const { + if (block_left >= block_right || block_right > top_block_count_ || + top_sparse_levels_ == 0) { + return {}; + } + const std::size_t length = block_right - block_left; + const std::size_t level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << level; + const TopCandidate* table = + top_sparse_candidates_.data() + level * top_block_count_; + return better_top_candidate(table[block_left], table[block_right - span]); + } + + /** + * @brief Return whether a candidate lies inside a half-open value range. + */ + bool top_candidate_inside(TopCandidate candidate, + std::size_t left, + std::size_t right) const { + const std::size_t position = static_cast(candidate.position); + return valid_value_position(position) && left <= position && + position < right; + } + + /** + * @brief Return the top-overlay answer, or `npos` when the tree should run. + */ + std::size_t top_sparse_arg_min(std::size_t left, std::size_t right) const { + if (top_block_count_ <= 1) { + return npos; + } + + const std::size_t padded_block_left = left / top_block_size_; + const std::size_t padded_block_right = (right - 1) / top_block_size_ + 1; + if (padded_block_left + 1 >= padded_block_right) { + return npos; + } + + const TopCandidate padded = + top_sparse_block_arg_min(padded_block_left, padded_block_right); + if (top_candidate_inside(padded, left, right)) { + return static_cast(padded.position); + } + + const std::size_t first_full_block = + (left + top_block_size_ - 1) / top_block_size_; + const std::size_t full_block_right = right / top_block_size_; + if (first_full_block >= full_block_right) { + return npos; + } + + TopCandidate answer = + top_sparse_block_arg_min(first_full_block, full_block_right); + + const std::size_t left_border_end = first_full_block * top_block_size_; + if (left < left_border_end) { + answer = better_top_candidate( + answer, make_top_candidate(tree_arg_min(left, left_border_end))); + } + + const std::size_t right_border_begin = full_block_right * top_block_size_; + if (right_border_begin < right) { + answer = better_top_candidate( + answer, make_top_candidate(tree_arg_min(right_border_begin, right))); + } + + return valid_value_position(static_cast(answer.position)) + ? static_cast(answer.position) + : npos; + } + + std::span values_; + Compare compare_; + std::vector leaf_selectors_; + std::vector medium_selectors_; + std::vector medium_min_values_; + std::vector top_sparse_candidates_; + std::vector medium_level_offsets_; + std::vector level_sizes_; + std::vector level_value_spans_; + std::vector level_fanouts_; + std::size_t top_block_size_ = kMinTopSparseBlockSize; + std::size_t top_block_count_ = 0; + std::size_t top_sparse_levels_ = 0; +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/rmq_base.h b/include/pixie/rmq/rmq_base.h new file mode 100644 index 0000000..0008b4b --- /dev/null +++ b/include/pixie/rmq/rmq_base.h @@ -0,0 +1,97 @@ +#pragma once + +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief CRTP facade for static range-minimum-query indexes. + * + * Implementations are non-owning indexes over an external random-access array. + * Queries use half-open zero-based ranges `[left, right)` and return the first + * position attaining the minimum. Invalid or empty ranges return `npos`. + */ +template +class RmqBase { + public: + /** + * @brief Sentinel returned when no valid query answer exists. + */ + static constexpr std::size_t npos = std::numeric_limits::max(); + + /** + * @brief Number of indexed values. + * + * @return The number of values covered by the underlying RMQ index. + */ + std::size_t size() const { return impl().size_impl(); } + + /** + * @brief Whether the indexed array is empty. + * + * @return `true` when `size() == 0`. + */ + bool empty() const { return size() == 0; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + * + * @details The query range is half-open. Ties are resolved by returning the + * smallest position attaining the minimum. Invalid or empty ranges return + * `npos`. + * + * @param left First position in the query range. + * @param right One past the last position in the query range. + * @return Zero-based position of the first range minimum, or `npos`. + */ + std::size_t arg_min(std::size_t left, std::size_t right) const { + return impl().arg_min_impl(left, right); + } + + /** + * @brief Return the minimum value in [@p left, @p right). + * + * @details Invalid or empty ranges return a default-constructed value. + * + * @param left First position in the query range. + * @param right One past the last position in the query range. + * @return The minimum value in the half-open range, or `Value{}`. + */ + Value range_min(std::size_t left, std::size_t right) const { + const std::size_t position = arg_min(left, right); + if (position == npos) { + return Value{}; + } + return impl().value_at_impl(position); + } + + /** + * @brief Return owned auxiliary memory usage in bytes when implemented. + * + * @details Implementations opt in by exposing + * `memory_usage_bytes_impl() const`. The reported value should include the + * index object itself and heap buffers owned by the index, but not the + * external value array indexed by the RMQ. + */ + std::size_t memory_usage_bytes() const + requires requires(const Impl& concrete) { + { + concrete.memory_usage_bytes_impl() + } -> std::convertible_to; + } + { + return impl().memory_usage_bytes_impl(); + } + + private: + /** + * @brief Return this object as its concrete CRTP implementation. + * + * @return Reference to the derived RMQ implementation. + */ + const Impl& impl() const { return static_cast(*this); } +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/sdsl_sct.h b/include/pixie/rmq/sdsl_sct.h new file mode 100644 index 0000000..329ad75 --- /dev/null +++ b/include/pixie/rmq/sdsl_sct.h @@ -0,0 +1,115 @@ +#pragma once + +#ifndef SDSL_SUPPORT +#error "pixie/rmq/sdsl_sct.h requires SDSL_SUPPORT" +#endif + +#include + +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief Optional value-RMQ adapter over `sdsl::rmq_succinct_sct`. + * + * @details SDSL's SCT implementation builds a global Super-Cartesian-tree + * balanced-parentheses sequence (BPS-SCT) over the indexed values. It then + * answers inclusive range queries with balanced-parentheses navigation + * operations such as `select`, `find_close`, `rr_enclose`, and `rank`. + * + * SDSL's helper comments attribute the Super-Cartesian-tree BP construction to + * Ohlebusch and Gog in a compressed enhanced suffix-array context. That is not + * the canonical RMQ citation. For the succinct RMQ result behind this style of + * representation, see: + * + * Johannes Fischer, "Optimal Succinctness for Range Minimum Queries", + * LATIN 2010; arXiv:0812.2775. + * + * This adapter exposes the Pixie value-RMQ interface specification: half-open + * ranges `[left, right)`, invalid ranges returning `npos`, and first-minimum + * tie semantics. + * + * The SDSL structure does not retain values after construction, so this adapter + * keeps a non-owning span for `range_min()`. The indexed values must outlive + * the adapter. SDSL SCT supports min/max through a boolean template parameter, + * not arbitrary comparators; this wrapper currently exposes only + * `std::less`. + * + * @tparam T Value type in the indexed array. + * @tparam Compare Must be `std::less`. + * @tparam Index Interface compatibility parameter for benchmark templates. + */ +template , class Index = std::size_t> +class SdslSct : public RmqBase, T> { + public: + static_assert(std::is_same_v>, + "SDSL SCT RMQ wrapper supports only std::less"); + + static constexpr std::size_t npos = + RmqBase, T>::npos; + + /** + * @brief Construct an empty SDSL SCT RMQ adapter. + */ + SdslSct() = default; + + /** + * @brief Build the SDSL SCT index over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values keep the smaller index as the RMQ answer. + * + * @param values Values to index. + * @param compare Must be the default `std::less` comparator. + */ + explicit SdslSct(std::span values, Compare compare = Compare()) + : values_(values), rmq_(&values_) { + (void)compare; + } + + /** + * @brief Return the number of indexed values. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + * + * @details SDSL's query is inclusive, so the adapter translates the Pixie + * half-open range to `[left, right - 1]`. + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size()) { + return npos; + } + return static_cast(rmq_(left, right - 1)); + } + + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts the wrapper object plus SDSL's serialized SCT byte count. + * The external input values are not owned and are excluded. + */ + std::size_t memory_usage_bytes_impl() const { + sdsl::nullstream out; + return sizeof(*this) + static_cast(rmq_.serialize(out)); + } + + private: + std::span values_; + sdsl::rmq_succinct_sct rmq_; +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/segment_tree.h b/include/pixie/rmq/segment_tree.h new file mode 100644 index 0000000..86d4125 --- /dev/null +++ b/include/pixie/rmq/segment_tree.h @@ -0,0 +1,200 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief Static iterative segment-tree RMQ baseline. + * + * @details Stores the index of the first minimum for each segment in a flat + * binary tree. Query time is O(log n), build time is O(n), and storage is O(n) + * indices. The indexed values are not owned and must outlive this object. + * + * @tparam T Value type in the indexed array. + * @tparam Compare Strict weak ordering used to choose minima. + * @tparam Index Unsigned integer type used for stored positions. + */ +template , class Index = std::size_t> +class SegmentTree : public RmqBase, T> { + public: + static constexpr std::size_t npos = + RmqBase, T>::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + + /** + * @brief Construct an empty segment tree. + */ + SegmentTree() = default; + + /** + * @brief Build an iterative segment tree over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values keep the smaller index as the RMQ answer. + * + * @param values Values to index. + * @param compare Ordering used to choose minima. + * @throws std::length_error if @p Index cannot represent all positions. + */ + explicit SegmentTree(std::span values, Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + /** + * @brief Return the number of indexed values. + * + * @return `values.size()` from construction. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + * + * @param position Zero-based position in the indexed values. + * @return Copy of the value at @p position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + * + * @details Answers in O(log n) by walking the flat iterative segment tree. + * Ties return the smaller position. + * + * @param left First position in the query range. + * @param right One past the last position in the query range. + * @return Zero-based position of the first range minimum, or `npos`. + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size()) { + return npos; + } + + left += leaf_base_; + right += leaf_base_; + std::size_t answer = npos; + while (left < right) { + if ((left & 1u) != 0) { + answer = better(answer, tree_[left]); + ++left; + } + if ((right & 1u) != 0) { + --right; + answer = better(answer, tree_[right]); + } + left >>= 1; + right >>= 1; + } + return answer; + } + + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this segment-tree object and its flat index buffer. The + * external input values are not owned and are excluded. + */ + std::size_t memory_usage_bytes_impl() const { + return sizeof(*this) + pixie::vector_capacity_bytes(tree_); + } + + private: + /** + * @brief Choose the better of two candidate positions. + * + * @details `npos` and `invalid_index` are treated as missing. If both values + * compare equal, the smaller position wins to preserve first-minimum + * semantics. + * + * @param left First candidate position, `npos`, or `invalid_index`. + * @param right Second candidate position, `npos`, or `invalid_index`. + * @return Position of the selected candidate. + */ + std::size_t better(std::size_t left, std::size_t right) const { + if (left == npos || left == invalid_index) { + return right; + } + if (right == npos || right == invalid_index) { + return left; + } + if (compare_(values_[right], values_[left])) { + return right; + } + if (compare_(values_[left], values_[right])) { + return left; + } + return std::min(left, right); + } + + /** + * @brief Choose the better child while building the tree. + * + * @details Child ranges are disjoint and ordered, so ties keep the left + * child. Missing tail leaves use `invalid_index`. + * + * @param left Candidate from the left child. + * @param right Candidate from the right child. + * @return Candidate for the parent node, or `invalid_index`. + */ + Index build_better(Index left, Index right) const { + if (left == invalid_index) { + return right; + } + if (right == invalid_index) { + return left; + } + return compare_(values_[right], values_[left]) ? right : left; + } + + /** + * @brief Build the flat iterative segment tree. + * + * @details Leaves start at `leaf_base_`, which is the next power of two. + * Unused leaves contain `invalid_index`, and internal nodes store the first + * minimum position of their covered segment. + * + * @throws std::length_error if @p Index cannot represent all positions. + */ + void build() { + tree_.clear(); + leaf_base_ = 0; + if (values_.empty()) { + return; + } + if (values_.size() > static_cast(invalid_index)) { + throw std::length_error("RMQ segment tree index type is too small"); + } + + leaf_base_ = std::bit_ceil(values_.size()); + tree_.clear(); + tree_.resize(2 * leaf_base_); + for (std::size_t i = 0; i < values_.size(); ++i) { + tree_[leaf_base_ + i] = static_cast(i); + } + std::fill(tree_.begin() + leaf_base_ + values_.size(), tree_.end(), + invalid_index); + for (std::size_t node = leaf_base_; node > 1;) { + --node; + tree_[node] = build_better(tree_[node << 1], tree_[(node << 1) | 1]); + } + } + + std::span values_; + Compare compare_; + std::size_t leaf_base_ = 0; + std::vector tree_; +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/sparse_table.h b/include/pixie/rmq/sparse_table.h new file mode 100644 index 0000000..e6b9566 --- /dev/null +++ b/include/pixie/rmq/sparse_table.h @@ -0,0 +1,250 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::rmq { + +/** + * @brief Static sparse-table RMQ baseline. + * + * @details Stores the index of the first minimum for each power-of-two range. + * Query time is O(1), build time is O(n log n), and storage is O(n log n) + * indices. The indexed values are not owned and must outlive this object. + * + * @tparam T Value type in the indexed array. + * @tparam Compare Strict weak ordering used to choose minima. + * @tparam Index Unsigned integer type used for stored positions. + * @tparam Alignment Byte alignment for each sparse-table level allocation. + * Use 0 to select the standard allocator. + */ +template , + class Index = std::size_t, + std::size_t Alignment = pixie::kAlignedStorageLineBytes> +class SparseTable + : public RmqBase, T> { + public: + static constexpr std::size_t npos = + RmqBase, T>::npos; + static constexpr Index invalid_index = std::numeric_limits::max(); + + /** + * @brief Construct an empty sparse table. + */ + SparseTable() = default; + + /** + * @brief Build a sparse table over @p values. + * + * @details The values are not copied and must outlive this object. Equal + * values keep the smaller index as the RMQ answer. + * + * @param values Values to index. + * @param compare Ordering used to choose minima. + * @throws std::length_error if @p Index cannot represent all positions. + */ + explicit SparseTable(std::span values, Compare compare = Compare()) + : values_(values), compare_(compare) { + build(); + } + + /** + * @brief Return the number of indexed values. + * + * @return `values.size()` from construction. + */ + std::size_t size_impl() const { return values_.size(); } + + /** + * @brief Return the value at an indexed position. + * + * @param position Zero-based position in the indexed values. + * @return Copy of the value at @p position. + */ + T value_at_impl(std::size_t position) const { return values_[position]; } + + /** + * @brief Return the first minimum position in [@p left, @p right). + * + * @details Answers in O(1) by comparing the two power-of-two ranges covering + * the half-open query interval. Ties return the smaller position. + * + * @param left First position in the query range. + * @param right One past the last position in the query range. + * @return Zero-based position of the first range minimum, or `npos`. + */ + std::size_t arg_min_impl(std::size_t left, std::size_t right) const { + if (left >= right || right > values_.size()) { + return npos; + } + const std::size_t length = right - left; + const std::size_t level = std::bit_width(length) - 1; + const std::size_t span = std::size_t{1} << level; + const std::size_t first = table_[level][left]; + const std::size_t second = table_[level][right - span]; + return better(first, second); + } + + /** + * @brief Return owned auxiliary memory usage in bytes. + * + * @details Counts this sparse-table object and all stored index levels. + * The external input values are not owned and are excluded. + */ + std::size_t memory_usage_bytes_impl() const { + std::size_t bytes = sizeof(*this); + bytes += pixie::vector_capacity_bytes(table_); + for (const TableLevel& level : table_) { + bytes += pixie::vector_capacity_bytes(level); + } + return bytes; + } + + private: + /** + * @brief Choose the better of two candidate positions. + * + * @details Both candidates are valid table entries. If both values compare + * equal, the smaller position wins to preserve first-minimum semantics. + * + * @param left First candidate position. + * @param right Second candidate position. + * @return Position of the selected candidate. + */ + std::size_t better(std::size_t left, std::size_t right) const { + if (compare_(values_[right], values_[left])) { + return right; + } + if (compare_(values_[left], values_[right])) { + return left; + } + return std::min(left, right); + } + + /** + * @brief Choose the better candidate while building adjacent ranges. + * + * @details Build ranges are disjoint and ordered, so ties always keep the + * left candidate. This avoids the second strict comparison needed by the + * general query-time helper. + * + * @param left Candidate from the left half. + * @param right Candidate from the right half. + * @return First-minimum candidate for the combined range. + */ + Index build_better(Index left, Index right) const { + return compare_(values_[right], values_[left]) ? right : left; + } + + /** + * @brief Build all sparse-table levels over the indexed values. + * + * @details Level 0 stores singleton positions. Each higher level stores the + * first minimum of two adjacent half ranges from the previous level. + * + * @throws std::length_error if @p Index cannot represent all positions. + */ + void build() { + table_.clear(); + if (values_.empty()) { + return; + } + if (values_.size() > static_cast(invalid_index)) { + throw std::length_error("RMQ sparse table index type is too small"); + } + + table_.reserve(std::bit_width(values_.size())); + table_.emplace_back(values_.size()); + std::iota(table_[0].begin(), table_[0].end(), Index{0}); + + for (std::size_t span = 2, half = 1; span <= values_.size(); + half = span, span <<= 1) { + const std::size_t level = table_.size(); + table_.emplace_back(values_.size() - span + 1); + for (std::size_t i = 0; i < table_[level].size(); ++i) { + table_[level][i] = static_cast( + build_better(table_[level - 1][i], table_[level - 1][i + half])); + } + } + } + + template + class AlignedAllocator { + public: + static_assert(AllocationAlignment >= alignof(Value)); + static_assert((AllocationAlignment & (AllocationAlignment - 1)) == 0); + + using value_type = Value; + + AlignedAllocator() = default; + + template + AlignedAllocator( + const AlignedAllocator&) noexcept {} + + [[nodiscard]] Value* allocate(std::size_t count) { + if (count == 0) { + return nullptr; + } + if (count > std::numeric_limits::max() / sizeof(Value)) { + throw std::bad_array_new_length(); + } + return static_cast(::operator new( + count * sizeof(Value), std::align_val_t{AllocationAlignment})); + } + + void deallocate(Value* pointer, std::size_t) noexcept { + ::operator delete(pointer, std::align_val_t{AllocationAlignment}); + } + + template + bool operator==( + const AlignedAllocator&) const noexcept { + return true; + } + + template + bool operator!=( + const AlignedAllocator&) const noexcept { + return false; + } + + template + struct rebind { + using other = AlignedAllocator; + }; + }; + + template + struct LevelAllocator { + using type = AlignedAllocator; + }; + + template + struct LevelAllocator { + using type = std::allocator; + }; + + using TableLevel = + std::vector::type>; + + std::span values_; + Compare compare_; + std::vector table_; +}; + +} // namespace pixie::rmq diff --git a/include/pixie/rmq/utils/succinct_monotone_stack.h b/include/pixie/rmq/utils/succinct_monotone_stack.h new file mode 100644 index 0000000..acd8552 --- /dev/null +++ b/include/pixie/rmq/utils/succinct_monotone_stack.h @@ -0,0 +1,213 @@ +#pragma once + +/** + * Monotone-stack construction experiment, 2026-06-13. + * + * SIMD investigation: for the RMQ benchmark's uniform-random value arrays, the + * Cartesian construction pop-run distribution is short: roughly 50% of + * iterations pop 0 entries, 25% pop 1, 12.5% pop 2, and only about 6.25% pop + * 4 or more. A SIMD path that gathers several stack-top values would therefore + * pay gather/setup overhead on mostly short runs. The retained improvement is + * bit-parallel rather than SIMD: 64-bit key blocks plus compact non-empty block + * summaries replace the old 63-bit data/pointer-word layout. + * + * Command shape: + * taskset -c 0 ./build/release/bench_rmq + * --benchmark_filter='^rmq_build_(cartesian_rmm|cartesian_hybrid_btree|sdsl_sct)/(4194304|67108864)$' + * --benchmark_repetitions=5 + * + * CPU mean, milliseconds. + * + * | N | row | before | after | + * | ---: | ------------------------ | -----: | ------: | + * | 2^22 | CartesianRmM | 44.731 | 40.274 | + * | 2^22 | CartesianHybridBTree | 47.943 | 45.019 | + * | 2^22 | SdslSct control | 39.911 | 39.628 | + * | ---- | ------------------------ | ------ | ------- | + * | 2^26 | CartesianRmM | 750.316 | 678.925 | + * | 2^26 | CartesianHybridBTree | 772.157 | 732.582 | + * | 2^26 | SdslSct control | 642.540 | 644.627 | + */ + +#include +#include +#include +#include +#include + +namespace pixie::rmq::utils { + +/** + * @brief Construction-only stack for monotonically increasing integer keys. + * + * @details The stack stores the currently live keys in 64-bit blocks and keeps + * compact summary bitsets for non-empty blocks. It is intended for + * Cartesian-tree construction passes where pushed keys are strictly increasing. + * Under that precondition the logical stack top is the largest stored key. + * + * The helper keeps construction memory succinct in the worst case: an input of + * `capacity` keys uses one bit per key plus about 1/64 extra summary bits, + * instead of one full index per live stack entry. It does not own or inspect + * input values and deliberately does not define the Cartesian-tree tie rule; + * callers keep their existing pop predicate. + */ +class SuccinctIncreasingStack { + public: + explicit SuccinctIncreasingStack(std::size_t capacity) + : words_(word_count(capacity)), + nonempty_words_(word_count(words_.size())), + nonempty_super_words_(word_count(nonempty_words_.size())), + capacity_(capacity) {} + + /** + * @brief Whether the logical stack is empty. + */ + bool empty() const { return size_ == 0; } + + /** + * @brief Return the current top key. + * + * @pre The stack is not empty. + */ + std::size_t top() const { + assert(!empty()); + return top_key_; + } + + /** + * @brief Push a key larger than the current top key. + * + * @details Keys are expected to be pushed in strictly increasing order over + * the lifetime of this stack. The assertion checks the live top, which is the + * only ordering constraint needed by this bitset representation. + */ + void push(std::size_t key) { + assert(key < capacity_); + assert(empty() || top() < key); + + const std::size_t block = block_index(key); + words_[block] |= std::uint64_t{1} << block_position(key); + set_nonempty(block); + top_key_ = key; + ++size_; + } + + /** + * @brief Remove the current top key. + * + * @pre The stack is not empty. + */ + void pop() { + assert(!empty()); + + const std::size_t block = block_index(top_key_); + std::uint64_t word = words_[block]; + word &= ~(std::uint64_t{1} << block_position(top_key_)); + words_[block] = word; + + --size_; + if (size_ == 0) { + if (word == 0) { + clear_nonempty(block); + } + top_key_ = 0; + return; + } + + if (word != 0) { + top_key_ = block * kBlockBits + highest_bit(word); + return; + } + + clear_nonempty(block); + const std::size_t previous_block = previous_nonempty_block(block); + const std::uint64_t previous_word = words_[previous_block]; + assert(previous_word != 0); + top_key_ = previous_block * kBlockBits + highest_bit(previous_word); + } + + private: + static constexpr std::size_t kBlockBits = 64; + static constexpr std::size_t kBlockShift = 6; + static constexpr std::size_t kBlockMask = kBlockBits - 1; + + static std::size_t word_count(std::size_t bit_count) { + return (bit_count + kBlockBits - 1) / kBlockBits; + } + + static std::size_t block_index(std::size_t key) { return key >> kBlockShift; } + + static std::size_t block_position(std::size_t key) { + return key & kBlockMask; + } + + static std::size_t highest_bit(std::uint64_t word) { + assert(word != 0); + return static_cast(std::bit_width(word) - 1); + } + + static std::uint64_t low_bits(std::size_t count) { + return count == 0 ? 0 : ((std::uint64_t{1} << count) - 1); + } + + void set_nonempty(std::size_t block) { + const std::size_t summary = block_index(block); + const std::uint64_t bit = std::uint64_t{1} << block_position(block); + if ((nonempty_words_[summary] & bit) == 0) { + nonempty_words_[summary] |= bit; + nonempty_super_words_[block_index(summary)] |= std::uint64_t{1} + << block_position(summary); + } + } + + void clear_nonempty(std::size_t block) { + const std::size_t summary = block_index(block); + nonempty_words_[summary] &= ~(std::uint64_t{1} << block_position(block)); + if (nonempty_words_[summary] == 0) { + nonempty_super_words_[block_index(summary)] &= + ~(std::uint64_t{1} << block_position(summary)); + } + } + + std::size_t previous_nonempty_block(std::size_t block) const { + assert(block > 0); + std::size_t summary = block_index(block); + const std::size_t summary_position = block_position(block); + if (summary_position != 0) { + const std::uint64_t same_summary = + nonempty_words_[summary] & low_bits(summary_position); + if (same_summary != 0) { + return summary * kBlockBits + highest_bit(same_summary); + } + } + + std::size_t super = block_index(summary); + const std::size_t super_position = block_position(summary); + std::uint64_t super_mask = + super_position == 0 + ? 0 + : nonempty_super_words_[super] & low_bits(super_position); + while (super_mask == 0) { + assert(super > 0); + --super; + super_mask = nonempty_super_words_[super]; + } + + const std::size_t previous_summary = + super * kBlockBits + highest_bit(super_mask); + assert(previous_summary < nonempty_words_.size()); + const std::uint64_t previous_summary_word = + nonempty_words_[previous_summary]; + assert(previous_summary_word != 0); + return previous_summary * kBlockBits + highest_bit(previous_summary_word); + } + + std::vector words_; + std::vector nonempty_words_; + std::vector nonempty_super_words_; + std::size_t capacity_ = 0; + std::size_t size_ = 0; + std::size_t top_key_ = 0; +}; + +} // namespace pixie::rmq::utils diff --git a/scripts/coverage_report.sh b/scripts/coverage_report.sh index 84bec46..4666d08 100755 --- a/scripts/coverage_report.sh +++ b/scripts/coverage_report.sh @@ -7,11 +7,16 @@ BUILD_DIR="${ROOT_DIR}/build/coverage" cmake --preset coverage cmake --build --preset coverage +find "${BUILD_DIR}" -name "*.gcda" -delete +find "${BUILD_DIR}" -name "*.gcov" -delete +rm -f "${BUILD_DIR}/coverage.txt" "${BUILD_DIR}/gcov_files.txt" + "${BUILD_DIR}/unittests" "${BUILD_DIR}/excess_positions_tests" "${BUILD_DIR}/louds_tree_tests" "${BUILD_DIR}/dfuds_tree_tests" "${BUILD_DIR}/test_rmm" +"${BUILD_DIR}/rmq_tests" cd "${BUILD_DIR}" find . -name "*.gcda" > gcov_files.txt diff --git a/src/benchmarks/bench_rmq.cpp b/src/benchmarks/bench_rmq.cpp new file mode 100644 index 0000000..3b60633 --- /dev/null +++ b/src/benchmarks/bench_rmq.cpp @@ -0,0 +1,584 @@ +#include +#include +#include +#include + +#ifdef PIXIE_THIRD_PARTY_BENCHMARKS +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr std::uint64_t kSeed = 42; +constexpr std::size_t kQueryPoolBytes = 512 * 1024; +constexpr std::size_t kQueryCount = + kQueryPoolBytes / sizeof(std::pair); +constexpr std::int64_t kValueDatasetGlobalMinimum = + std::numeric_limits::min() / 4; +constexpr double kBenchmarkWarmupSeconds = 0.2; +constexpr double kBenchmarkMinSeconds = 2.0; +using Index = std::size_t; + +static_assert(kQueryPoolBytes % sizeof(std::pair) == + 0); + +template +concept HasRmqMemoryUsage = requires(const Rmq& rmq) { + { rmq.memory_usage_bytes() } -> std::convertible_to; +}; + +std::uint64_t splitmix64(std::uint64_t value) { + value += 0x9E3779B97F4A7C15ull; + value = (value ^ (value >> 30)) * 0xBF58476D1CE4E5B9ull; + value = (value ^ (value >> 27)) * 0x94D049BB133111EBull; + return value ^ (value >> 31); +} + +std::uint64_t mix_seed(std::uint64_t seed, std::uint64_t value) { + return splitmix64(seed ^ splitmix64(value)); +} + +class BenchmarkRepetitionSeeds { + public: + std::uint64_t repetition_index(const benchmark::State& state, + std::size_t size, + std::size_t max_width) { + std::lock_guard lock(mutex_); + Entry& entry = entries_[key_for(state, size, max_width)]; + const benchmark::IterationCount max_iterations = state.max_iterations; + if (!entry.seen) { + entry.seen = true; + entry.last_max_iterations = max_iterations; + return entry.repetition_index; + } + + if (max_iterations == entry.last_max_iterations) { + if (!entry.saw_iteration_count_change && + !entry.skipped_initial_warmup_equal) { + entry.skipped_initial_warmup_equal = true; + } else { + ++entry.repetition_index; + } + } else { + entry.saw_iteration_count_change = true; + } + entry.last_max_iterations = max_iterations; + return entry.repetition_index; + } + + private: + struct Entry { + benchmark::IterationCount last_max_iterations = 0; + std::uint64_t repetition_index = 0; + bool seen = false; + bool saw_iteration_count_change = false; + bool skipped_initial_warmup_equal = kBenchmarkWarmupSeconds == 0.0; + }; + + std::string key_for(const benchmark::State& state, + std::size_t size, + std::size_t max_width) const { + return state.name() + "/" + std::to_string(size) + "/" + + std::to_string(max_width); + } + + std::mutex mutex_; + std::unordered_map entries_; +}; + +BenchmarkRepetitionSeeds& repetition_seeds() { + static BenchmarkRepetitionSeeds seeds; + return seeds; +} + +struct SeedContext { + std::uint64_t repetition_index = 0; + std::uint64_t value_seed = 0; + std::uint64_t query_seed = 0; +}; + +SeedContext make_seed_context(const benchmark::State& state, + std::size_t size, + std::size_t max_width = 0) { + const std::uint64_t repetition_index = + repetition_seeds().repetition_index(state, size, max_width); + std::uint64_t seed = mix_seed(kSeed, static_cast(size)); + seed = mix_seed(seed, repetition_index); + + SeedContext context; + context.repetition_index = repetition_index; + context.value_seed = mix_seed(seed, 0x4F1BBCDCBFA54005ull); + context.query_seed = + mix_seed(mix_seed(seed, static_cast(max_width)), + 0x9D2C5680D3F6C58Bull); + return context; +} + +class RandomQueryGenerator { + public: + RandomQueryGenerator(std::size_t size, + std::size_t max_width, + std::uint64_t seed) + : size_(size), rng_state_(seed), pool_(kQueryCount) { + std::mt19937_64 rng(seed); + const std::size_t width_limit = std::min(max_width, size_); + std::uniform_int_distribution width_dist(1, width_limit); + for (auto& [left, width] : pool_) { + width = width_dist(rng); + std::uniform_int_distribution left_dist(0, size_ - width); + left = left_dist(rng); + } + shift_ = next_shift(); + } + + std::pair get_random_query() { + if (index_ == pool_.size()) { + index_ = 0; + shift_ = next_shift(); + } + const std::size_t query_number = query_number_++; + const auto [left, width] = pool_[index_++]; + const std::size_t shifted_left = + shifted_left_position(left, width, query_number); + return {shifted_left, shifted_left + width}; + } + + private: + std::uint64_t splitmix64() { + std::uint64_t z = (rng_state_ += 0x9E3779B97F4A7C15ull); + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); + } + + std::size_t next_shift() { return static_cast(splitmix64()); } + + std::size_t shifted_left_position(std::size_t left, + std::size_t width, + std::size_t query_number) const { + const std::size_t valid_left_count = size_ - width + 1; + const std::size_t offset = + (shift_ % valid_left_count + query_number % valid_left_count) % + valid_left_count; + return (left + offset) % valid_left_count; + } + + std::size_t size_ = 0; + std::uint64_t rng_state_ = 0; + std::vector> pool_; + std::size_t index_ = 0; + std::size_t query_number_ = 0; + std::size_t shift_ = 0; +}; + +/** + * @brief Single generated value dataset for value-RMQ benchmarks. + */ +class ValueDataset { + public: + explicit ValueDataset(std::size_t size, std::uint64_t seed) : values_(size) { + size_ = size; + + std::mt19937_64 rng(seed); + std::uniform_int_distribution value_dist(-1'000'000, + 1'000'000); + std::generate(values_.begin(), values_.end(), + [&] { return value_dist(rng); }); + + if (!values_.empty()) { + values_[values_.size() / 2] = kValueDatasetGlobalMinimum; + } + } + + std::size_t size() const { return size_; } + + std::span values() const { return values_; } + + private: + std::size_t size_ = 0; + std::vector values_; +}; + +void set_aux_memory_counters(benchmark::State& state, + std::size_t size, + double aux_bytes); + +template +void set_query_aux_memory_counters(benchmark::State& state, + std::size_t size, + const Rmq& rmq); + +/** + * @brief Run value-RMQ query benchmarks over one live RMQ instance. + */ +template +void run_queries(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const std::size_t max_width = static_cast(state.range(1)); + const SeedContext seeds = make_seed_context(state, size, max_width); + ValueDataset dataset(size, seeds.value_seed); + Rmq rmq(dataset.values()); + RandomQueryGenerator queries(size, max_width, seeds.query_seed); + + for (auto _ : state) { + const auto [left, right] = queries.get_random_query(); + std::size_t result = rmq.arg_min(left, right); + benchmark::DoNotOptimize(result); + } + + state.counters["N"] = static_cast(size); + state.counters["max_width"] = static_cast(max_width); + state.counters["index_bytes"] = static_cast(sizeof(Index)); + state.counters["seed_repetition"] = + static_cast(seeds.repetition_index); + state.counters["value_arrays"] = 1.0; + set_query_aux_memory_counters(state, size, rmq); +} + +void set_build_counters(benchmark::State& state, + std::size_t size, + std::size_t input_bytes) { + state.counters["N"] = static_cast(size); + state.counters["input_bytes"] = static_cast(input_bytes); + state.counters["index_bytes"] = static_cast(sizeof(Index)); + state.SetItemsProcessed(static_cast(state.iterations()) * + static_cast(size)); +} + +void set_aux_memory_counters(benchmark::State& state, + std::size_t size, + double aux_bytes) { + state.counters["aux_bytes"] = aux_bytes; + state.counters["aux_mib"] = aux_bytes / (1024.0 * 1024.0); + state.counters["aux_bits_per_value"] = + size == 0 ? 0.0 : (8.0 * aux_bytes) / static_cast(size); +} + +template +void set_query_aux_memory_counters(benchmark::State& state, + std::size_t size, + const Rmq& rmq) { + if constexpr (HasRmqMemoryUsage) { + set_aux_memory_counters(state, size, + static_cast(rmq.memory_usage_bytes())); + } +} + +/** + * @brief Run value-RMQ construction benchmarks with one value array. + */ +template +void run_value_rmq_build(benchmark::State& state) { + const std::size_t size = static_cast(state.range(0)); + const SeedContext seeds = make_seed_context(state, size); + ValueDataset dataset(size, seeds.value_seed); + + for (auto _ : state) { + Rmq rmq(dataset.values()); + std::size_t built_size = rmq.size(); + benchmark::DoNotOptimize(built_size); + benchmark::ClobberMemory(); + } + + set_build_counters(state, size, dataset.size() * sizeof(std::int64_t)); + state.counters["seed_repetition"] = + static_cast(seeds.repetition_index); + state.counters["value_arrays"] = 1.0; + if constexpr (HasRmqMemoryUsage) { + const Rmq rmq(dataset.values()); + const std::size_t aux_bytes = rmq.memory_usage_bytes(); + set_aux_memory_counters(state, size, static_cast(aux_bytes)); + } +} + +void register_benchmarks() { + using CartesianRmM = + pixie::rmq::CartesianRmM, Index>; + using CartesianHybrid = + pixie::rmq::CartesianHybridBTree, + Index>; + + const std::vector sizes = {1ull << 10, 1ull << 14, 1ull << 18, + 1ull << 22, 1ull << 24, 1ull << 26}; + const std::vector build_sizes = { + 1ull << 10, 1ull << 14, 1ull << 18, 1ull << 22, 1ull << 24, 1ull << 26}; + const std::vector widths = {64, 4096, 1ull << 18, 1ull << 22, + 1ull << 26}; + // Pure sparse-table rows materialize O(n log n) indexes; above 2^22 they + // dominate memory/runtime and make large benchmark passes noisy. + constexpr std::size_t kSparseTableBenchmarkMaxSize = 1ull << 22; + constexpr std::size_t kFocusedCartesianRmMLargeSize = 1ull << 28; + constexpr std::size_t kFocusedCartesianHybridLargeSize = 1ull << 28; + constexpr std::size_t kFocusedHybridBTreeLargeSize = 1ull << 28; + // Focused 2^28/2^30 rows are intentionally limited to the RMQ variants whose + // construction and query memory stay bounded at those sizes. CartesianRmM is + // included here because its Cartesian BP construction now uses a succinct + // monotone bit-stack instead of an n-entry index stack. + constexpr std::size_t kVeryLargeHybridSize = 1ull << 30; + + auto effective_widths_for = [&](std::size_t size) { + std::vector effective_widths; + for (const std::size_t width : widths) { + if (width > size) { + continue; + } + effective_widths.push_back(width); + } + effective_widths.push_back(size); + std::sort(effective_widths.begin(), effective_widths.end()); + effective_widths.erase( + std::unique(effective_widths.begin(), effective_widths.end()), + effective_widths.end()); + return effective_widths; + }; + + for (const std::size_t size : build_sizes) { + if (size <= kSparseTableBenchmarkMaxSize) { + benchmark::RegisterBenchmark( + "rmq_build_sparse_table", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + benchmark::RegisterBenchmark( + "rmq_build_segment_tree", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_build_cartesian_rmm", + &run_value_rmq_build) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_build_cartesian_hybrid_btree", + &run_value_rmq_build) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); +#ifdef PIXIE_THIRD_PARTY_BENCHMARKS + benchmark::RegisterBenchmark( + "rmq_build_sdsl_sct", + &run_value_rmq_build< + pixie::rmq::SdslSct, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); +#endif + benchmark::RegisterBenchmark( + "rmq_build_hybrid_btree", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(size)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + + benchmark::RegisterBenchmark("rmq_build_cartesian_rmm", + &run_value_rmq_build) + ->Arg(static_cast(kFocusedCartesianRmMLargeSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_build_cartesian_rmm", + &run_value_rmq_build) + ->Arg(static_cast(kVeryLargeHybridSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_build_cartesian_hybrid_btree", + &run_value_rmq_build) + ->Arg(static_cast(kFocusedCartesianHybridLargeSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_build_cartesian_hybrid_btree", + &run_value_rmq_build) + ->Arg(static_cast(kVeryLargeHybridSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); +#ifdef PIXIE_THIRD_PARTY_BENCHMARKS + benchmark::RegisterBenchmark( + "rmq_build_sdsl_sct", + &run_value_rmq_build< + pixie::rmq::SdslSct, Index>>) + ->Arg(static_cast(kFocusedCartesianRmMLargeSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); +#endif + benchmark::RegisterBenchmark( + "rmq_build_hybrid_btree", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(kFocusedHybridBTreeLargeSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark( + "rmq_build_hybrid_btree", + &run_value_rmq_build, Index>>) + ->Arg(static_cast(kVeryLargeHybridSize)) + ->Unit(benchmark::kMillisecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + + for (const std::size_t size : sizes) { + const std::vector effective_widths = + effective_widths_for(size); + + for (const std::size_t width : effective_widths) { + if (size <= kSparseTableBenchmarkMaxSize) { + benchmark::RegisterBenchmark( + "rmq_sparse_table", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + benchmark::RegisterBenchmark( + "rmq_segment_tree", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_cartesian_rmm", + &run_queries) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + benchmark::RegisterBenchmark("rmq_cartesian_hybrid_btree", + &run_queries) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); +#ifdef PIXIE_THIRD_PARTY_BENCHMARKS + benchmark::RegisterBenchmark( + "rmq_sdsl_sct", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); +#endif + benchmark::RegisterBenchmark( + "rmq_hybrid_btree", + &run_queries, Index>>) + ->Args({static_cast(size), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + } + + for (const std::size_t width : + effective_widths_for(kFocusedCartesianRmMLargeSize)) { + benchmark::RegisterBenchmark("rmq_cartesian_rmm", + &run_queries) + ->Args({static_cast(kFocusedCartesianRmMLargeSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + for (const std::size_t width : effective_widths_for(kVeryLargeHybridSize)) { + benchmark::RegisterBenchmark("rmq_cartesian_rmm", + &run_queries) + ->Args({static_cast(kVeryLargeHybridSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + for (const std::size_t width : + effective_widths_for(kFocusedCartesianHybridLargeSize)) { + benchmark::RegisterBenchmark("rmq_cartesian_hybrid_btree", + &run_queries) + ->Args({static_cast(kFocusedCartesianHybridLargeSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + for (const std::size_t width : effective_widths_for(kVeryLargeHybridSize)) { + benchmark::RegisterBenchmark("rmq_cartesian_hybrid_btree", + &run_queries) + ->Args({static_cast(kVeryLargeHybridSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + for (const std::size_t width : + effective_widths_for(kFocusedHybridBTreeLargeSize)) { + benchmark::RegisterBenchmark( + "rmq_hybrid_btree", + &run_queries, Index>>) + ->Args({static_cast(kFocusedHybridBTreeLargeSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } + for (const std::size_t width : effective_widths_for(kVeryLargeHybridSize)) { + benchmark::RegisterBenchmark( + "rmq_hybrid_btree", + &run_queries, Index>>) + ->Args({static_cast(kVeryLargeHybridSize), + static_cast(width)}) + ->Unit(benchmark::kNanosecond) + ->MinWarmUpTime(kBenchmarkWarmupSeconds) + ->MinTime(kBenchmarkMinSeconds); + } +} + +} // namespace + +int main(int argc, char** argv) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + benchmark::Initialize(&argc, argv); + register_benchmarks(); + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + return 0; +} diff --git a/src/benchmarks/benchmarks.cpp b/src/benchmarks/benchmarks.cpp index 11acbf6..8e5a9ea 100644 --- a/src/benchmarks/benchmarks.cpp +++ b/src/benchmarks/benchmarks.cpp @@ -92,7 +92,7 @@ static void PrepareRandomBits87p5Fill(std::span bits) { static void BM_RankNonInterleaved(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); auto bits_as_words = bits.As64BitInts(); PrepareRandomBits50pFill(bits_as_words); pixie::BitVector bv(bits_as_words, n); @@ -106,7 +106,7 @@ static void BM_RankNonInterleaved(benchmark::State& state) { static void BM_RankZeroNonInterleaved(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits50pFill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -120,7 +120,7 @@ static void BM_RankZeroNonInterleaved(benchmark::State& state) { static void BM_RankInterleaved(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits50pFill(bits.As64BitInts()); pixie::BitVectorInterleaved bv(bits.As64BitInts(), n); @@ -134,7 +134,7 @@ static void BM_RankInterleaved(benchmark::State& state) { static void BM_SelectNonInterleaved(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits50pFill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -150,7 +150,7 @@ static void BM_SelectNonInterleaved(benchmark::State& state) { static void BM_SelectZeroNonInterleaved(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits50pFill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -166,7 +166,7 @@ static void BM_SelectZeroNonInterleaved(benchmark::State& state) { static void BM_RankNonInterleaved12p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits12p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -187,7 +187,7 @@ static void BM_RankNonInterleaved12p5PercentFill(benchmark::State& state) { static void BM_RankZeroNonInterleaved12p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits12p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -201,7 +201,7 @@ static void BM_RankZeroNonInterleaved12p5PercentFill(benchmark::State& state) { static void BM_SelectNonInterleaved12p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits12p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -218,7 +218,7 @@ static void BM_SelectZeroNonInterleaved12p5PercentFill( benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits12p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -234,7 +234,7 @@ static void BM_SelectZeroNonInterleaved12p5PercentFill( static void BM_RankNonInterleaved87p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits87p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -248,7 +248,7 @@ static void BM_RankNonInterleaved87p5PercentFill(benchmark::State& state) { static void BM_RankZeroNonInterleaved87p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits87p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -262,7 +262,7 @@ static void BM_RankZeroNonInterleaved87p5PercentFill(benchmark::State& state) { static void BM_SelectNonInterleaved87p5PercentFill(benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits87p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); @@ -279,7 +279,7 @@ static void BM_SelectZeroNonInterleaved87p5PercentFill( benchmark::State& state) { size_t n = state.range(0); - AlignedStorage bits(n); + pixie::AlignedStorage bits(n); PrepareRandomBits87p5Fill(bits.As64BitInts()); pixie::BitVector bv(bits.As64BitInts(), n); diff --git a/src/benchmarks/excess_positions_benchmarks.cpp b/src/benchmarks/excess_positions_benchmarks.cpp index 4bbf28f..6cd5df7 100644 --- a/src/benchmarks/excess_positions_benchmarks.cpp +++ b/src/benchmarks/excess_positions_benchmarks.cpp @@ -8,12 +8,29 @@ #include #include +using pixie::experimental::excess_min_128_byte_lut; +using pixie::experimental::excess_min_128_hybrid_lut; using pixie::experimental::excess_positions_512_branching_lut; using pixie::experimental::excess_positions_512_byte_lut; using pixie::experimental::excess_positions_512_expand; using pixie::experimental::excess_positions_512_expand8; using pixie::experimental::excess_positions_512_expand_avx512; using pixie::experimental::excess_positions_512_lut_avx512; +#ifdef PIXIE_AVX2_SUPPORT +using pixie::experimental::excess_min_128_deinterleaved_byte16_sse; +using pixie::experimental::excess_min_128_deinterleaved_full_sse; +using pixie::experimental::excess_min_128_deinterleaved_sse; +using pixie::experimental::excess_min_128_expand16_avx2; +using pixie::experimental::excess_min_128_lane64_sse; +using pixie::experimental::excess_min_128_short_skip; +using pixie::experimental::excess_min_128_split64_sse; +#endif +using pixie::experimental::excess_min_128_nibble_lut; +using pixie::experimental::excess_min_128_scalar_bits; + +// excess_record_lows_128, excess_record_lows_128_byte_lut, +// excess_record_lows_128_lut, excess_record_lows_128_avx2 are in the global +// namespace from pixie/bits.h static std::vector> make_blocks( size_t num_blocks = 4096) { @@ -27,6 +44,441 @@ static std::vector> make_blocks( return blocks; } +static std::vector> make_128_blocks( + size_t num_blocks = 4096) { + std::mt19937_64 rng(42); + std::vector> blocks(num_blocks); + for (auto& b : blocks) { + b = {rng(), rng()}; + } + return blocks; +} + +static std::vector> make_64_blocks( + size_t num_blocks = 4096) { + std::mt19937_64 rng(42); + std::vector> blocks(num_blocks); + for (auto& b : blocks) { + b = {rng()}; + } + return blocks; +} + +static std::vector> make_128_ranges( + size_t num_ranges = 4096) { + std::mt19937_64 rng(43); + std::uniform_int_distribution offset_dist(0, 128); + std::vector> ranges(num_ranges); + for (auto& range : ranges) { + size_t left = offset_dist(rng); + size_t right = offset_dist(rng); + if (left > right) { + std::swap(left, right); + } + range = {left, right}; + } + return ranges; +} + +static std::vector> make_64_ranges( + size_t num_ranges = 4096) { + std::mt19937_64 rng(43); + std::uniform_int_distribution offset_dist(0, 64); + std::vector> ranges(num_ranges); + for (auto& range : ranges) { + size_t left = offset_dist(rng); + size_t right = offset_dist(rng); + if (left > right) { + std::swap(left, right); + } + range = {left, right}; + } + return ranges; +} + +static std::vector> make_disjoint_boundary_ranges( + size_t num_ranges = 4096) { + std::mt19937_64 rng(45); + std::uniform_int_distribution prefix_dist(0, 126); + std::vector> ranges(num_ranges); + for (auto& [suffix_left, prefix_right] : ranges) { + prefix_right = prefix_dist(rng); + std::uniform_int_distribution suffix_dist(prefix_right + 1, 127); + suffix_left = suffix_dist(rng); + } + return ranges; +} + +static std::vector make_512_targets(size_t num_targets = 4096) { + std::mt19937 rng(44); + std::uniform_int_distribution target_dist(-128, 128); + std::vector targets(num_targets); + for (int& target : targets) { + target = target_dist(rng); + } + return targets; +} + +static void BM_ExcessMin128(benchmark::State& state) { + const size_t left = static_cast(state.range(0)); + const size_t right = static_cast(state.range(1)); + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + ExcessResult result = excess_min_128(s.data(), left, right); + benchmark::DoNotOptimize(result.min_excess); + benchmark::DoNotOptimize(result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin128) + ->ArgNames({"left", "right"}) + ->Args({0, 128}) + ->Args({0, 127}) + ->Args({0, 16}) + ->Args({0, 32}) + ->Args({0, 48}) + ->Args({0, 64}) + ->Args({0, 31}) + ->Args({1, 17}) + ->Args({3, 35}) + ->Args({5, 37}) + ->Args({32, 64}) + ->Args({33, 65}) + ->Args({64, 96}) + ->Args({61, 93}) + ->Args({96, 128}) + ->Args({56, 72}) + ->Args({60, 68}) + ->Args({63, 64}) + ->Args({17, 17}); + +static void BM_ExcessMin64(benchmark::State& state) { + const size_t left = static_cast(state.range(0)); + const size_t right = static_cast(state.range(1)); + const auto blocks = make_64_blocks(); + const size_t num_blocks = blocks.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + ExcessResult result = excess_min_64(s.data(), left, right); + benchmark::DoNotOptimize(result.min_excess); + benchmark::DoNotOptimize(result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin64) + ->ArgNames({"left", "right"}) + ->Args({0, 64}) + ->Args({0, 63}) + ->Args({0, 16}) + ->Args({0, 32}) + ->Args({0, 48}) + ->Args({0, 31}) + ->Args({1, 17}) + ->Args({3, 35}) + ->Args({5, 37}) + ->Args({32, 64}) + ->Args({33, 64}) + ->Args({56, 64}) + ->Args({56, 63}) + ->Args({60, 64}) + ->Args({60, 63}) + ->Args({63, 64}) + ->Args({17, 17}); + +static void BM_ExcessMin128BoundaryPairIndependent(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const auto ranges = make_disjoint_boundary_ranges(); + const size_t num_blocks = blocks.size(); + const size_t num_ranges = ranges.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& suffix = blocks[idx % num_blocks]; + const auto& prefix = blocks[(idx + 1) % num_blocks]; + const auto [suffix_left, prefix_right] = ranges[idx % num_ranges]; + ExcessResult suffix_result = + excess_min_128(suffix.data(), suffix_left, 127); + ExcessResult prefix_result = excess_min_128(prefix.data(), 0, prefix_right); + benchmark::DoNotOptimize(suffix_result.min_excess); + benchmark::DoNotOptimize(suffix_result.offset); + benchmark::DoNotOptimize(prefix_result.min_excess); + benchmark::DoNotOptimize(prefix_result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin128BoundaryPairIndependent); + +static void BM_ExcessMin128BoundaryPairFused(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const auto ranges = make_disjoint_boundary_ranges(); + const size_t num_blocks = blocks.size(); + const size_t num_ranges = ranges.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& suffix = blocks[idx % num_blocks]; + const auto& prefix = blocks[(idx + 1) % num_blocks]; + const auto [suffix_left, prefix_right] = ranges[idx % num_ranges]; + ExcessBoundaryPairResult result = excess_min_128_disjoint_suffix_prefix( + suffix.data(), suffix_left, prefix.data(), prefix_right); + benchmark::DoNotOptimize(result.suffix.min_excess); + benchmark::DoNotOptimize(result.suffix.offset); + benchmark::DoNotOptimize(result.prefix.min_excess); + benchmark::DoNotOptimize(result.prefix.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin128BoundaryPairFused); + +static void BM_ExcessMin128BoundaryPairFixedIndependent( + benchmark::State& state) { + const size_t suffix_left = static_cast(state.range(0)); + const size_t prefix_right = static_cast(state.range(1)); + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& suffix = blocks[idx % num_blocks]; + const auto& prefix = blocks[(idx + 1) % num_blocks]; + ExcessResult suffix_result = + excess_min_128(suffix.data(), suffix_left, 127); + ExcessResult prefix_result = excess_min_128(prefix.data(), 0, prefix_right); + benchmark::DoNotOptimize(suffix_result.min_excess); + benchmark::DoNotOptimize(suffix_result.offset); + benchmark::DoNotOptimize(prefix_result.min_excess); + benchmark::DoNotOptimize(prefix_result.offset); + ++idx; + } + + state.counters["gate_score"] = + static_cast((127 - suffix_left) + prefix_right); + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin128BoundaryPairFixedIndependent) + ->ArgNames({"suffix_left", "prefix_right"}) + ->Args({127, 0}) + ->Args({126, 0}) + ->Args({127, 1}) + ->Args({126, 1}) + ->Args({125, 1}) + ->Args({125, 2}) + ->Args({125, 3}) + ->Args({124, 3}) + ->Args({120, 7}) + ->Args({112, 15}) + ->Args({111, 16}) + ->Args({96, 31}) + ->Args({80, 47}) + ->Args({64, 63}); + +static void BM_ExcessMin128BoundaryPairFixedFused(benchmark::State& state) { + const size_t suffix_left = static_cast(state.range(0)); + const size_t prefix_right = static_cast(state.range(1)); + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& suffix = blocks[idx % num_blocks]; + const auto& prefix = blocks[(idx + 1) % num_blocks]; + ExcessBoundaryPairResult result = excess_min_128_disjoint_suffix_prefix( + suffix.data(), suffix_left, prefix.data(), prefix_right); + benchmark::DoNotOptimize(result.suffix.min_excess); + benchmark::DoNotOptimize(result.suffix.offset); + benchmark::DoNotOptimize(result.prefix.min_excess); + benchmark::DoNotOptimize(result.prefix.offset); + ++idx; + } + + state.counters["gate_score"] = + static_cast((127 - suffix_left) + prefix_right); + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin128BoundaryPairFixedFused) + ->ArgNames({"suffix_left", "prefix_right"}) + ->Args({127, 0}) + ->Args({126, 0}) + ->Args({127, 1}) + ->Args({126, 1}) + ->Args({125, 1}) + ->Args({125, 2}) + ->Args({125, 3}) + ->Args({124, 3}) + ->Args({120, 7}) + ->Args({112, 15}) + ->Args({111, 16}) + ->Args({96, 31}) + ->Args({80, 47}) + ->Args({64, 63}); + +template +static void BM_ExcessMin128Variant(benchmark::State& state) { + const size_t left = static_cast(state.range(0)); + const size_t right = static_cast(state.range(1)); + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + ExcessResult result = Fn(s.data(), left, right); + benchmark::DoNotOptimize(result.min_excess); + benchmark::DoNotOptimize(result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +#define PIXIE_BENCH_EXCESS_MIN_VARIANT(name, fn) \ + BENCHMARK_TEMPLATE(BM_ExcessMin128Variant, fn) \ + ->Name(name) \ + ->ArgNames({"left", "right"}) \ + ->Args({0, 128}) \ + ->Args({0, 127}) \ + ->Args({0, 16}) \ + ->Args({0, 32}) \ + ->Args({0, 48}) \ + ->Args({0, 64}) \ + ->Args({0, 31}) \ + ->Args({1, 17}) \ + ->Args({3, 35}) \ + ->Args({5, 37}) \ + ->Args({32, 64}) \ + ->Args({33, 65}) \ + ->Args({64, 96}) \ + ->Args({61, 93}) \ + ->Args({96, 128}) \ + ->Args({56, 72}) \ + ->Args({60, 68}) \ + ->Args({63, 64}) \ + ->Args({17, 17}) + +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_ScalarBits", + excess_min_128_scalar_bits); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_NibbleLUT", + excess_min_128_nibble_lut); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_ByteLUT", + excess_min_128_byte_lut); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_HybridLUT", + excess_min_128_hybrid_lut); +#ifdef PIXIE_AVX2_SUPPORT +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_Expand16AVX2", + excess_min_128_expand16_avx2); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_DeinterleavedSSE", + excess_min_128_deinterleaved_sse); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_DeinterleavedFullSSE", + excess_min_128_deinterleaved_full_sse); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_DeinterleavedByte16SSE", + excess_min_128_deinterleaved_byte16_sse); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_Lane64SSE", + excess_min_128_lane64_sse); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_Split64SSE", + excess_min_128_split64_sse); +PIXIE_BENCH_EXCESS_MIN_VARIANT("BM_ExcessMin128_ShortSkip", + excess_min_128_short_skip); +#endif + +#undef PIXIE_BENCH_EXCESS_MIN_VARIANT + +template +static void BM_ExcessMin128RandomRange(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const auto ranges = make_128_ranges(); + const size_t num_blocks = blocks.size(); + const size_t num_ranges = ranges.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + const auto [left, right] = ranges[idx % num_ranges]; + ExcessResult result = Fn(s.data(), left, right); + benchmark::DoNotOptimize(result.min_excess); + benchmark::DoNotOptimize(result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +static void BM_ExcessMin64RandomRange(benchmark::State& state) { + const auto blocks = make_64_blocks(); + const auto ranges = make_64_ranges(); + const size_t num_blocks = blocks.size(); + const size_t num_ranges = ranges.size(); + + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + const auto [left, right] = ranges[idx % num_ranges]; + ExcessResult result = excess_min_64(s.data(), left, right); + benchmark::DoNotOptimize(result.min_excess); + benchmark::DoNotOptimize(result.offset); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessMin64RandomRange); + +#define PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT(name, fn) \ + BENCHMARK_TEMPLATE(BM_ExcessMin128RandomRange, fn)->Name(name) + +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_RandomRange", + excess_min_128); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_ScalarBits_RandomRange", + excess_min_128_scalar_bits); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_NibbleLUT_RandomRange", + excess_min_128_nibble_lut); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_ByteLUT_RandomRange", + excess_min_128_byte_lut); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_HybridLUT_RandomRange", + excess_min_128_hybrid_lut); +#ifdef PIXIE_AVX2_SUPPORT +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT( + "BM_ExcessMin128_Expand16AVX2_RandomRange", + excess_min_128_expand16_avx2); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT( + "BM_ExcessMin128_DeinterleavedSSE_RandomRange", + excess_min_128_deinterleaved_sse); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT( + "BM_ExcessMin128_DeinterleavedFullSSE_RandomRange", + excess_min_128_deinterleaved_full_sse); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT( + "BM_ExcessMin128_DeinterleavedByte16SSE_RandomRange", + excess_min_128_deinterleaved_byte16_sse); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_Lane64SSE_RandomRange", + excess_min_128_lane64_sse); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_Split64SSE_RandomRange", + excess_min_128_split64_sse); +PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT("BM_ExcessMin128_ShortSkip_RandomRange", + excess_min_128_short_skip); +#endif + +#undef PIXIE_BENCH_EXCESS_MIN_RANDOM_VARIANT + static void BM_ExcessPositions512(benchmark::State& state) { const int target_x = state.range(0); const auto blocks = make_blocks(); @@ -183,6 +635,22 @@ BENCHMARK(BM_ExcessPositions512_ExpandAVX512) ->Args({8}) ->Args({64}); +static void excess_positions_512_scalar_benchmark(const uint64_t* s, + int target_x, + uint64_t* out) noexcept { + for (int w = 0; w < 8; ++w) { + out[w] = 0; + } + int cur = 0; + for (size_t i = 0; i < 512; ++i) { + const int bit = int((s[i >> 6] >> (i & 63)) & 1ull); + cur += bit ? +1 : -1; + if (cur == target_x) { + out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } +} + static void BM_ExcessPositions512_Scalar(benchmark::State& state) { const int target_x = state.range(0); const auto blocks = make_blocks(); @@ -193,17 +661,7 @@ static void BM_ExcessPositions512_Scalar(benchmark::State& state) { for (auto _ : state) { const auto& s = blocks[idx % num_blocks]; - for (int w = 0; w < 8; ++w) { - out[w] = 0; - } - int cur = 0; - for (size_t i = 0; i < 512; ++i) { - const int bit = int((s[i >> 6] >> (i & 63)) & 1ull); - cur += bit ? +1 : -1; - if (cur == target_x) { - out[i >> 6] |= (uint64_t{1} << (i & 63)); - } - } + excess_positions_512_scalar_benchmark(s.data(), target_x, out); benchmark::DoNotOptimize(out); ++idx; } @@ -244,3 +702,177 @@ BENCHMARK(BM_ExcessPositions512_ByteLUT) ->Args({0}) ->Args({8}) ->Args({64}); + +template +static void BM_ExcessPositions512RandomTarget(benchmark::State& state) { + const auto blocks = make_blocks(); + const auto targets = make_512_targets(); + const size_t num_blocks = blocks.size(); + const size_t num_targets = targets.size(); + + alignas(64) uint64_t out[8]; + size_t idx = 0; + + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + Fn(s.data(), targets[idx % num_targets], out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +#define PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM(name, fn) \ + BENCHMARK_TEMPLATE(BM_ExcessPositions512RandomTarget, fn)->Name(name) + +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM("BM_ExcessPositions512_RandomTarget", + excess_positions_512); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_BranchingLUT_RandomTarget", + excess_positions_512_branching_lut); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_LUTAVX512_RandomTarget", + excess_positions_512_lut_avx512); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_Expand_RandomTarget", + excess_positions_512_expand); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_Expand8_RandomTarget", + excess_positions_512_expand8); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_ExpandAVX512_RandomTarget", + excess_positions_512_expand_avx512); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_Scalar_RandomTarget", + excess_positions_512_scalar_benchmark); +PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM( + "BM_ExcessPositions512_ByteLUT_RandomTarget", + excess_positions_512_byte_lut); + +static void naive_excess_record_lows_128(const uint64_t* s, uint64_t* out) { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + for (size_t i = 0; i < 128; ++i) { + const uint64_t w = s[i >> 6]; + const int bit = static_cast((w >> (i & 63)) & 1ull); + cur += bit ? +1 : -1; + if (cur < best) { + best = cur; + out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } +} + +static void BM_ExcessRecordLows128(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + excess_record_lows_128(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128); + +static void BM_ExcessRecordLows128_Naive(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + naive_excess_record_lows_128(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128_Naive); + +static void BM_ExcessRecordLows128_ByteLUT(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + excess_record_lows_128_byte_lut(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128_ByteLUT); + +static void BM_ExcessRecordLows128_LUT(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + excess_record_lows_128_lut(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128_LUT); + +#ifdef PIXIE_AVX2_SUPPORT +static void BM_ExcessRecordLows128_AVX2(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + excess_record_lows_128_avx2(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128_AVX2); + +static void BM_ExcessRecordLows128_NibbleLUT(benchmark::State& state) { + const auto blocks = make_128_blocks(); + const size_t num_blocks = blocks.size(); + + alignas(16) uint64_t out[2]; + size_t idx = 0; + for (auto _ : state) { + const auto& s = blocks[idx % num_blocks]; + excess_record_lows_128_nibble_lut(s.data(), out); + benchmark::DoNotOptimize(out); + ++idx; + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_ExcessRecordLows128_NibbleLUT); +#endif + +#undef PIXIE_BENCH_EXCESS_POSITIONS_512_RANDOM diff --git a/src/docs/excess_positions_benchmark_results.md b/src/docs/excess_positions_benchmark_results.md new file mode 100644 index 0000000..17adf71 --- /dev/null +++ b/src/docs/excess_positions_benchmark_results.md @@ -0,0 +1,9 @@ +| Method | X=-64 | X=-8 | X=0 | X=8 | X=64 | +|---|---:|---:|---:|---:|---:| +| BranchingLUT | 15.71 ns | 26.56 ns | 26.55 ns | 26.43 ns | 15.37 ns | +| Current | 10.73 ns | 18.09 ns | 18.58 ns | 18.24 ns | 10.43 ns | +| Expand | 60.70 ns | 88.41 ns | 87.38 ns | 88.50 ns | 56.77 ns | +| Expand8 | 19.13 ns | 53.05 ns | 47.83 ns | 49.35 ns | 17.44 ns | +| ExpandAVX512 | 23.13 ns | 38.39 ns | 38.56 ns | 39.24 ns | 23.19 ns | +| LUTAVX512 | 12.33 ns | 18.34 ns | 18.06 ns | 18.21 ns | 12.75 ns | +| Scalar | 304.42 ns | 389.58 ns | 446.94 ns | 399.80 ns | 316.23 ns | diff --git a/src/tests/excess_positions_tests.cpp b/src/tests/excess_positions_tests.cpp index 57d69c8..6321478 100644 --- a/src/tests/excess_positions_tests.cpp +++ b/src/tests/excess_positions_tests.cpp @@ -8,13 +8,27 @@ #include #include #include +#include +using pixie::experimental::excess_min_128_byte_lut; +using pixie::experimental::excess_min_128_hybrid_lut; using pixie::experimental::excess_positions_512_branching_lut; using pixie::experimental::excess_positions_512_byte_lut; using pixie::experimental::excess_positions_512_expand; using pixie::experimental::excess_positions_512_expand8; using pixie::experimental::excess_positions_512_expand_avx512; using pixie::experimental::excess_positions_512_lut_avx512; +#ifdef PIXIE_AVX2_SUPPORT +using pixie::experimental::excess_min_128_deinterleaved_byte16_sse; +using pixie::experimental::excess_min_128_deinterleaved_full_sse; +using pixie::experimental::excess_min_128_deinterleaved_sse; +using pixie::experimental::excess_min_128_expand16_avx2; +using pixie::experimental::excess_min_128_lane64_sse; +using pixie::experimental::excess_min_128_short_skip; +using pixie::experimental::excess_min_128_split64_sse; +#endif +using pixie::experimental::excess_min_128_nibble_lut; +using pixie::experimental::excess_min_128_scalar_bits; static void naive_excess_positions_512(const uint64_t* s, int target_x, @@ -68,6 +82,86 @@ static int naive_prefix_excess_128(const uint64_t* s, size_t end_offset) { return cur; } +static int naive_prefix_excess_64(const uint64_t* s, size_t end_offset) { + end_offset = std::min(end_offset, 64); + int cur = 0; + for (size_t i = 0; i < end_offset; ++i) { + const int bit = int((s[0] >> i) & 1ull); + cur += bit ? +1 : -1; + } + return cur; +} + +static ExcessResult naive_excess_min_128(const uint64_t* s, + size_t left, + size_t right) { + if (left > right) { + return {}; + } + left = std::min(left, 128); + right = std::min(right, 128); + + int cur = 0; + int best = 0; + size_t best_offset = 0; + bool found = false; + if (left == 0) { + found = true; + } + for (size_t bit = 0; bit < right; ++bit) { + cur += ((s[bit >> 6] >> (bit & 63)) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (offset < left) { + continue; + } + if (!found || cur < best) { + best = cur; + best_offset = offset; + found = true; + } + } + if (!found) { + best = naive_prefix_excess_128(s, left); + best_offset = left; + } + return {best, best_offset}; +} + +static ExcessResult naive_excess_min_64(const uint64_t* s, + size_t left, + size_t right) { + if (left > right) { + return {}; + } + left = std::min(left, 64); + right = std::min(right, 64); + + int cur = 0; + int best = 0; + size_t best_offset = 0; + bool found = false; + if (left == 0) { + found = true; + } + for (size_t bit = 0; bit < right; ++bit) { + cur += ((s[0] >> bit) & 1ull) != 0 ? 1 : -1; + const size_t offset = bit + 1; + if (offset < left) { + continue; + } + if (!found || cur < best) { + best = cur; + best_offset = offset; + found = true; + } + } + if (!found) { + best = naive_prefix_excess_64(s, left); + best_offset = left; + } + return {best, best_offset}; +} + static size_t naive_forward_search_128(const uint64_t* s, int target_x, size_t start_offset) { @@ -116,6 +210,28 @@ static size_t count_matches(const uint64_t* out) { return cnt; } +static void check_boundary_pair_matches_independent( + const std::array& suffix, + size_t suffix_left, + const std::array& prefix, + size_t prefix_right) { + const ExcessBoundaryPairResult result = excess_min_128_disjoint_suffix_prefix( + suffix.data(), suffix_left, prefix.data(), prefix_right); + const ExcessResult expected_suffix = + excess_min_128(suffix.data(), suffix_left, 127); + const ExcessResult expected_prefix = + excess_min_128(prefix.data(), 0, prefix_right); + + ASSERT_EQ(result.suffix.min_excess, expected_suffix.min_excess) + << "suffix_left=" << suffix_left << " prefix_right=" << prefix_right; + ASSERT_EQ(result.suffix.offset, expected_suffix.offset) + << "suffix_left=" << suffix_left << " prefix_right=" << prefix_right; + ASSERT_EQ(result.prefix.min_excess, expected_prefix.min_excess) + << "suffix_left=" << suffix_left << " prefix_right=" << prefix_right; + ASSERT_EQ(result.prefix.offset, expected_prefix.offset) + << "suffix_left=" << suffix_left << " prefix_right=" << prefix_right; +} + template static void check_matches_naive(Fn fn, const char* fn_name, @@ -134,6 +250,23 @@ static void check_matches_naive(Fn fn, << fn_name << " case=" << case_id << " x=" << target_x; } +template +static void check_min_matches_naive(Fn fn, + const char* fn_name, + const uint64_t* s, + size_t left, + size_t right, + int case_id = 0) { + const ExcessResult result = fn(s, left, right); + const ExcessResult expected = naive_excess_min_128(s, left, right); + ASSERT_EQ(result.min_excess, expected.min_excess) + << fn_name << " case=" << case_id << " left=" << left + << " right=" << right; + ASSERT_EQ(result.offset, expected.offset) + << fn_name << " case=" << case_id << " left=" << left + << " right=" << right; +} + TEST(ExcessPositions128, MatchesNaiveMasksAndDelta) { const std::array, 4> cases = {{ {0, 0}, @@ -170,6 +303,448 @@ TEST(ExcessPositions128, PrefixExcessMatchesNaive) { } } +TEST(ExcessPositions64, PrefixExcessMatchesNaive) { + std::mt19937_64 rng(44); + const std::array offsets = {0, 1, 2, 31, 32, 33, + 62, 63, 64, 65, 96}; + + for (int t = 0; t < 1000; ++t) { + const std::array s = {rng()}; + for (size_t offset : offsets) { + EXPECT_EQ(prefix_excess_64(s.data(), offset), + naive_prefix_excess_64(s.data(), offset)) + << "case=" << t << " offset=" << offset; + } + } +} + +TEST(ExcessPositions128, MinMatchesNaiveFixedCases) { + const std::array, 5> cases = {{ + {0, 0}, + {UINT64_MAX, UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull}, + {0x0000FFFF0000FFFFull, 0xFFFF0000FFFF0000ull}, + }}; + const std::array, 12> ranges = {{ + {0, 128}, + {0, 0}, + {1, 1}, + {63, 65}, + {64, 64}, + {64, 128}, + {3, 6}, + {5, 5}, + {127, 128}, + {128, 128}, + {120, 127}, + {129, 140}, + }}; + + for (const auto& s : cases) { + for (const auto [left, right] : ranges) { + const ExcessResult result = excess_min_128(s.data(), left, right); + const ExcessResult expected = naive_excess_min_128(s.data(), left, right); + EXPECT_EQ(result.min_excess, expected.min_excess) + << "left=" << left << " right=" << right; + EXPECT_EQ(result.offset, expected.offset) + << "left=" << left << " right=" << right; + } + } +} + +TEST(ExcessPositions64, MinMatchesNaiveFixedCases) { + const std::array, 5> cases = {{ + {0}, + {UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull}, + {0x0123456789ABCDEFull}, + {0x0000FFFF0000FFFFull}, + }}; + const std::array, 12> ranges = {{ + {0, 64}, + {0, 0}, + {1, 1}, + {31, 33}, + {32, 32}, + {32, 64}, + {3, 6}, + {5, 5}, + {63, 64}, + {64, 64}, + {56, 63}, + {65, 96}, + }}; + + for (const auto& s : cases) { + for (const auto [left, right] : ranges) { + const ExcessResult result = excess_min_64(s.data(), left, right); + const ExcessResult expected = naive_excess_min_64(s.data(), left, right); + EXPECT_EQ(result.min_excess, expected.min_excess) + << "left=" << left << " right=" << right; + EXPECT_EQ(result.offset, expected.offset) + << "left=" << left << " right=" << right; + } + } +} + +TEST(ExcessPositions128, MinReturnsFirstTie) { + const std::array s = {0x5555555555555555ull, + 0x5555555555555555ull}; + const ExcessResult result = excess_min_128(s.data(), 0, 128); + EXPECT_EQ(result.min_excess, 0); + EXPECT_EQ(result.offset, 0u); + + const ExcessResult shifted = excess_min_128(s.data(), 1, 128); + EXPECT_EQ(shifted.min_excess, 0); + EXPECT_EQ(shifted.offset, 2u); + + const ExcessResult left_tie = excess_min_128(s.data(), 2, 128); + EXPECT_EQ(left_tie.min_excess, 0); + EXPECT_EQ(left_tie.offset, 2u); +} + +TEST(ExcessPositions128, MinHandlesRightBoundary) { + const std::array s = {0, 0}; + + const ExcessResult without_last = excess_min_128(s.data(), 0, 127); + EXPECT_EQ(without_last.min_excess, -127); + EXPECT_EQ(without_last.offset, 127u); + + const ExcessResult with_last = excess_min_128(s.data(), 0, 128); + EXPECT_EQ(with_last.min_excess, -128); + EXPECT_EQ(with_last.offset, 128u); +} + +TEST(ExcessPositions64, MinHandlesRightBoundary) { + const std::array s = {0}; + + const ExcessResult without_last = excess_min_64(s.data(), 0, 63); + EXPECT_EQ(without_last.min_excess, -63); + EXPECT_EQ(without_last.offset, 63u); + + const ExcessResult with_last = excess_min_64(s.data(), 0, 64); + EXPECT_EQ(with_last.min_excess, -64); + EXPECT_EQ(with_last.offset, 64u); +} + +TEST(ExcessPositions128, MinPartialNibbleBoundsExcludeOuterMin) { + const std::array s = {0, 0}; + + const ExcessResult short_prefix = excess_min_128(s.data(), 1, 2); + EXPECT_EQ(short_prefix.min_excess, -2); + EXPECT_EQ(short_prefix.offset, 2u); + + const ExcessResult short_suffix = excess_min_128(s.data(), 2, 3); + EXPECT_EQ(short_suffix.min_excess, -3); + EXPECT_EQ(short_suffix.offset, 3u); +} + +TEST(ExcessPositions128, MinPositiveRangeKeepsLeftBoundary) { + const std::array s = {UINT64_MAX, UINT64_MAX}; + + const ExcessResult result = excess_min_128(s.data(), 64, 128); + EXPECT_EQ(result.min_excess, 64); + EXPECT_EQ(result.offset, 64u); +} + +TEST(ExcessPositions128, MinInvalidRangeUsesSentinel) { + const std::array s = {0, 0}; + const ExcessResult result = excess_min_128(s.data(), 17, 16); + EXPECT_EQ(result.min_excess, 0); + EXPECT_EQ(result.offset, 128u); +} + +TEST(ExcessPositions64, MinInvalidRangeUsesSentinel) { + const std::array s = {0}; + const ExcessResult result = excess_min_64(s.data(), 17, 16); + EXPECT_EQ(result.min_excess, 0); + EXPECT_EQ(result.offset, 128u); +} + +TEST(ExcessPositions128, MinMatchesNaiveRandom) { + std::mt19937_64 rng(43); + std::uniform_int_distribution offset_dist(0, 128); + + for (int t = 0; t < 1000; ++t) { + const std::array s = {rng(), rng()}; + for (int q = 0; q < 32; ++q) { + size_t left = offset_dist(rng); + size_t right = offset_dist(rng); + if (left > right) { + std::swap(left, right); + } + const ExcessResult result = excess_min_128(s.data(), left, right); + const ExcessResult expected = naive_excess_min_128(s.data(), left, right); + ASSERT_EQ(result.min_excess, expected.min_excess) + << "case=" << t << " left=" << left << " right=" << right; + ASSERT_EQ(result.offset, expected.offset) + << "case=" << t << " left=" << left << " right=" << right; + } + } +} + +TEST(ExcessPositions64, MinMatchesNaiveRandom) { + std::mt19937_64 rng(45); + std::uniform_int_distribution offset_dist(0, 64); + + for (int t = 0; t < 1000; ++t) { + const std::array s = {rng()}; + for (int q = 0; q < 32; ++q) { + size_t left = offset_dist(rng); + size_t right = offset_dist(rng); + if (left > right) { + std::swap(left, right); + } + const ExcessResult result = excess_min_64(s.data(), left, right); + const ExcessResult expected = naive_excess_min_64(s.data(), left, right); + ASSERT_EQ(result.min_excess, expected.min_excess) + << "case=" << t << " left=" << left << " right=" << right; + ASSERT_EQ(result.offset, expected.offset) + << "case=" << t << " left=" << left << " right=" << right; + } + } +} + +TEST(ExcessPositions64, DisjointBoundaryPairMatchesIndependentFixedCases) { + const std::array, 5> cases = {{ + {0}, + {UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull}, + {0x0123456789ABCDEFull}, + {0x0000FFFF0000FFFFull}, + }}; + const std::array, 8> ranges = {{ + {1, 0}, + {4, 3}, + {17, 5}, + {31, 8}, + {47, 11}, + {61, 13}, + {62, 62}, + {64, 64}, + }}; + + for (const auto& suffix : cases) { + for (const auto& prefix : cases) { + for (const auto [suffix_left, prefix_right] : ranges) { + const ExcessBoundaryPairResult result = + excess_min_64_disjoint_suffix_prefix(suffix.data(), suffix_left, + prefix.data(), prefix_right); + const ExcessResult expected_suffix = + excess_min_64(suffix.data(), suffix_left, 63); + const ExcessResult expected_prefix = + excess_min_64(prefix.data(), 0, prefix_right); + EXPECT_EQ(result.suffix.min_excess, expected_suffix.min_excess) + << "suffix_left=" << suffix_left + << " prefix_right=" << prefix_right; + EXPECT_EQ(result.suffix.offset, expected_suffix.offset) + << "suffix_left=" << suffix_left + << " prefix_right=" << prefix_right; + EXPECT_EQ(result.prefix.min_excess, expected_prefix.min_excess) + << "suffix_left=" << suffix_left + << " prefix_right=" << prefix_right; + EXPECT_EQ(result.prefix.offset, expected_prefix.offset) + << "suffix_left=" << suffix_left + << " prefix_right=" << prefix_right; + } + } + } +} + +TEST(ExcessPositions128, DisjointBoundaryPairMatchesIndependentFixedCases) { + const std::array, 5> cases = {{ + {0, 0}, + {UINT64_MAX, UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull}, + {0x0000FFFF0000FFFFull, 0xFFFF0000FFFF0000ull}, + }}; + const std::array, 10> ranges = {{ + {1, 0}, + {4, 3}, + {17, 16}, + {32, 31}, + {63, 62}, + {64, 32}, + {65, 63}, + {96, 31}, + {127, 0}, + {120, 119}, + }}; + + for (const auto& suffix : cases) { + for (const auto& prefix : cases) { + for (const auto [suffix_left, prefix_right] : ranges) { + check_boundary_pair_matches_independent(suffix, suffix_left, prefix, + prefix_right); + } + } + } +} + +TEST(ExcessPositions128, DisjointBoundaryPairMatchesIndependentRandom) { + std::mt19937_64 rng(45); + std::uniform_int_distribution prefix_dist(0, 126); + + for (int t = 0; t < 1000; ++t) { + const std::array suffix = {rng(), rng()}; + const std::array prefix = {rng(), rng()}; + for (int q = 0; q < 16; ++q) { + const size_t prefix_right = prefix_dist(rng); + std::uniform_int_distribution suffix_dist(prefix_right + 1, 127); + const size_t suffix_left = suffix_dist(rng); + check_boundary_pair_matches_independent(suffix, suffix_left, prefix, + prefix_right); + } + } +} + +TEST(ExcessPositions128, BoundaryPairFallbackMatchesIndependent) { + const std::array suffix = {0x0123456789ABCDEFull, + 0xFEDCBA9876543210ull}; + const std::array prefix = {0x0000FFFF0000FFFFull, + 0xFFFF0000FFFF0000ull}; + + check_boundary_pair_matches_independent(suffix, 32, prefix, 32); + check_boundary_pair_matches_independent(suffix, 0, prefix, 127); + check_boundary_pair_matches_independent(suffix, 128, prefix, 0); +} + +TEST(ExcessPositions128Experimental, MinVariantsMatchNaive) { + const std::array, 6> cases = {{ + {0, 0}, + {UINT64_MAX, UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull}, + {0x0000FFFF0000FFFFull, 0xFFFF0000FFFF0000ull}, + {0x1111222233334444ull, 0x8888777766665555ull}, + }}; + const std::array, 25> ranges = {{ + {0, 128}, {0, 127}, {0, 16}, {0, 31}, {0, 32}, + {0, 48}, {0, 64}, {32, 64}, {64, 96}, {96, 128}, + {56, 72}, {60, 68}, {63, 64}, {17, 17}, {1, 2}, + {2, 3}, {3, 6}, {5, 5}, {64, 64}, {64, 128}, + {127, 128}, {128, 128}, {120, 127}, {129, 140}, {17, 16}, + }}; + + int case_id = 0; + for (const auto& s : cases) { + for (const auto [left, right] : ranges) { + check_min_matches_naive(excess_min_128_scalar_bits, "scalar_bits", + s.data(), left, right, case_id); + check_min_matches_naive(excess_min_128_nibble_lut, "nibble_lut", s.data(), + left, right, case_id); + check_min_matches_naive(excess_min_128_byte_lut, "byte_lut", s.data(), + left, right, case_id); + check_min_matches_naive(excess_min_128_hybrid_lut, "hybrid_lut", s.data(), + left, right, case_id); +#ifdef PIXIE_AVX2_SUPPORT + check_min_matches_naive(excess_min_128_expand16_avx2, "expand16_avx2", + s.data(), left, right, case_id); + check_min_matches_naive(excess_min_128_deinterleaved_sse, + "deinterleaved_sse", s.data(), left, right, + case_id); + check_min_matches_naive(excess_min_128_deinterleaved_full_sse, + "deinterleaved_full_sse", s.data(), left, right, + case_id); + check_min_matches_naive(excess_min_128_deinterleaved_byte16_sse, + "deinterleaved_byte16_sse", s.data(), left, right, + case_id); + check_min_matches_naive(excess_min_128_lane64_sse, "lane64_sse", s.data(), + left, right, case_id); + check_min_matches_naive(excess_min_128_split64_sse, "split64_sse", + s.data(), left, right, case_id); + check_min_matches_naive(excess_min_128_short_skip, "short_skip", s.data(), + left, right, case_id); +#endif + ++case_id; + } + } +} + +TEST(ExcessPositions128Experimental, MinVariantsMatchNaiveRandom) { + std::mt19937_64 rng(44); + std::uniform_int_distribution offset_dist(0, 140); + + for (int t = 0; t < 500; ++t) { + const std::array s = {rng(), rng()}; + for (int q = 0; q < 16; ++q) { + const size_t left = offset_dist(rng); + const size_t right = offset_dist(rng); + check_min_matches_naive(excess_min_128_scalar_bits, "scalar_bits", + s.data(), left, right, t); + check_min_matches_naive(excess_min_128_nibble_lut, "nibble_lut", s.data(), + left, right, t); + check_min_matches_naive(excess_min_128_byte_lut, "byte_lut", s.data(), + left, right, t); + check_min_matches_naive(excess_min_128_hybrid_lut, "hybrid_lut", s.data(), + left, right, t); +#ifdef PIXIE_AVX2_SUPPORT + check_min_matches_naive(excess_min_128_expand16_avx2, "expand16_avx2", + s.data(), left, right, t); + check_min_matches_naive(excess_min_128_deinterleaved_sse, + "deinterleaved_sse", s.data(), left, right, t); + check_min_matches_naive(excess_min_128_deinterleaved_full_sse, + "deinterleaved_full_sse", s.data(), left, right, + t); + check_min_matches_naive(excess_min_128_deinterleaved_byte16_sse, + "deinterleaved_byte16_sse", s.data(), left, right, + t); + check_min_matches_naive(excess_min_128_lane64_sse, "lane64_sse", s.data(), + left, right, t); + check_min_matches_naive(excess_min_128_split64_sse, "split64_sse", + s.data(), left, right, t); + check_min_matches_naive(excess_min_128_short_skip, "short_skip", s.data(), + left, right, t); +#endif + } + } +} + +#ifdef PIXIE_AVX2_SUPPORT +TEST(ExcessPositions128Experimental, DeinterleavedSseBoundaryAndTieCases) { + const std::array, 5> cases = {{ + {0x0F0F0F0F0F0F0F0Full, 0xF0F0F0F0F0F0F0F0ull}, + {0x3333333333333333ull, 0xCCCCCCCCCCCCCCCCull}, + {0x6666666666666666ull, 0x9999999999999999ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull}, + {0x1111111111111111ull, 0xEEEEEEEEEEEEEEEEull}, + }}; + const std::array, 13> ranges = {{ + {0, 5}, + {4, 7}, + {4, 11}, + {8, 15}, + {12, 19}, + {1, 17}, + {3, 35}, + {5, 37}, + {33, 65}, + {61, 93}, + {124, 127}, + {125, 128}, + {0, 127}, + }}; + + int case_id = 0; + for (const auto& s : cases) { + for (const auto [left, right] : ranges) { + check_min_matches_naive(excess_min_128_deinterleaved_sse, + "deinterleaved_sse", s.data(), left, right, + case_id); + check_min_matches_naive(excess_min_128_deinterleaved_full_sse, + "deinterleaved_full_sse", s.data(), left, right, + case_id); + check_min_matches_naive(excess_min_128_deinterleaved_byte16_sse, + "deinterleaved_byte16_sse", s.data(), left, right, + case_id); + ++case_id; + } + } +} +#endif + TEST(ExcessPositions128, ForwardAndBackwardSearchMatchNaive) { std::mt19937_64 rng(42); const std::array offsets = {0, 1, 63, 64, 65, 126, 127, 128}; diff --git a/src/tests/excess_record_lows_tests.cpp b/src/tests/excess_record_lows_tests.cpp new file mode 100644 index 0000000..a39d85f --- /dev/null +++ b/src/tests/excess_record_lows_tests.cpp @@ -0,0 +1,292 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Naive reference implementations +// --------------------------------------------------------------------------- + +static void naive_excess_record_lows_128(const uint64_t* s, uint64_t* out) { + out[0] = out[1] = 0; + int cur = 0; + int best = 0; + for (size_t i = 0; i < 128; ++i) { + const uint64_t w = s[i >> 6]; + const int bit = static_cast((w >> (i & 63)) & 1ull); + cur += bit ? +1 : -1; + if (cur < best) { + best = cur; + out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } +} + +static void naive_excess_record_lows_512(const uint64_t* s, uint64_t* out) { + for (int w = 0; w < 8; ++w) { + out[w] = 0; + } + int cur = 0; + int best = 0; + for (size_t i = 0; i < 512; ++i) { + const uint64_t w = s[i >> 6]; + const int bit = static_cast((w >> (i & 63)) & 1ull); + cur += bit ? +1 : -1; + if (cur < best) { + best = cur; + out[i >> 6] |= (uint64_t{1} << (i & 63)); + } + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +static void check_128_matches_naive(const uint64_t* s, int case_id = 0) { + alignas(16) uint64_t out[2]; + alignas(16) uint64_t ref[2]; + excess_record_lows_128(s, out); + naive_excess_record_lows_128(s, ref); + EXPECT_EQ(out[0], ref[0]) << "case=" << case_id; + EXPECT_EQ(out[1], ref[1]) << "case=" << case_id; +} + +static void check_512_matches_naive(const uint64_t* s, int case_id = 0) { + alignas(64) uint64_t out[8]; + alignas(64) uint64_t ref[8]; + excess_record_lows_512(s, out); + naive_excess_record_lows_512(s, ref); + for (int w = 0; w < 8; ++w) { + EXPECT_EQ(out[w], ref[w]) << "case=" << case_id << " word=" << w; + } +} + +static size_t count_bits(const uint64_t* words, int num_words) { + size_t cnt = 0; + for (int i = 0; i < num_words; ++i) { + cnt += static_cast(std::popcount(words[i])); + } + return cnt; +} + +// --------------------------------------------------------------------------- +// 128-bit tests +// --------------------------------------------------------------------------- + +TEST(ExcessRecordLows128, AllZeros) { + const std::array s = {0, 0}; + uint64_t out[2]; + excess_record_lows_128(s.data(), out); + // Excess goes 0, -1, -2, ..., -128. Every position is a new record low. + EXPECT_EQ(out[0], UINT64_MAX); + EXPECT_EQ(out[1], UINT64_MAX); +} + +TEST(ExcessRecordLows128, AllOnes) { + const std::array s = {UINT64_MAX, UINT64_MAX}; + uint64_t out[2]; + excess_record_lows_128(s.data(), out); + // Excess goes 0, 1, 2, ..., 128. No new record lows after start. + EXPECT_EQ(out[0], 0u); + EXPECT_EQ(out[1], 0u); +} + +TEST(ExcessRecordLows128, Alternating) { + const std::array s = {0xAAAAAAAAAAAAAAAAull, + 0x5555555555555555ull}; + uint64_t out[2]; + excess_record_lows_128(s.data(), out); + // Pattern 1010... -> excess: 0, -1, 0, -1, 0, ... + // Only the first position (i=0) is a record low (excess=-1). + // Actually i=0 -> first bit=1 -> excess=+1, i=1 -> bit=0 -> excess=0, wait. + // 0xAA = 10101010, 0x55 = 01010101. + // Let's just trust the naive for correctness. + alignas(16) uint64_t ref[2]; + naive_excess_record_lows_128(s.data(), ref); + EXPECT_EQ(out[0], ref[0]); + EXPECT_EQ(out[1], ref[1]); +} + +TEST(ExcessRecordLows128, FixedCases) { + const std::array, 6> cases = {{ + {0, 0}, + {UINT64_MAX, UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull}, + {0x0000FFFF0000FFFFull, 0xFFFF0000FFFF0000ull}, + {0x1111222233334444ull, 0x8888777766665555ull}, + }}; + + int case_id = 0; + for (const auto& s : cases) { + check_128_matches_naive(s.data(), case_id++); + } +} + +TEST(ExcessRecordLows128, ExhaustiveSmall16) { + alignas(16) uint64_t s[2]; + alignas(16) uint64_t out[2]; + alignas(16) uint64_t ref[2]; + + for (uint64_t pattern = 0; pattern < (1ull << 16); ++pattern) { + s[0] = pattern; + s[1] = 0; + excess_record_lows_128(s, out); + naive_excess_record_lows_128(s, ref); + EXPECT_EQ(out[0], ref[0]) << "pattern=" << pattern; + EXPECT_EQ(out[1], ref[1]) << "pattern=" << pattern; + } +} + +TEST(ExcessRecordLows128, Random) { + const int cases = [] { + const char* env = std::getenv("RECORD_LOWS_CASES"); + return env ? std::atoi(env) : 10000; + }(); + const uint64_t seed = [] { + const char* env = std::getenv("RECORD_LOWS_SEED"); + return env ? std::stoull(env) : 42ull; + }(); + + std::mt19937_64 rng(static_cast(seed)); + + for (int t = 0; t < cases; ++t) { + const std::array s = {rng(), rng()}; + check_128_matches_naive(s.data(), t); + } +} + +#ifdef PIXIE_AVX2_SUPPORT +static void check_nibble_lut_matches_naive(const uint64_t* s, int case_id = 0) { + alignas(16) uint64_t out[2]; + alignas(16) uint64_t ref[2]; + excess_record_lows_128_nibble_lut(s, out); + naive_excess_record_lows_128(s, ref); + EXPECT_EQ(out[0], ref[0]) << "nibble case=" << case_id; + EXPECT_EQ(out[1], ref[1]) << "nibble case=" << case_id; +} + +TEST(ExcessRecordLows128, NibbleLUTExhaustiveSmall16) { + alignas(16) uint64_t s[2]; + for (uint64_t pattern = 0; pattern < (1ull << 16); ++pattern) { + s[0] = pattern; + s[1] = 0; + check_nibble_lut_matches_naive(s, static_cast(pattern)); + } +} + +TEST(ExcessRecordLows128, NibbleLUTRandom) { + std::mt19937_64 rng(12345); + for (int t = 0; t < 10000; ++t) { + const std::array s = {rng(), rng()}; + check_nibble_lut_matches_naive(s.data(), t); + } +} +#endif + +// --------------------------------------------------------------------------- +// 512-bit tests +// --------------------------------------------------------------------------- + +TEST(ExcessRecordLows512, AllZeros) { + alignas(64) uint64_t s[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + alignas(64) uint64_t out[8]; + excess_record_lows_512(s, out); + for (int w = 0; w < 8; ++w) { + EXPECT_EQ(out[w], UINT64_MAX) << "word=" << w; + } +} + +TEST(ExcessRecordLows512, AllOnes) { + alignas(64) uint64_t s[8] = {UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX, + UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX}; + alignas(64) uint64_t out[8]; + excess_record_lows_512(s, out); + for (int w = 0; w < 8; ++w) { + EXPECT_EQ(out[w], 0u) << "word=" << w; + } +} + +TEST(ExcessRecordLows512, FixedCases) { + const std::array, 4> cases = {{ + {0, 0, 0, 0, 0, 0, 0, 0}, + {UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX, + UINT64_MAX, UINT64_MAX}, + {0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull, 0xAAAAAAAAAAAAAAAAull, + 0x5555555555555555ull, 0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull, + 0xAAAAAAAAAAAAAAAAull, 0x5555555555555555ull}, + {0x0123456789ABCDEFull, 0xFEDCBA9876543210ull, 0x0000FFFF0000FFFFull, + 0xFFFF0000FFFF0000ull, 0x1111222233334444ull, 0x8888777766665555ull, + 0x5555555555555555ull, 0xAAAAAAAAAAAAAAAAull}, + }}; + + int case_id = 0; + for (const auto& s : cases) { + check_512_matches_naive(s.data(), case_id++); + } +} + +TEST(ExcessRecordLows512, ExhaustiveSmall16) { + alignas(64) uint64_t s[8]; + alignas(64) uint64_t out[8]; + alignas(64) uint64_t ref[8]; + + for (uint64_t pattern = 0; pattern < (1ull << 16); ++pattern) { + s[0] = pattern; + for (int w = 1; w < 8; ++w) { + s[w] = 0; + } + excess_record_lows_512(s, out); + naive_excess_record_lows_512(s, ref); + for (int w = 0; w < 8; ++w) { + EXPECT_EQ(out[w], ref[w]) << "pattern=" << pattern << " word=" << w; + } + } +} + +TEST(ExcessRecordLows512, Random) { + const int cases = [] { + const char* env = std::getenv("RECORD_LOWS_CASES"); + return env ? std::atoi(env) : 10000; + }(); + const uint64_t seed = [] { + const char* env = std::getenv("RECORD_LOWS_SEED"); + return env ? std::stoull(env) : 42ull; + }(); + + std::mt19937_64 rng(static_cast(seed)); + alignas(64) uint64_t s[8]; + + for (int t = 0; t < cases; ++t) { + for (int w = 0; w < 8; ++w) { + s[w] = rng(); + } + check_512_matches_naive(s, t); + } +} + +TEST(ExcessRecordLows512, CountMonotonic) { + // Record-low count in a 512-bit all-zeros should be 512. + alignas(64) uint64_t s[8] = {0}; + alignas(64) uint64_t out[8]; + excess_record_lows_512(s, out); + EXPECT_EQ(count_bits(out, 8), 512u); + + // Record-low count in a 512-bit all-ones should be 0. + for (int w = 0; w < 8; ++w) { + s[w] = UINT64_MAX; + } + excess_record_lows_512(s, out); + EXPECT_EQ(count_bits(out, 8), 0u); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/tests/rmq_tests.cpp b/src/tests/rmq_tests.cpp new file mode 100644 index 0000000..6e12d55 --- /dev/null +++ b/src/tests/rmq_tests.cpp @@ -0,0 +1,1189 @@ +#include +#include +#include +#include +#include +#include + +#ifdef SDSL_SUPPORT +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +template +std::size_t naive_arg_min(std::span values, + std::size_t left, + std::size_t right, + Compare compare) { + if (left >= right || right > values.size()) { + return pixie::rmq::SparseTable::npos; + } + std::size_t best = left; + for (std::size_t i = left + 1; i < right; ++i) { + if (compare(values[i], values[best])) { + best = i; + } + } + return best; +} + +template +void check_all_ranges(const Rmq& rmq, + std::span values, + Compare compare) { + ASSERT_EQ(rmq.size(), values.size()); + for (std::size_t left = 0; left < values.size(); ++left) { + for (std::size_t right = left + 1; right <= values.size(); ++right) { + const std::size_t expected = naive_arg_min(values, left, right, compare); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(rmq.range_min(left, right), values[expected]) + << "range=[" << left << "," << right << ")"; + } + } +} + +template +void check_all_arg_min_ranges(const Rmq& rmq, + std::span values, + Compare compare) { + ASSERT_EQ(rmq.size(), values.size()); + for (std::size_t left = 0; left < values.size(); ++left) { + for (std::size_t right = left + 1; right <= values.size(); ++right) { + const std::size_t expected = naive_arg_min(values, left, right, compare); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } + } +} + +bool packed_bit(std::span words, std::size_t position) { + return ((words[position >> 6] >> (position & 63)) & 1u) != 0; +} + +std::vector make_packed_delta_bits(std::size_t bit_count) { + std::vector words((bit_count + 63) / 64); + for (std::size_t position = 0; position < bit_count; ++position) { + const bool bit = ((position * 11 + position / 3) % 17) < 8; + if (bit) { + words[position >> 6] |= std::uint64_t{1} << (position & 63); + } + } + return words; +} + +std::vector depths_from_delta_bits( + std::span words, + std::size_t depth_count) { + std::vector depths(depth_count); + for (std::size_t position = 1; position < depth_count; ++position) { + depths[position] = + depths[position - 1] + (packed_bit(words, position - 1) ? 1 : -1); + } + return depths; +} + +std::size_t naive_depth_arg_min(std::span depths, + std::size_t left, + std::size_t right) { + if (left >= right || right > depths.size()) { + return std::numeric_limits::max(); + } + std::size_t best = left; + for (std::size_t position = left + 1; position < right; ++position) { + if (depths[position] < depths[best]) { + best = position; + } + } + return best; +} + +template +void expect_valid_bp_encoding(const Rmq& rmq, std::size_t value_count) { + const std::size_t bit_count = rmq.bp_bit_count(); + const std::span words = rmq.bp_words(); + EXPECT_EQ(bit_count, 2 * value_count); + EXPECT_EQ(words.size(), (bit_count + 63) / 64); + + std::size_t ones = 0; + std::size_t zeros = 0; + std::int64_t excess = 0; + for (std::size_t position = 0; position < bit_count; ++position) { + if (packed_bit(words, position)) { + ++ones; + ++excess; + } else { + ++zeros; + --excess; + } + EXPECT_GE(excess, 0) << "position=" << position; + } + + EXPECT_EQ(ones, value_count); + EXPECT_EQ(zeros, value_count); + EXPECT_EQ(excess, 0); +} + +template +std::string bp_string(const Rmq& rmq) { + std::string bits; + bits.reserve(rmq.bp_bit_count()); + const std::span words = rmq.bp_words(); + for (std::size_t position = 0; position < rmq.bp_bit_count(); ++position) { + bits.push_back(packed_bit(words, position) ? '1' : '0'); + } + return bits; +} + +template +void expect_cartesian_bp_shape(const std::vector& values, + const std::string& expected) { + const Rmq rmq{std::span(values)}; + EXPECT_EQ(bp_string(rmq), expected); + check_all_arg_min_ranges(rmq, std::span(values), std::less()); +} + +struct SparseTableCase { + using Rmq = pixie::rmq::SparseTable; + using MaxRmq = pixie::rmq::SparseTable>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + +struct SegmentTreeCase { + using Rmq = pixie::rmq::SegmentTree; + using MaxRmq = pixie::rmq::SegmentTree>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + +struct CartesianRmMCase { + using Rmq = pixie::rmq::CartesianRmM; + using MaxRmq = pixie::rmq::CartesianRmM>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + +struct CartesianHybridBTreeCase { + using Rmq = pixie::rmq::CartesianHybridBTree; + using MaxRmq = pixie::rmq::CartesianHybridBTree>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + +struct HybridBTreeCase { + using Rmq = pixie::rmq::HybridBTree; + using MaxRmq = pixie::rmq::HybridBTree>; + + static Rmq make(std::span values) { return Rmq(values); } + + static MaxRmq make_max(std::span values) { return MaxRmq(values); } +}; + +} // namespace + +template +class ValueRmqSpecificationTest : public ::testing::Test {}; + +using ValueRmqCases = ::testing::Types; +TYPED_TEST_SUITE(ValueRmqSpecificationTest, ValueRmqCases); + +TYPED_TEST(ValueRmqSpecificationTest, ExhaustiveSmallArray) { + const std::vector values = {4, 1, 3, 1, 5, 0, 0, 2}; + const typename TypeParam::Rmq rmq = + TypeParam::make(std::span(values)); + check_all_ranges(rmq, std::span(values), std::less()); +} + +TEST(RmqCartesianBuildShape, SuccinctStackPreservesBpEncoding) { + using Hybrid = pixie::rmq::CartesianHybridBTree; + using RmM = pixie::rmq::CartesianRmM; + + const std::vector, std::string>> cases = { + {{1}, "10"}, + {{1, 2}, "1010"}, + {{2, 1}, "1100"}, + {{1, 1}, "1010"}, + {{3, 1, 2}, "110010"}, + {{1, 3, 2}, "101100"}, + {{3, 2, 2, 2, 1, 1, 2, 1, 3}, "111001010010110010"}, + {{4, 1, 3, 1, 5, 0, 0, 2}, "1110011001001010"}, + }; + + for (const auto& [values, expected] : cases) { + SCOPED_TRACE(expected); + expect_cartesian_bp_shape(values, expected); + expect_cartesian_bp_shape(values, expected); + } +} + +TEST(RmqSuccinctIncreasingStack, PopsAcrossEmptyBlocks) { + pixie::rmq::utils::SuccinctIncreasingStack stack(256); + EXPECT_TRUE(stack.empty()); + + stack.push(0); + EXPECT_EQ(stack.top(), 0u); + stack.push(62); + stack.push(63); + EXPECT_EQ(stack.top(), 63u); + stack.pop(); + EXPECT_EQ(stack.top(), 62u); + stack.pop(); + EXPECT_EQ(stack.top(), 0u); + + stack.push(130); + EXPECT_EQ(stack.top(), 130u); + stack.pop(); + EXPECT_EQ(stack.top(), 0u); + stack.push(131); + stack.pop(); + EXPECT_EQ(stack.top(), 0u); + + stack.push(190); + stack.push(191); + EXPECT_EQ(stack.top(), 191u); + stack.pop(); + EXPECT_EQ(stack.top(), 190u); + stack.pop(); + EXPECT_EQ(stack.top(), 0u); + stack.pop(); + EXPECT_TRUE(stack.empty()); +} + +TYPED_TEST(ValueRmqSpecificationTest, FirstMinimumTieBreaking) { + const std::vector values = {7, 2, 2, 3, 2}; + const typename TypeParam::Rmq rmq = + TypeParam::make(std::span(values)); + + EXPECT_EQ(rmq.arg_min(0, 5), 1u); + EXPECT_EQ(rmq.arg_min(2, 5), 2u); +} + +TYPED_TEST(ValueRmqSpecificationTest, InvalidAndEmptyRanges) { + using Rmq = typename TypeParam::Rmq; + + const std::vector values = {3, 1, 2}; + const Rmq rmq = TypeParam::make(std::span(values)); + + EXPECT_EQ(rmq.arg_min(2, 2), Rmq::npos); + EXPECT_EQ(rmq.arg_min(2, 1), Rmq::npos); + EXPECT_EQ(rmq.arg_min(0, values.size() + 1), Rmq::npos); + EXPECT_EQ(rmq.range_min(2, 2), 0); + EXPECT_EQ(rmq.arg_min(0, values.size()), 1u); + + const std::vector empty_values; + const Rmq default_rmq; + const Rmq empty_rmq = TypeParam::make(std::span(empty_values)); + EXPECT_TRUE(default_rmq.empty()); + EXPECT_TRUE(empty_rmq.empty()); + EXPECT_EQ(default_rmq.arg_min(0, 0), Rmq::npos); + EXPECT_EQ(empty_rmq.arg_min(0, 0), Rmq::npos); + EXPECT_EQ(default_rmq.range_min(0, 0), 0); + EXPECT_EQ(empty_rmq.range_min(0, 0), 0); +} + +TYPED_TEST(ValueRmqSpecificationTest, MemoryUsageCountsOwnedIndexStorage) { + using Rmq = typename TypeParam::Rmq; + using MaxRmq = typename TypeParam::MaxRmq; + + const Rmq default_rmq; + EXPECT_GE(default_rmq.memory_usage_bytes(), sizeof(Rmq)); + + std::vector values(1537); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 19 + i / 5) % 127); + } + + const Rmq rmq = TypeParam::make(std::span(values)); + const MaxRmq max_rmq = TypeParam::make_max(std::span(values)); + EXPECT_GE(rmq.memory_usage_bytes(), sizeof(Rmq)); + EXPECT_GE(max_rmq.memory_usage_bytes(), sizeof(MaxRmq)); + EXPECT_GT(rmq.memory_usage_bytes(), default_rmq.memory_usage_bytes()); +} + +TYPED_TEST(ValueRmqSpecificationTest, ComparatorCanSelectMaximum) { + using MaxRmq = typename TypeParam::MaxRmq; + + const std::vector values = {1, 8, 3, 8, 4}; + const MaxRmq rmq = TypeParam::make_max(std::span(values)); + + check_all_ranges(rmq, std::span(values), std::greater()); + EXPECT_EQ(rmq.arg_min(0, 5), 1u); + EXPECT_EQ(rmq.arg_min(2, 2), MaxRmq::npos); + EXPECT_EQ(rmq.arg_min(4, 3), MaxRmq::npos); + EXPECT_EQ(rmq.arg_min(0, values.size() + 1), MaxRmq::npos); + EXPECT_EQ(rmq.range_min(2, 2), 0); + EXPECT_EQ(rmq.range_min(4, 3), 0); + EXPECT_EQ(rmq.range_min(0, values.size() + 1), 0); + + const std::vector empty_values; + const MaxRmq default_rmq; + const MaxRmq empty_rmq = + TypeParam::make_max(std::span(empty_values)); + EXPECT_TRUE(default_rmq.empty()); + EXPECT_TRUE(empty_rmq.empty()); + EXPECT_EQ(default_rmq.arg_min(0, 0), MaxRmq::npos); + EXPECT_EQ(empty_rmq.arg_min(0, 0), MaxRmq::npos); + EXPECT_EQ(default_rmq.range_min(0, 0), 0); + EXPECT_EQ(empty_rmq.range_min(0, 0), 0); +} + +TYPED_TEST(ValueRmqSpecificationTest, MonotoneArrays) { + const std::vector increasing = {1, 2, 3, 4, 5, 6}; + const std::vector decreasing = {6, 5, 4, 3, 2, 1}; + const typename TypeParam::Rmq increasing_rmq = + TypeParam::make(std::span(increasing)); + const typename TypeParam::Rmq decreasing_rmq = + TypeParam::make(std::span(decreasing)); + + check_all_ranges(increasing_rmq, std::span(increasing), + std::less()); + check_all_ranges(decreasing_rmq, std::span(decreasing), + std::less()); +} + +TYPED_TEST(ValueRmqSpecificationTest, DifferentialRandom) { + std::mt19937_64 rng(42); + std::uniform_int_distribution value_dist(-50, 50); + for (std::size_t size = 1; size <= 257; size += 17) { + std::vector values(size); + std::generate(values.begin(), values.end(), + [&] { return value_dist(rng); }); + + const typename TypeParam::Rmq rmq = + TypeParam::make(std::span(values)); + check_all_ranges(rmq, std::span(values), std::less()); + } +} + +TEST(RmqSparseTable, OverlappingBlockCandidateDirectionsAndTies) { + { + const std::vector values = {5, 4, 3, 2, 1, 0, 9}; + const pixie::rmq::SparseTable rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 5u); + EXPECT_EQ(rmq.range_min(0, 6), 0); + } + + { + const std::vector values = {0, 5, 6, 7, 8, 9, 1}; + const pixie::rmq::SparseTable rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 0u); + EXPECT_EQ(rmq.range_min(0, 6), 0); + } + + { + const std::vector values = {1, 0, 9, 9, 0, 2}; + const pixie::rmq::SparseTable rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 1u); + EXPECT_EQ(rmq.range_min(0, 6), 0); + } + + { + const std::vector values = {1, 2, 3, 4, 9, 10, 0}; + const pixie::rmq::SparseTable> rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 5u); + EXPECT_EQ(rmq.range_min(0, 6), 10); + } + + { + const std::vector values = {10, 9, 8, 7, 6, 5, 20}; + const pixie::rmq::SparseTable> rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 0u); + EXPECT_EQ(rmq.range_min(0, 6), 10); + } + + { + const std::vector values = {10, 1, 2, 3, 10, 4}; + const pixie::rmq::SparseTable> rmq(values); + EXPECT_EQ(rmq.arg_min(0, 6), 0u); + EXPECT_EQ(rmq.range_min(0, 6), 10); + } +} + +TEST(RmqSegmentTree, NonPowerOfTwoTailRanges) { + const std::vector values = {6, 5, 4, 3, 2}; + const pixie::rmq::SegmentTree rmq(values); + check_all_ranges(rmq, std::span(values), std::less()); +} + +#ifdef SDSL_SUPPORT +TEST(RmqSdslSct, + MatchesPixieValueRmqSpecificationForSignedValuesAndDuplicates) { + const std::vector values = {4, -3, 7, -3, 0, -8, -8, 2, 2}; + const pixie::rmq::SdslSct rmq{std::span(values)}; + + check_all_ranges(rmq, std::span(values), std::less()); + EXPECT_EQ(rmq.arg_min(0, values.size()), 5u); + EXPECT_EQ(rmq.arg_min(5, 7), 5u); + EXPECT_EQ(rmq.arg_min(6, 7), 6u); + EXPECT_EQ(rmq.arg_min(3, 3), pixie::rmq::SdslSct::npos); + EXPECT_EQ(rmq.arg_min(0, values.size() + 1), pixie::rmq::SdslSct::npos); +} + +TEST(RmqSdslSct, DifferentialRandom) { + std::mt19937_64 rng(123); + std::uniform_int_distribution value_dist(-20, 20); + for (std::size_t size = 1; size <= 129; size += 16) { + std::vector values(size); + std::generate(values.begin(), values.end(), + [&] { return value_dist(rng); }); + + const pixie::rmq::SdslSct rmq{std::span(values)}; + check_all_ranges(rmq, std::span(values), std::less()); + } + + const std::vector empty_values; + const pixie::rmq::SdslSct empty{std::span(empty_values)}; + EXPECT_TRUE(empty.empty()); + EXPECT_EQ(empty.arg_min(0, 0), pixie::rmq::SdslSct::npos); +} +#endif + +TEST(RmqHybridBTree, BoundaryAndFallbackRanges) { + using Rmq = pixie::rmq::HybridBTree; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + static_assert(kLeaf == 496); + + std::vector values(2 * kLeaf + 17, 2000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 7 + i / 3) % 31); + } + + values[32] = 120; + values[96] = 70; + values[160] = 55; + values[240] = -100; + values[241] = -100; + values[320] = 30; + values[440] = 5; + values[kLeaf - 1] = 5; + + values[kLeaf + 11] = -9; + values[kLeaf + 173] = -17; + values[2 * kLeaf + 5] = -21; + + const Rmq rmq{std::span(values)}; + const std::vector> ranges = { + {0, kLeaf}, {0, 97}, + {0, 240}, {96, 241}, + {250, kLeaf}, {441, kLeaf}, + {300, 450}, {kLeaf - 7, kLeaf + 12}, + {kLeaf, 2 * kLeaf}, {2 * kLeaf - 5, values.size()}, + {0, values.size()}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(rmq.range_min(left, right), values[expected]) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqHybridBTree, MaskLeafSelectorPrefixSuffixAndInteriorFallback) { + using Rmq = pixie::rmq::HybridBTree; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + + { + std::vector values(kLeaf, 1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast(i % 17); + } + values[0] = -10; + values[kLeaf - 1] = -100; + + const Rmq rmq{std::span(values)}; + const std::size_t right = kLeaf / 2; + EXPECT_EQ(rmq.arg_min(0, right), naive_arg_min(std::span(values), + 0, right, std::less())); + } + + { + std::vector values(kLeaf, 1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 5) % 23); + } + values[0] = -100; + values[kLeaf - 1] = -10; + + const Rmq rmq{std::span(values)}; + const std::size_t left = kLeaf / 2; + EXPECT_EQ(rmq.arg_min(left, kLeaf), + naive_arg_min(std::span(values), left, kLeaf, + std::less())); + } + + { + std::vector values(kLeaf, 1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 7) % 31); + } + values[0] = -100; + values[kLeaf / 2] = -50; + values[kLeaf - 1] = -80; + + const Rmq rmq{std::span(values)}; + const std::size_t left = kLeaf / 2 - 20; + const std::size_t right = kLeaf / 2 + 21; + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())); + } +} + +TEST(RmqHybridBTree, LeafSelectorEnumVariants) { + using MaskRmq = pixie::rmq::HybridBTree< + int, std::less, std::size_t, 248, 256, + pixie::rmq::HybridBTreeLeafSelector::PrefixSuffix>; + using BpRmq = + pixie::rmq::HybridBTree, std::size_t, 252, 256, + pixie::rmq::HybridBTreeLeafSelector::BP>; + + EXPECT_EQ(MaskRmq::top_sparse_block_size_for(0), + MaskRmq::kMinTopSparseBlockSize); + EXPECT_EQ(MaskRmq::top_sparse_block_count_for(0), 0u); + EXPECT_EQ(BpRmq::top_sparse_block_size_for(0), BpRmq::kMinTopSparseBlockSize); + EXPECT_EQ(BpRmq::top_sparse_block_count_for(0), 0u); + + std::vector values(4099, 10000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 37 + i / 7) % 257); + } + values[13] = -50; + values[251] = -70; + values[252] = -70; + values[747] = -90; + values[2048] = -120; + values[4098] = -110; + + const MaskRmq mask_rmq{std::span(values)}; + const BpRmq bp_rmq{std::span(values)}; + const std::vector empty_values; + const MaskRmq empty_mask_rmq; + const BpRmq empty_bp_rmq; + const MaskRmq empty_mask_span_rmq{std::span(empty_values)}; + const BpRmq empty_bp_span_rmq{std::span(empty_values)}; + EXPECT_EQ(empty_mask_rmq.arg_min(0, 0), MaskRmq::npos); + EXPECT_EQ(empty_bp_rmq.arg_min(0, 0), BpRmq::npos); + EXPECT_EQ(empty_mask_span_rmq.arg_min(0, 0), MaskRmq::npos); + EXPECT_EQ(empty_bp_span_rmq.arg_min(0, 0), BpRmq::npos); + EXPECT_EQ(empty_mask_rmq.range_min(0, 0), 0); + EXPECT_EQ(empty_bp_rmq.range_min(0, 0), 0); + EXPECT_EQ(empty_mask_span_rmq.range_min(0, 0), 0); + EXPECT_EQ(empty_bp_span_rmq.range_min(0, 0), 0); + + const std::vector> ranges = { + {0, 1}, {0, 13}, {0, 252}, {1, 251}, + {20, 40}, {100, 200}, {200, 248}, {248, 253}, + {251, 753}, {700, 2100}, {2048, 2049}, {0, values.size()}, + {3000, 4099}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(mask_rmq.arg_min(left, right), expected) + << "mask range=[" << left << "," << right << ")"; + EXPECT_EQ(bp_rmq.arg_min(left, right), expected) + << "bp range=[" << left << "," << right << ")"; + EXPECT_EQ(mask_rmq.range_min(left, right), values[expected]) + << "mask range=[" << left << "," << right << ")"; + EXPECT_EQ(bp_rmq.range_min(left, right), values[expected]) + << "bp range=[" << left << "," << right << ")"; + } + + const auto check_small_tree_paths = []() { + constexpr std::size_t kLeaf = Rmq::kLeafSize; + std::vector one_leaf_values(kLeaf / 2 + 3, 1000); + for (std::size_t i = 0; i < one_leaf_values.size(); ++i) { + one_leaf_values[i] += static_cast((i * 17 + i / 5) % 71); + } + one_leaf_values[one_leaf_values.size() / 2] = -20; + const Rmq one_leaf_rmq{std::span(one_leaf_values)}; + EXPECT_EQ(one_leaf_rmq.arg_min(0, one_leaf_values.size()), + naive_arg_min(std::span(one_leaf_values), 0, + one_leaf_values.size(), std::less())) + << "one leaf path leaf=" << kLeaf; + + std::vector small_values(3 * kLeaf + 5, 10000); + for (std::size_t i = 0; i < small_values.size(); ++i) { + small_values[i] += static_cast((i * 41 + i / 9) % 263); + } + small_values[7] = -100; + small_values[kLeaf + 3] = -500; + small_values[2 * kLeaf + 11] = -300; + + const Rmq rmq{std::span(small_values)}; + const std::vector> small_ranges = { + {0, small_values.size()}, {0, kLeaf}, + {kLeaf, 2 * kLeaf}, {2 * kLeaf, 3 * kLeaf}, + {1, small_values.size() - 1}, {kLeaf + 1, kLeaf + 2}, + }; + for (const auto [left, right] : small_ranges) { + const std::size_t expected = naive_arg_min( + std::span(small_values), left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "small tree range=[" << left << "," << right << ") leaf=" << kLeaf; + } + }; + + check_small_tree_paths.template operator()(); + check_small_tree_paths.template operator()(); +} + +TEST(RmqHybridBTree, BorderCorrectionForLeafSelectorVariants) { + using DefaultRmq = pixie::rmq::HybridBTree; + using SmallMaskRmq = pixie::rmq::HybridBTree< + int, std::less, std::size_t, 248, 256, + pixie::rmq::HybridBTreeLeafSelector::PrefixSuffix>; + using BpRmq = + pixie::rmq::HybridBTree, std::size_t, 252, 256, + pixie::rmq::HybridBTreeLeafSelector::BP>; + + const auto check = []() { + constexpr std::size_t kLeaf = Rmq::kLeafSize; + + { + std::vector values(3 * kLeaf, 100000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 13 + i / 5) % 97); + } + values[0] = -100000; + values[17] = -100; + values[kLeaf + 23] = -1000; + values[2 * kLeaf + 31] = -500; + + const Rmq rmq{std::span(values)}; + const std::size_t left = 1; + const std::size_t right = values.size() - 1; + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())) + << "left-border correction leaf=" << kLeaf; + } + + { + std::vector values(3 * kLeaf, 100000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 11 + i / 7) % 89); + } + values[11] = -500; + values[kLeaf + 29] = -1000; + values[2 * kLeaf + 41] = -100; + values[values.size() - 1] = -100000; + + const Rmq rmq{std::span(values)}; + const std::size_t left = 1; + const std::size_t right = values.size() - 1; + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())) + << "right-border correction leaf=" << kLeaf; + } + }; + + check.template operator()(); + check.template operator()(); + check.template operator()(); +} + +TEST(RmqHybridBTree, MiddleFanoutBoundaryRanges) { + using Rmq = pixie::rmq::HybridBTree; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + constexpr std::size_t kMiddleBoundary = kLeaf * Rmq::kMiddleFanout; + + std::vector values(kMiddleBoundary + 2 * kLeaf + 17, 10000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 17 + i / 11) % 257); + } + values[13] = -100; + values[kMiddleBoundary - 29] = -90; + values[kMiddleBoundary + kLeaf + 7] = -110; + + const Rmq rmq{std::span(values)}; + const std::vector> ranges = { + {0, values.size()}, + {kMiddleBoundary - 2 * kLeaf, kMiddleBoundary + 2 * kLeaf}, + {kMiddleBoundary - 31, kMiddleBoundary + kLeaf + 31}, + {kMiddleBoundary + 1, values.size()}, + {kLeaf * (Rmq::kMiddleFanout - 1) + 9, kMiddleBoundary + kLeaf + 13}, + {values.size() - 3 * kLeaf, values.size()}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqHybridBTree, TopSparseOverlayBoundaryRanges) { + using Rmq = pixie::rmq::HybridBTree; + constexpr std::size_t kTopBlock = Rmq::kMinTopSparseBlockSize; + + std::vector values(3 * kTopBlock + 77, 100000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 23 + i / 7) % 311); + } + values[0] = -1000; + values[kTopBlock + 17] = -900; + values[kTopBlock + 18] = -900; + values[2 * kTopBlock + 41] = -950; + values[3 * kTopBlock + 40] = -800; + + const Rmq rmq{std::span(values)}; + EXPECT_EQ(rmq.top_sparse_block_size(), kTopBlock); + EXPECT_EQ(rmq.top_sparse_block_count(), 4u); + + const std::vector> ranges = { + {1, 2 * kTopBlock + 100}, + {kTopBlock, kTopBlock + 100}, + {kTopBlock + 18, values.size()}, + {kTopBlock - 5, 2 * kTopBlock + 5}, + {0, values.size()}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqHybridBTree, TopSparseOverlayCapsBlockCount) { + using Rmq = pixie::rmq::HybridBTree; + constexpr std::size_t kMinBlock = Rmq::kMinTopSparseBlockSize; + constexpr std::size_t kMaxBlocks = Rmq::kMaxTopSparseBlocks; + constexpr std::size_t kLargestFixedBlockInput = kMinBlock * kMaxBlocks; + + EXPECT_EQ(Rmq::top_sparse_block_size_for(0), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(0), 0u); + EXPECT_EQ(Rmq::top_sparse_block_size_for(1), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(1), 1u); + EXPECT_EQ(Rmq::top_sparse_block_size_for(kLargestFixedBlockInput), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(kLargestFixedBlockInput), + kMaxBlocks); + EXPECT_GT(Rmq::top_sparse_block_size_for(kLargestFixedBlockInput + 1), + kMinBlock); + EXPECT_LE(Rmq::top_sparse_block_count_for(kLargestFixedBlockInput + 1), + kMaxBlocks); + + const std::size_t huge = std::numeric_limits::max() / 4; + EXPECT_LE(Rmq::top_sparse_block_count_for(huge), kMaxBlocks); + + std::vector values(3 * kMinBlock + 7, 0); + const Rmq rmq(values); + EXPECT_EQ(rmq.top_sparse_block_size(), kMinBlock); + EXPECT_EQ(rmq.top_sparse_block_count(), 4u); +} + +TEST(RmqHybridBTree, TopSparseOverlayComparatorMaximum) { + using Rmq = pixie::rmq::HybridBTree>; + constexpr std::size_t kTopBlock = Rmq::kMinTopSparseBlockSize; + EXPECT_EQ(Rmq::top_sparse_block_size_for(0), kTopBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(0), 0u); + + std::vector values(2 * kTopBlock + 33, -1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = -1000 + static_cast((i * 13 + i / 5) % 71); + } + values[0] = 1000; + values[kTopBlock + 17] = 900; + values[kTopBlock + 18] = 900; + + const Rmq rmq(values); + const std::vector> ranges = { + {1, values.size()}, + {kTopBlock, kTopBlock + 100}, + {kTopBlock + 18, values.size()}, + {0, values.size()}, + }; + + for (const auto [left, right] : ranges) { + const std::size_t expected = naive_arg_min( + std::span(values), left, right, std::greater()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqHybridBTree, InternalRootFullRangeShortcut) { + using Rmq = pixie::rmq::HybridBTree; + constexpr std::size_t kLeaf = Rmq::kLeafSize; + + std::vector values(7 * kLeaf + 19, 10000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 29 + i / 13) % 257); + } + values[17] = -100; + values[3 * kLeaf + 11] = -700; + values[values.size() - 3] = -200; + + const Rmq rmq{std::span(values)}; + EXPECT_EQ(rmq.arg_min(0, values.size()), 3 * kLeaf + 11); + EXPECT_EQ(rmq.range_min(0, values.size()), -700); + EXPECT_EQ(rmq.arg_min(0, kLeaf), naive_arg_min(std::span(values), + 0, kLeaf, std::less())); + EXPECT_EQ(rmq.arg_min(kLeaf - 3, 2 * kLeaf + 5), + naive_arg_min(std::span(values), kLeaf - 3, + 2 * kLeaf + 5, std::less())); +} + +TEST(RmqHybridBTree, DuplicateHeavyRandomTo8193) { + using Rmq = pixie::rmq::HybridBTree; + std::mt19937_64 rng(901496); + std::uniform_int_distribution value_dist(-3, 3); + const std::vector sizes = { + 1, 2, 17, 495, 496, 497, 1024, 4096, 8191, 8192, 8193, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = value_dist(rng); + if ((i % 11) < 6) { + values[i] = 0; + } + } + + const Rmq rmq{std::span(values)}; + std::uniform_int_distribution width_dist(1, size); + for (std::size_t query = 0; query < 2000; ++query) { + const std::size_t width = width_dist(rng); + std::uniform_int_distribution left_dist(0, size - width); + const std::size_t left = left_dist(rng); + const std::size_t right = left + width; + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } + } +} + +TEST(RmqCartesianHybridBTree, BoundarySizesAndBpEncoding) { + using Rmq = pixie::rmq::CartesianHybridBTree; + const std::vector sizes = { + 1u, 255u, 256u, 257u, 511u, 512u, 513u, 8191u, 8192u, 8193u, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 37 + i / 5) % 53); + if (i % 17 == 0 || i % 257 == 3) { + values[i] = -4; + } + } + + const Rmq rmq(values); + expect_valid_bp_encoding(rmq, values.size()); + + const std::vector> ranges = { + {0, 1}, + {0, size}, + {size - 1, size}, + {size / 3, std::min(size, size / 3 + 19)}, + {size / 2, std::min(size, size / 2 + 37)}, + }; + for (const auto [left, right] : ranges) { + ASSERT_LT(left, right); + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())) + << "range=[" << left << "," << right << ")"; + } + } +} + +TEST(RmqCartesianHybridBTree, BorderMinimaOutsideQueryFallBackToMiddle) { + using Rmq = pixie::rmq::CartesianHybridBTree; + constexpr std::size_t kLeaf = 512; + + { + std::vector values(3 * kLeaf, 100000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 17 + i / 9) % 101); + } + values[0] = -100000; + values[13] = -100; + values[kLeaf + 37] = -1000; + values[2 * kLeaf + 53] = -500; + + const Rmq rmq{std::span(values)}; + const std::size_t left = 1; + const std::size_t right = values.size() - 1; + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())); + } + + { + std::vector values(3 * kLeaf, 100000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] += static_cast((i * 19 + i / 11) % 107); + } + values[17] = -500; + values[kLeaf + 41] = -1000; + values[2 * kLeaf + 59] = -100; + values[values.size() - 1] = -100000; + + const Rmq rmq{std::span(values)}; + const std::size_t left = 1; + const std::size_t right = values.size() - 1; + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::less())); + } +} + +TEST(RmqCartesianHybridBTree, DuplicateHeavyRandomDifferentialTo8193) { + using Rmq = pixie::rmq::CartesianHybridBTree; + std::mt19937_64 rng(20260610); + std::uniform_int_distribution value_dist(-3, 3); + const std::vector sizes = { + 1, 2, 17, 255, 256, 257, 1024, 4096, 8191, 8192, 8193, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = value_dist(rng); + if ((i % 11) < 6) { + values[i] = 0; + } + } + + const Rmq rmq(values); + std::uniform_int_distribution width_dist(1, size); + for (std::size_t query = 0; query < 2000; ++query) { + const std::size_t width = width_dist(rng); + std::uniform_int_distribution left_dist(0, size - width); + const std::size_t left = left_dist(rng); + const std::size_t right = left + width; + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(rmq.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } + } +} + +TEST(RmqCartesianHybridBTree, DepthBackendDirectPaths) { + using HighSparseBackend = + pixie::rmq::detail::HybridBTreePlusMinusOne; + constexpr std::size_t kDepthCount = 3 * 512 + 17; + + const std::vector words = + make_packed_delta_bits(kDepthCount - 1); + const std::vector depths = depths_from_delta_bits( + std::span(words), kDepthCount); + const HighSparseBackend high_sparse(std::span(words), + kDepthCount); + + EXPECT_EQ(high_sparse.arg_min(kDepthCount, kDepthCount), + HighSparseBackend::npos); + EXPECT_EQ(high_sparse.arg_min(37, 38), 37u); + EXPECT_GT(high_sparse.memory_usage_bytes(), sizeof(HighSparseBackend)); + + const std::vector> ranges = { + {0, kDepthCount}, + {0, 512}, + {511, 1027}, + {9, 1400}, + {700, kDepthCount - 3}, + {kDepthCount - 65, kDepthCount}, + }; + for (const auto [left, right] : ranges) { + const std::size_t expected = + naive_depth_arg_min(std::span(depths), left, right); + EXPECT_EQ(high_sparse.arg_min(left, right), expected) + << "high sparse depth range=[" << left << "," << right << ")"; + } + + EXPECT_THROW(HighSparseBackend(std::span(), 65), + std::invalid_argument); +} + +TEST(RmqCartesianHybridBTree, BpStorageIsCacheLineAligned) { + using Rmq = pixie::rmq::CartesianHybridBTree; + constexpr std::size_t kLeafWords = 512 / 64; + const auto expect_aligned = [](const Rmq& rmq) { + const std::span words = rmq.bp_words(); + ASSERT_FALSE(words.empty()); + const auto base = reinterpret_cast( + static_cast(words.data())); + EXPECT_EQ(base % pixie::kAlignedStorageLineBytes, 0u); + + for (std::size_t word = 0; word < words.size(); word += kLeafWords) { + EXPECT_EQ((base + word * sizeof(std::uint64_t)) % + pixie::kAlignedStorageLineBytes, + 0u) + << "leaf_start_word=" << word; + } + }; + + const std::vector sizes = { + 1, 255, 256, 257, 4096, 8193, + }; + + for (const std::size_t size : sizes) { + SCOPED_TRACE(size); + std::vector values(size); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = static_cast((i * 31 + i / 3) % 97); + } + + const Rmq rmq(values); + Rmq copied(rmq); + Rmq assigned; + assigned = copied; + Rmq moved(std::move(copied)); + expect_aligned(rmq); + expect_aligned(assigned); + expect_aligned(moved); + } +} + +TEST(RmqCartesianHybridBTree, TopSparseOverlayBoundaryRanges) { + using Rmq = pixie::rmq::CartesianHybridBTree; + constexpr std::size_t kTopBlock = 4096; + std::vector values(3 * kTopBlock + 77, 1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = 1000 + static_cast((i * 17 + i / 7) % 89); + } + values[0] = -1000; + values[211] = -700; + values[kTopBlock + 123] = -900; + values[2 * kTopBlock + 300] = -800; + + const Rmq original(values); + Rmq copied(original); + Rmq assigned; + assigned = copied; + Rmq moved(std::move(copied)); + + const std::vector> ranges = { + {0, kTopBlock + 100}, + {100, 2 * kTopBlock + 100}, + {kTopBlock - 20, 2 * kTopBlock + 20}, + {333, kTopBlock + 10}, + {kTopBlock + 1, kTopBlock + 2000}, + {2 * kTopBlock + 10, values.size()}, + {0, values.size()}, + }; + + for (const auto [left, right] : ranges) { + ASSERT_LT(left, right); + const std::size_t expected = naive_arg_min(std::span(values), + left, right, std::less()); + EXPECT_EQ(original.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(assigned.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + EXPECT_EQ(moved.arg_min(left, right), expected) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqCartesianHybridBTree, TopSparseOverlayCapsBlockCount) { + using Rmq = pixie::rmq::CartesianHybridBTree; + constexpr std::size_t kMinBlock = Rmq::kMinTopSparseBlockSize; + constexpr std::size_t kMaxBlocks = Rmq::kMaxTopSparseBlocks; + constexpr std::size_t kLargestFixedBlockInput = kMinBlock * kMaxBlocks; + + EXPECT_EQ(Rmq::top_sparse_block_size_for(0), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(0), 0u); + EXPECT_EQ(Rmq::top_sparse_block_size_for(1), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(1), 1u); + EXPECT_EQ(Rmq::top_sparse_block_size_for(kLargestFixedBlockInput), kMinBlock); + EXPECT_EQ(Rmq::top_sparse_block_count_for(kLargestFixedBlockInput), + kMaxBlocks); + EXPECT_GT(Rmq::top_sparse_block_size_for(kLargestFixedBlockInput + 1), + kMinBlock); + EXPECT_LE(Rmq::top_sparse_block_count_for(kLargestFixedBlockInput + 1), + kMaxBlocks); + + const std::size_t huge = std::numeric_limits::max() / 4; + EXPECT_LE(Rmq::top_sparse_block_count_for(huge), kMaxBlocks); + + std::vector values(3 * kMinBlock + 7, 0); + const Rmq rmq(values); + EXPECT_EQ(rmq.top_sparse_block_size(), kMinBlock); + EXPECT_EQ(rmq.top_sparse_block_count(), 4u); +} + +TEST(RmqCartesianHybridBTree, TopSparseOverlayComparatorMaximum) { + using Rmq = pixie::rmq::CartesianHybridBTree>; + constexpr std::size_t kTopBlock = 4096; + std::vector values(2 * kTopBlock + 33, -1000); + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = -1000 + static_cast((i * 13 + i / 5) % 71); + } + values[0] = 1000; + values[kTopBlock + 17] = 900; + values[kTopBlock + 18] = 900; + + const Rmq rmq(values); + const std::vector> ranges = { + {1, values.size()}, + {kTopBlock, kTopBlock + 100}, + {kTopBlock + 18, values.size()}, + {0, values.size()}, + }; + + for (const auto [left, right] : ranges) { + ASSERT_LT(left, right); + EXPECT_EQ(rmq.arg_min(left, right), + naive_arg_min(std::span(values), left, right, + std::greater())) + << "range=[" << left << "," << right << ")"; + } +} + +TEST(RmqCartesianHybridBTree, ComparatorCopyAndMove) { + using MaxRmq = pixie::rmq::CartesianHybridBTree>; + const std::vector values = {1, 8, 8, 7, 8, 3, 8, 4, 4, 8}; + const MaxRmq original(values); + MaxRmq copied(original); + MaxRmq assigned; + assigned = copied; + MaxRmq moved(std::move(copied)); + + check_all_ranges(original, std::span(values), std::greater()); + check_all_ranges(assigned, std::span(values), std::greater()); + check_all_ranges(moved, std::span(values), std::greater()); + EXPECT_EQ(original.arg_min(0, values.size()), 1u); +} diff --git a/src/tests/test_rmm.cpp b/src/tests/test_rmm.cpp index 6955634..dc0d7d8 100644 --- a/src/tests/test_rmm.cpp +++ b/src/tests/test_rmm.cpp @@ -1084,6 +1084,97 @@ TEST(RmMBTreeExperimental, RankSelectIgnoresDirtyTrailingStorage) { EXPECT_EQ(rm.select0(2), pixie::experimental::RmMBTree<>::npos); } +TEST(RmMBTreeExperimental, OptionalSelectSupport) { + const std::string bits = "101100"; + auto words = pack_words_lsb_first(bits); + pixie::experimental::RmMBTree<> select0_only( + std::span(words), bits.size(), + pixie::BitVector::SelectSupport::kSelect0, /*one_count=*/3); + pixie::experimental::RmMBTree<> select1_only( + std::span(words), bits.size(), + pixie::BitVector::SelectSupport::kSelect1, /*one_count=*/3); + pixie::experimental::RmMBTree<> rank_only( + std::span(words), bits.size(), + pixie::BitVector::SelectSupport::kNone, /*one_count=*/3); + + EXPECT_EQ(select0_only.rank1(bits.size()), 3u); + EXPECT_EQ(select0_only.rank0(bits.size()), 3u); + EXPECT_EQ(select0_only.select1(1), pixie::experimental::RmMBTree<>::npos); + EXPECT_EQ(select0_only.select0(1), 1u); + EXPECT_EQ(select0_only.select0(2), 4u); + EXPECT_EQ(select0_only.select0(3), 5u); + EXPECT_EQ(select0_only.select0(4), pixie::experimental::RmMBTree<>::npos); + + EXPECT_EQ(select1_only.rank1(bits.size()), 3u); + EXPECT_EQ(select1_only.rank0(bits.size()), 3u); + EXPECT_EQ(select1_only.select1(1), 0u); + EXPECT_EQ(select1_only.select1(2), 2u); + EXPECT_EQ(select1_only.select1(3), 3u); + EXPECT_EQ(select1_only.select1(4), pixie::experimental::RmMBTree<>::npos); + EXPECT_EQ(select1_only.select0(1), pixie::experimental::RmMBTree<>::npos); + + EXPECT_EQ(rank_only.rank1(bits.size()), 3u); + EXPECT_EQ(rank_only.rank0(bits.size()), 3u); + EXPECT_EQ(rank_only.select1(1), pixie::experimental::RmMBTree<>::npos); + EXPECT_EQ(rank_only.select0(1), pixie::experimental::RmMBTree<>::npos); +} + +TEST(RmMBTreeExperimental, RangeMinResultAndMemoryUsage) { + using Tree = pixie::experimental::RmMBTree<>; + + Tree empty; + EXPECT_GE(empty.memory_usage_bytes(), sizeof(Tree)); + EXPECT_EQ(empty.size(), 0u); + EXPECT_EQ(empty.rank1(7), 0u); + EXPECT_EQ(empty.rank0(7), 0u); + EXPECT_EQ(empty.select1(1), Tree::npos); + EXPECT_EQ(empty.select0(1), Tree::npos); + EXPECT_EQ(empty.excess(7), 0); + EXPECT_EQ(empty.range_min_query_pos(0, 0), Tree::npos); + EXPECT_EQ(empty.range_min_query_val(0, 0), 0); + EXPECT_EQ(empty.range_max_query_pos(0, 0), Tree::npos); + EXPECT_EQ(empty.range_max_query_val(0, 0), 0); + EXPECT_EQ(empty.mincount(0, 0), 0u); + EXPECT_EQ(empty.minselect(0, 0, 1), Tree::npos); + + std::string bits; + bits.reserve(512 * 40 + 37); + for (size_t i = 0; i < 512 * 40 + 37; ++i) { + bits.push_back(((i * 19 + i / 13) % 9) < 4 ? '1' : '0'); + } + + auto words = pack_words_lsb_first(bits); + Tree rm(std::span(words), bits.size()); + NaiveRmM nv(bits); + + EXPECT_GT(rm.memory_usage_bytes(), empty.memory_usage_bytes()); + + const std::array, 6> ranges = { + std::pair{0, bits.size() - 1}, + std::pair{3, 97}, + std::pair{511, 512 * 3 + 7}, + std::pair{512, 512 * 32 + 11}, + std::pair{512 * 31 - 5, 512 * 40 + 7}, + std::pair{bits.size() - 19, bits.size() - 1}, + }; + + for (const auto& [left, right] : ranges) { + SCOPED_TRACE(::testing::Message() + << "range=[" << left << "," << right << "]"); + const auto result = rm.range_min_query_result(left, right); + EXPECT_EQ(result.position, nv.range_min_query_pos(left, right)); + EXPECT_EQ(result.value, nv.range_min_query_val(left, right)); + } + + const auto reversed = rm.range_min_query_result(3, 2); + EXPECT_EQ(reversed.position, Tree::npos); + EXPECT_EQ(reversed.value, 0); + + const auto out_of_bounds = rm.range_min_query_result(0, bits.size()); + EXPECT_EQ(out_of_bounds.position, Tree::npos); + EXPECT_EQ(out_of_bounds.value, 0); +} + TEST(RmMBTreeExperimental, ParenthesesOnUnmatchedBoundaryBits) { const std::string bits = "1"; auto words = pack_words_lsb_first(bits); @@ -1136,6 +1227,62 @@ TEST(RmMBTreeExperimental, InvalidArgumentsGuards) { EXPECT_EQ(rm.enclose(bits.size()), pixie::experimental::RmMBTree<>::npos); } +TEST(RmMBTreeExperimental, PartialBlockSearchesAndTailMinSelect) { + std::string bits; + bits.reserve(385); + for (size_t i = 0; i < 385; ++i) { + bits.push_back((i % 7 == 0 || i % 11 == 3 || i % 29 == 5) ? '1' : '0'); + } + + auto words = pack_words_lsb_first(bits); + pixie::experimental::RmMBTree<> rm(std::span(words), + bits.size()); + NaiveRmM nv(bits); + + const std::array positions = { + 0, 1, 63, 64, 127, 128, 255, 320, bits.size() - 1, + }; + for (size_t position : positions) { + for (int delta : {-120, -17, -2, -1, 0, 1, 2, 17, 120}) { + SCOPED_TRACE(::testing::Message() + << "position=" << position << " delta=" << delta); + EXPECT_EQ(rm.fwdsearch(position, delta), nv.fwdsearch(position, delta)); + EXPECT_EQ(rm.bwdsearch(position + 1, delta), + nv.bwdsearch(position + 1, delta)); + } + } + + const std::array, 4> ranges = { + std::pair{0, bits.size() - 1}, + std::pair{3, 97}, + std::pair{64, 255}, + std::pair{257, bits.size() - 1}, + }; + for (const auto& [left, right] : ranges) { + SCOPED_TRACE(::testing::Message() + << "range=[" << left << "," << right << "]"); + EXPECT_EQ(rm.range_min_query_pos(left, right), + nv.range_min_query_pos(left, right)); + EXPECT_EQ(rm.range_min_query_val(left, right), + nv.range_min_query_val(left, right)); + EXPECT_EQ(rm.range_max_query_pos(left, right), + nv.range_max_query_pos(left, right)); + EXPECT_EQ(rm.range_max_query_val(left, right), + nv.range_max_query_val(left, right)); + EXPECT_EQ(rm.mincount(left, right), nv.mincount(left, right)); + + const size_t count = nv.mincount(left, right); + for (size_t rank : {size_t{1}, count, count + 1}) { + if (rank == 0) { + continue; + } + EXPECT_EQ(rm.minselect(left, right, rank), + nv.minselect(left, right, rank)) + << "rank=" << rank; + } + } +} + TEST(RmMBTreeExperimental, FwdBwdSearchAcrossHighLevels) { std::mt19937_64 rng(kSeed); const size_t n = 512 * 32 * 8 + 4096; diff --git a/src/tests/unittests.cpp b/src/tests/unittests.cpp index 1c435d3..54b0dd4 100644 --- a/src/tests/unittests.cpp +++ b/src/tests/unittests.cpp @@ -4,6 +4,7 @@ #include #include +#include using pixie::BitVector; using pixie::BitVectorInterleaved; @@ -367,6 +368,80 @@ TEST(BitVectorTest, ExactShortSpanRankSelect) { EXPECT_EQ(bv.select0(9), bv.size()); } +TEST(BitVectorTest, OptionalSelectSupport) { + std::vector bits = {0b1100010110010110}; + + BitVector select0_only(bits, 16, BitVector::SelectSupport::kSelect0); + EXPECT_FALSE(select0_only.supports_select1()); + EXPECT_TRUE(select0_only.supports_select0()); + EXPECT_EQ(select0_only.rank(16), 8); + EXPECT_EQ(select0_only.rank0(16), 8); + EXPECT_EQ(select0_only.select(1), select0_only.size()); + EXPECT_EQ(select0_only.select0(1), 0); + EXPECT_EQ(select0_only.select0(8), 13); + + BitVector hinted_both(bits, 16, BitVector::SelectSupport::kBoth, 8); + EXPECT_TRUE(hinted_both.supports_select1()); + EXPECT_TRUE(hinted_both.supports_select0()); + EXPECT_EQ(hinted_both.select(1), 1); + EXPECT_EQ(hinted_both.select(8), 15); + EXPECT_EQ(hinted_both.select0(1), 0); + EXPECT_EQ(hinted_both.select0(8), 13); + EXPECT_THROW((BitVector(bits, 16, BitVector::SelectSupport::kBoth, 17)), + std::invalid_argument); + + BitVector select1_only(bits, 16, BitVector::SelectSupport::kSelect1); + EXPECT_TRUE(select1_only.supports_select1()); + EXPECT_FALSE(select1_only.supports_select0()); + EXPECT_EQ(select1_only.rank(16), 8); + EXPECT_EQ(select1_only.rank0(16), 8); + EXPECT_EQ(select1_only.select(1), 1); + EXPECT_EQ(select1_only.select(8), 15); + EXPECT_EQ(select1_only.select0(1), select1_only.size()); + + BitVector rank_only(bits, 16, BitVector::SelectSupport::kNone); + EXPECT_FALSE(rank_only.supports_select1()); + EXPECT_FALSE(rank_only.supports_select0()); + EXPECT_EQ(rank_only.rank(16), 8); + EXPECT_EQ(rank_only.rank0(16), 8); + EXPECT_EQ(rank_only.select(0), 0); + EXPECT_EQ(rank_only.select0(0), 0); + EXPECT_EQ(rank_only.select(1), rank_only.size()); + EXPECT_EQ(rank_only.select0(1), rank_only.size()); + + std::vector zero_words(512, 0); + BitVector large_select0_only(zero_words, zero_words.size() * 64, + BitVector::SelectSupport::kSelect0); + EXPECT_EQ(large_select0_only.select(1), large_select0_only.size()); + EXPECT_EQ(large_select0_only.select0(1), 0); + EXPECT_EQ(large_select0_only.select0(16384), 16383); + EXPECT_EQ(large_select0_only.select0(16385), 16384); + EXPECT_EQ(large_select0_only.select0(32768), 32767); + + BitVector large_hinted_select0(zero_words, zero_words.size() * 64, + BitVector::SelectSupport::kSelect0, 0); + EXPECT_EQ(large_hinted_select0.select(1), large_hinted_select0.size()); + EXPECT_EQ(large_hinted_select0.select0(1), 0); + EXPECT_EQ(large_hinted_select0.select0(32768), 32767); + + std::vector one_words(512, ~uint64_t{0}); + BitVector large_select1_only(one_words, one_words.size() * 64, + BitVector::SelectSupport::kSelect1); + EXPECT_EQ(large_select1_only.select0(1), large_select1_only.size()); + EXPECT_EQ(large_select1_only.select(1), 0); + EXPECT_EQ(large_select1_only.select(16384), 16383); + EXPECT_EQ(large_select1_only.select(16385), 16384); + EXPECT_EQ(large_select1_only.select(32768), 32767); +} + +TEST(BitVectorTest, ThrowsWhenOneCountHintUnderestimatesSamples) { + std::vector one_words(512, ~uint64_t{0}); + + EXPECT_THROW((BitVector(one_words, one_words.size() * 64, + BitVector::SelectSupport::kSelect1, 0)), + std::invalid_argument); +} + TEST(BitVectorTest, ShortSpanUsesScalarRankSelectFallbacks) { std::vector one_in_second_word = {0, 1}; BitVector ones(std::span(one_in_second_word), 65); @@ -384,6 +459,16 @@ TEST(BitVectorTest, ShortSpanUsesScalarRankSelectFallbacks) { EXPECT_EQ(zeros.select0(1), 64u); } +TEST(BitVectorTest, ShortSpanScalarRankUsesPartialWordTail) { + std::vector bits = {~uint64_t{0}, 0, 0b101}; + BitVector bv(std::span(bits), 130); + + EXPECT_EQ(bv.rank(128), 64u); + EXPECT_EQ(bv.rank(129), 65u); + EXPECT_EQ(bv.rank(130), 65u); + EXPECT_EQ(bv.rank0(129), 64u); +} + TEST(BitVectorTest, IgnoresDirtyPaddingAndTrailingWords) { std::vector bits = {~uint64_t{0}, ~uint64_t{0}}; BitVector bv(bits, 3); @@ -399,6 +484,24 @@ TEST(BitVectorTest, IgnoresDirtyPaddingAndTrailingWords) { EXPECT_EQ(bv.select0(1), bv.size()); } +TEST(BitVectorTest, SelectZeroSkewedDistributionFallback) { + constexpr size_t kWordsPerBasicBlock = 8; + constexpr size_t kBasicBlocksPerSuperBlock = 128; + constexpr size_t kZeroBasicBlocks = 40; + + std::vector bits(kWordsPerBasicBlock * kBasicBlocksPerSuperBlock, + ~uint64_t{0}); + std::fill(bits.begin(), bits.begin() + kZeroBasicBlocks * kWordsPerBasicBlock, + 0); + + BitVector bv(std::span(bits), bits.size() * 64); + EXPECT_EQ(bv.rank0(bv.size()), kZeroBasicBlocks * 512u); + EXPECT_EQ(bv.select0(1), 0u); + EXPECT_EQ(bv.select0(10000), 9999u); + EXPECT_EQ(bv.select0(kZeroBasicBlocks * 512u), kZeroBasicBlocks * 512u - 1); + EXPECT_EQ(bv.select0(kZeroBasicBlocks * 512u + 1), bv.size()); +} + TEST(BitVectorTest, MainRankZeroTest) { std::mt19937_64 rng(42); std::vector bits(65536 * 32);