diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..c3c4e3e --- /dev/null +++ b/.clang-format @@ -0,0 +1,78 @@ +# Defines the Chromium style for automatic reformatting. +# http://clang.llvm.org/docs/ClangFormatStyleOptions.html +BasedOnStyle: Chromium +# This defaults to 'Auto'. Explicitly set it for a while, so that +# 'vector >' in existing files gets formatted to +# 'vector>'. ('Auto' means that clang-format will only use +# 'int>>' if the file already contains at least one such instance.) +Standard: Cpp11 + +# TODO(crbug.com/1392808): Remove when InsertBraces has been upstreamed into +# the Chromium style (is implied by BasedOnStyle: Chromium). +InsertBraces: true +InsertNewlineAtEOF: true + +# Sort #includes by following +# https://google.github.io/styleguide/cppguide.html#Names_and_Order_of_Includes +# +# ref: https://clang.llvm.org/docs/ClangFormatStyleOptions.html#includeblocks +IncludeBlocks: Regroup +# ref: https://clang.llvm.org/docs/ClangFormatStyleOptions.html#includecategories +IncludeCategories: + # The win32 api has all sorts of implicit include order dependencies :-/ + # Give a few headers special priorities that make sure they appear before + # all other headers. + # Sync this with SerializeIncludes in tools/add_header.py. + # TODO(crbug.com/329138753): remove include sorting from tools/add_header.py + # after confirming clang-format sort works well. + # LINT.IfChange(winheader) + - Regex: '^' # This has to be before initguid.h. + Priority: 1 + - Regex: '^<(atlbase|initguid|mmdeviceapi|ocidl|ole2|shobjidl|tchar|unknwn|windows|winsock2|winternl|ws2tcpip)\.h>' + Priority: 2 + # LINT.ThenChange(/tools/add_header.py:winheader) + # UIAutomation*.h needs to be after base/win/atl.h. + # Note the low priority number. + - Regex: '^' + Priority: 6 + # Other C system headers. + - Regex: '^<.*\.h>' + Priority: 3 + # C++ standard library headers. + - Regex: '^<.*>' + Priority: 4 + # windows_h_disallowed.h should appear last. Note the low priority number. + - Regex: '"(.*/)?windows_h_disallowed\.h"' + Priority: 7 + # Other libraries. + - Regex: '.*' + Priority: 5 +# ref: https://clang.llvm.org/docs/ClangFormatStyleOptions.html#includeismainregex +IncludeIsMainRegex: "\ +(_(32|64|android|apple|chromeos|freebsd|fuchsia|fuzzer|ios|linux|mac|openbsd|posix|stubs?|win))?\ +(_(unit|browser|perf)?tests?)?$" + +# Make sure code like: +# IPC_BEGIN_MESSAGE_MAP() +# IPC_MESSAGE_HANDLER(WidgetHostViewHost_Update, OnUpdate) +# IPC_END_MESSAGE_MAP() +# gets correctly indented. +MacroBlockBegin: "^\ +BEGIN_MSG_MAP|\ +BEGIN_MSG_MAP_EX|\ +BEGIN_SAFE_MSG_MAP_EX|\ +CR_BEGIN_MSG_MAP_EX|\ +IPC_BEGIN_MESSAGE_MAP|\ +IPC_BEGIN_MESSAGE_MAP_WITH_PARAM|\ +IPC_PROTOBUF_MESSAGE_TRAITS_BEGIN|\ +IPC_STRUCT_TRAITS_BEGIN|\ +POLPARAMS_BEGIN|\ +PPAPI_BEGIN_MESSAGE_MAP$" +MacroBlockEnd: "^\ +CR_END_MSG_MAP|\ +END_MSG_MAP|\ +IPC_END_MESSAGE_MAP|\ +IPC_PROTOBUF_MESSAGE_TRAITS_END|\ +IPC_STRUCT_TRAITS_END|\ +POLPARAMS_END|\ +PPAPI_END_MESSAGE_MAP$" diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..4444654 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,28 @@ +name: Clang Format Lint + +on: + pull_request: + push: + branches: [main] + +jobs: + clang-format: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install clang-format + run: sudo apt-get update && sudo apt-get install -y clang-format + + - name: Run clang-format check + run: | + # Find all C/C++ files and check formatting + FILES=$(find . -regex '.*\.\(cpp\|hpp\|cc\|c\|h\)') + echo "Checking formatting on:" + echo "$FILES" + clang-format --version + + # Run clang-format in "diff" mode to see issues + clang-format --dry-run --Werror $FILES diff --git a/.gitignore b/.gitignore index 83de66c..b3ccf80 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ build/ .vscode/ Testing/ +plans/* diff --git a/CMakeLists.txt b/CMakeLists.txt index 634a22d..b520326 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,9 +2,7 @@ cmake_minimum_required(VERSION 3.12) project(pixie) set(MARCH "native" CACHE STRING "march compilier flag") -set(CMAKE_CXX_FLAGS "-march=${MARCH} -O3 -DNDEBUG") -set(BENCHMARK_ENABLE_GTEST_TESTS OFF) - +set(CMAKE_CXX_FLAGS "-march=${MARCH}") include(FetchContent) FetchContent_Declare( googletest @@ -18,7 +16,9 @@ FetchContent_Declare( GIT_REPOSITORY https://github.com/google/benchmark.git GIT_TAG v1.9.4 # Replace with the desired version/tag ) +set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable Google Benchmark tests") FetchContent_MakeAvailable(googlebenchmark) + add_executable(benchmarks src/benchmarks.cpp) target_include_directories(benchmarks diff --git a/include/bits.h b/include/bits.h index 2d46e5f..4f0bb25 100644 --- a/include/bits.h +++ b/include/bits.h @@ -1,26 +1,53 @@ #pragma once +#include + #include +#include #include -#include #include - -#if defined(__AVX512VPOPCNTDQ__) && defined(__AVX512F__) && defined(__AVX512BW__) -#define AVX512SUPP +#if defined(__AVX512VPOPCNTDQ__) && defined(__AVX512F__) && \ + defined(__AVX512BW__) +#define PIXIE_AVX512_SUPPORT #endif - #ifdef __AVX2__ -#define AVX2SUPP +#define PIXIE_AVX2_SUPPORT +// Lookup table for 4-bit popcount +// This table maps each 4-bit value (0-15) to its population count +// clang-format off +const __m256i lookup_popcount_4 = _mm256_setr_epi8( + 0, 1, 1, 2, // 0000, 0001, 0010, 0011 + 1, 2, 2, 3, // 0100, 0101, 0110, 0111 + 1, 2, 2, 3, // 1000, 1001, 1010, 1011 + 2, 3, 3, 4, // 1100, 1101, 1110, 1111 + + // Same table repeated for high 128 bits + 0, 1, 1, 2, // 0000, 0001, 0010, 0011 + 1, 2, 2, 3, // 0100, 0101, 0110, 0111 + 1, 2, 2, 3, // 1000, 1001, 1010, 1011 + 2, 3, 3, 4 // 1100, 1101, 1110, 1111 +); + +const __m256i mask_first_half = _mm256_setr_epi8( + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0 +); + +// clang-format on #endif - inline uint64_t first_bits_mask(size_t num) { - return num >= 64 ? UINT64_MAX : ((1llu << num) - 1); + return num >= 64 ? UINT64_MAX : ((1llu << num) - 1); } - /** * @brief Number of 1 bits in positions 0 .. count - 1 * @details @@ -41,8 +68,8 @@ inline uint64_t first_bits_mask(size_t num) { * The rest is standard, i.e. popcount_epi64 to perform popcount on * 64 bits and then reduce_add to sum the result. */ -uint64_t rank_512(const uint64_t *x, uint64_t count) { -#ifdef AVX512SUPP +uint64_t rank_512(const uint64_t* x, uint64_t count) { +#ifdef PIXIE_AVX512_SUPPORT __m512i a = _mm512_maskz_set1_epi64((1ull << ((count >> 6))) - 1, std::numeric_limits::max()); @@ -59,30 +86,27 @@ uint64_t rank_512(const uint64_t *x, uint64_t count) { uint64_t last_uint = count >> 6; - uint64_t pop_val = 0; + uint64_t pop_val = 0; - for (int i = 0; i < last_uint; i++) { - pop_val += std::popcount(x[i]); - } + for (int i = 0; i < last_uint; i++) { + pop_val += std::popcount(x[i]); + } - uint64_t final = x[last_uint] & first_bits_mask(count & 63); + uint64_t final = x[last_uint] & first_bits_mask(count & 63); - pop_val += std::popcount(final); - return pop_val; + pop_val += std::popcount(final); + return pop_val; #endif } - /** * @brief Return position of @p rank 1 bit in @p x */ uint64_t select_64(uint64_t x, uint64_t rank) { - auto a = _pdep_u64(1ull << rank, x); - return _tzcnt_u64(a); + return _tzcnt_u64(_pdep_u64(1ull << rank, x)); } - /** * @brief Return position of @p rank 1 bit in @p x * @details Selecting within 64-bit word can be done @@ -100,10 +124,10 @@ uint64_t select_64(uint64_t x, uint64_t rank) { * It can also be used as an alternative for linear search but i don't * see a proper SIMD algorithm to make it faster. */ -uint64_t select_512(const uint64_t *x, uint64_t rank) { -#ifdef AVX512SUPP +uint64_t select_512(const uint64_t* x, uint64_t rank) { +#ifdef PIXIE_AVX512_SUPPORT - __m512i res = _mm512_loadu_epi64(x); + __m512i res = _mm512_loadu_epi64(x); std::array counts; _mm512_storeu_epi64(counts.data(), _mm512_popcnt_epi64(res)); @@ -116,23 +140,22 @@ uint64_t select_512(const uint64_t *x, uint64_t rank) { #else size_t i = 0; - int popcount = std::popcount(x[0]); - while (i < 7 && popcount <= rank) { - rank -= popcount; - popcount = std::popcount(x[++i]); - } - return i * 64 + select_64(x[i], rank); + int popcount = std::popcount(x[0]); + while (i < 7 && popcount <= rank) { + rank -= popcount; + popcount = std::popcount(x[++i]); + } + return i * 64 + select_64(x[i], rank); #endif } - /** - * @brief Compare 4 64-bit numbers of @p x with @p y and + * @brief Compare 4 64-bit numbers of @p x with @p y and * return the length of the prefix where @p y is less then @p x */ -uint16_t cmpl_pref_len_256(const uint64_t* x, uint64_t y) { -#ifdef AVX512SUPP +uint16_t lower_bound_4x64(const uint64_t* x, uint64_t y) { +#ifdef PIXIE_AVX512_SUPPORT auto y_4 = _mm256_set1_epi64x(y); auto reg_256 = _mm256_loadu_epi64(x); @@ -141,7 +164,7 @@ uint16_t cmpl_pref_len_256(const uint64_t* x, uint64_t y) { return _tzcnt_u16(cmp); #else -#ifdef AVX2SUPP +#ifdef PIXIE_AVX2_SUPPORT auto y_4 = _mm256_set1_epi64x(y); __m256i reg_256 = _mm256_loadu_si256(reinterpret_cast(x)); @@ -149,28 +172,30 @@ uint16_t cmpl_pref_len_256(const uint64_t* x, uint64_t y) { const __m256i offset = _mm256_set1_epi64x(0x8000000000000000ULL); __m256i x_offset = _mm256_xor_si256(reg_256, offset); __m256i y_offset = _mm256_xor_si256(y_4, offset); - auto mask = _mm256_movemask_epi8(_mm256_cmpgt_epi64(x_offset, _mm256_sub_epi64(y_offset, _mm256_set1_epi64x(1)))); + auto mask = _mm256_movemask_epi8(_mm256_cmpgt_epi64( + x_offset, _mm256_sub_epi64(y_offset, _mm256_set1_epi64x(1)))); return _tzcnt_u32(mask) >> 3; #else - for (uint16_t i = 0; i < 4; ++i) - if (x[i] >= y) + for (uint16_t i = 0; i < 4; ++i) { + if (x[i] >= y) { return i; + } + } return 4; #endif #endif } - /** - * @brief Compare 8 64-bit numbers of @p x with @p y and + * @brief Compare 8 64-bit numbers of @p x with @p y and * return the length of the prefix where @p y is less then @p x */ -uint16_t cmpl_pref_len_512(const uint64_t* x, uint64_t y) { -#ifdef AVX512SUPP +uint16_t lower_bound_8x64(const uint64_t* x, uint64_t y) { +#ifdef PIXIE_AVX512_SUPPORT auto y_8 = _mm512_set1_epi64(y); auto reg_512 = _mm512_loadu_epi64(x); @@ -179,33 +204,35 @@ uint16_t cmpl_pref_len_512(const uint64_t* x, uint64_t y) { return _tzcnt_u16(cmp); #else -#ifdef AVX2SUPP +#ifdef PIXIE_AVX2_SUPPORT - uint16_t len = cmpl_pref_len_256(x, y); + uint16_t len = lower_bound_4x64(x, y); - if (len < 4) + if (len < 4) { return len; + } - return len + cmpl_pref_len_256(x + 4, y); + return len + lower_bound_4x64(x + 4, y); #else - for (uint16_t i = 0; i < 8; ++i) - if (x[i] >= y) + for (uint16_t i = 0; i < 8; ++i) { + if (x[i] >= y) { return i; + } + } return 8; #endif #endif } - /** - * @brief Compare 32 16-bit numbers of @p x with @p y and + * @brief Compare 32 16-bit numbers of @p x with @p y and * return the count of numbers where @p x is less then @p y */ -uint16_t cmpl_count_512(const uint16_t* x, uint64_t y) { -#ifdef AVX512SUPP +uint16_t lower_bound_32x16(const uint16_t* x, uint64_t y) { +#ifdef PIXIE_AVX512_SUPPORT auto y_32 = _mm512_set1_epi16(y); auto reg_512 = _mm512_loadu_epi16(x); @@ -213,7 +240,7 @@ uint16_t cmpl_count_512(const uint16_t* x, uint64_t y) { return std::popcount(cmp); #else -#ifdef AVX2SUPP +#ifdef PIXIE_AVX2_SUPPORT auto y_16 = _mm256_set1_epi16(y); __m256i reg_256 = _mm256_loadu_si256(reinterpret_cast(x)); @@ -235,11 +262,146 @@ uint16_t cmpl_count_512(const uint16_t* x, uint64_t y) { #else uint16_t cnt = 0; - for (uint16_t i = 0; i < 32; ++i) - if (x[i] < y) + for (uint16_t i = 0; i < 32; ++i) { + if (x[i] < y) { cnt++; + } + } return cnt; #endif #endif -} \ No newline at end of file +} + +/** + * @brief Calculates 64 popcounts of 4-bits integers and stores as 64 4-bits + * integers (packed into 32 bytes) + * + * This function counts the number of set bits in each 4-bit value + * using an efficient shuffle-based approach with lookup tables. + * + * @param x Pointer to 32 bytes containing 64 4-bit integers + * @param result Pointer to store the 64 resulting 4-bit popcount values (packed + * in 32 bytes) + */ +void popcount_64x4(const uint8_t* x, uint8_t* result) { +#ifdef PIXIE_AVX512_SUPPORT + __m256i data = _mm256_loadu_si256((__m256i const*)x); + + // Masks for extracting the lower and upper nibbles + const __m256i low_bits_mask = _mm256_set1_epi8(0x0F); + + // Count bits in the lower half + __m256i low_bits = _mm256_and_si256(data, low_bits_mask); + __m256i low_count = _mm256_shuffle_epi8(lookup_popcount_4, low_bits); + + // Count bits in the upper half + __m256i high_bits = _mm256_srli_epi16(data, 4); + high_bits = _mm256_and_si256(high_bits, low_bits_mask); + __m256i high_count = _mm256_shuffle_epi8(lookup_popcount_4, high_bits); + + // Pack the results into a single output vector + __m256i result_vec = + _mm256_or_si256(low_count, _mm256_slli_epi16(high_count, 4)); + _mm256_storeu_epi8(result, result_vec); +#else + // Fallback implementation for non-AVX2 platforms + for (size_t i = 0; i < 32; i++) { + // Count bits in the lower half + uint8_t a = x[i] & 0x0F; + uint8_t low_count = std::popcount(a); + // Count bits in the upper half + a = (x[i] >> 4) & 0x0F; + uint8_t high_count = std::popcount(a); + + // Pack the counts into the output byte + result[i] = low_count | (high_count << 4); + } +#endif +} + +/** + * @brief Calculates 64 popcounts of 4-bits integers and stores as 64 4-bits + * integers (packed into 32 bytes) + * + * This function counts the number of set bits in each 4-bit value + * using an efficient shuffle-based approach with lookup tables. + * + * @param x Pointer to 32 bytes containing 64 4-bit integers + * @param result Pointer to store the 64 resulting 4-bit popcount values + * (packed in 32 bytes) + */ +void popcount_32x8(const uint8_t* x, uint8_t* result) { +#ifdef PIXIE_AVX512_SUPPORT + // Load 64 4-bit integers (256 bits total) + __m256i data = _mm256_loadu_si256((__m256i const*)x); + auto popcount_8 = _mm256_popcnt_epi8(data); + _mm256_storeu_si256((__m256i*)result, popcount_8); +#else +#ifdef PIXIE_AVX2_SUPPORT + // Load 64 4-bit integers (256 bits total) + __m256i data = _mm256_loadu_si256((__m256i const*)x); + + // Masks for extracting the lower and upper nibbles + const __m256i low_bits_mask = _mm256_set1_epi8(0x0F); + + // Count bits in lower half + __m256i low_bits = _mm256_and_si256(data, low_bits_mask); + __m256i low_count = _mm256_shuffle_epi8(lookup_popcount_4, low_bits); + + // Count bits upper half + __m256i high_bits = _mm256_srli_epi16(data, 4); + high_bits = _mm256_and_si256(high_bits, low_bits_mask); + __m256i high_count = _mm256_shuffle_epi8(lookup_popcount_4, high_bits); + + __m256i result_vec = _mm256_add_epi8(low_count, high_count); + _mm256_storeu_si256((__m256i*)result, result_vec); +#else + // Fallback implementation for non-AVX2 platforms + for (size_t i = 0; i < 32; i++) { + result[i] = std::popcount(x[i]); + } +#endif +#endif +} + +/** + * @brief Calculates 32 bit ranks of every 8th bit, result is stored as 32 + * 8-bit integers. + * + * @param x Pointer to 32 input 8-bit integers + * @param result Pointer to store the resulting 32 8-bit integers + */ +void rank_32x8(const uint8_t* x, uint8_t* result) { +#ifdef PIXIE_AVX512_SUPPORT + // Step 1: Calculate popcount of each byte + popcount_32x8(x, result); + __m256i prefix_sums = _mm256_loadu_si256((__m256i const*)result); + const __m256i zero = _mm256_setzero_si256(); + + prefix_sums = _mm256_add_epi8(prefix_sums, + _mm256_alignr_epi8(prefix_sums, zero, 16 - 1)); + prefix_sums = _mm256_add_epi8(prefix_sums, + _mm256_alignr_epi8(prefix_sums, zero, 16 - 2)); + prefix_sums = _mm256_add_epi8(prefix_sums, + _mm256_alignr_epi8(prefix_sums, zero, 16 - 4)); + prefix_sums = _mm256_add_epi8(prefix_sums, + _mm256_alignr_epi8(prefix_sums, zero, 16 - 8)); + + // At this point we have prefix sums for two halfs, the last step is to + // extract 16-th value and add it to the whole second half + __m128i low_lane = _mm256_extracti128_si256(prefix_sums, 0); + __m128i high_lane = _mm256_extracti128_si256(prefix_sums, 1); + auto last_val_low = _mm_extract_epi8(low_lane, 15); + __m128i add_to_high = _mm_set1_epi8(last_val_low); + high_lane = _mm_add_epi8(high_lane, add_to_high); + prefix_sums = _mm256_set_m128i(high_lane, low_lane); + _mm256_storeu_epi8(result, prefix_sums); +#else + // Scalar fallback implementation + result[0] = std::popcount(x[0]); + for (size_t i = 1; i < 32; ++i) { + result[i] = std::popcount(x[i]) + result[i - 1]; + } +#endif +} diff --git a/include/bitvector.h b/include/bitvector.h index 79aa3ca..8abf686 100644 --- a/include/bitvector.h +++ b/include/bitvector.h @@ -62,7 +62,7 @@ namespace pixie { * involved (like {1} does). */ class BitVector { -private: + private: constexpr static size_t kWordSize = 64; constexpr static size_t kSuperBlockRankIntSize = 64; constexpr static size_t kBasicBlockRankIntSize = 16; @@ -141,21 +141,22 @@ class BitVector { uint64_t left = select_samples[rank / kSelectSampleFrequency]; while (left + 7 < super_block_rank.size()) { - auto len = cmpl_pref_len_512(&super_block_rank[left], rank); + auto len = lower_bound_8x64(&super_block_rank[left], rank); if (len < 8) { return left + len - 1; } left += 8; } if (left + 3 < super_block_rank.size()) { - auto len = cmpl_pref_len_256(&super_block_rank[left], rank); + auto len = lower_bound_4x64(&super_block_rank[left], rank); if (len < 4) { return left + len - 1; } left += 4; } - while (left < super_block_rank.size() && super_block_rank[left] < rank) + while (left < super_block_rank.size() && super_block_rank[left] < rank) { left++; + } return left - 1; } @@ -169,7 +170,8 @@ class BitVector { auto pos = 0; for (size_t i = 0; i < 4; ++i) { - auto count = cmpl_count_512(&basic_block_rank[128 * s_block + 32 * i], local_rank); + auto count = lower_bound_32x16(&basic_block_rank[128 * s_block + 32 * i], + local_rank); if (count < 32) { return kBlocksPerSuperBlock * s_block + pos + count - 1; } @@ -195,7 +197,8 @@ class BitVector { pos = pos + 16 < 32 ? 0 : (pos - 16); pos = pos > 96 ? 96 : pos; while (pos < 96) { - auto count = cmpl_count_512(&basic_block_rank[128 * s_block + pos], local_rank); + auto count = + lower_bound_32x16(&basic_block_rank[128 * s_block + pos], local_rank); if (count == 0) { return find_basicblock(local_rank, s_block); } @@ -205,14 +208,15 @@ class BitVector { pos += 32; } pos = 96; - auto count = cmpl_count_512(&basic_block_rank[128 * s_block + pos], local_rank); + auto count = + lower_bound_32x16(&basic_block_rank[128 * s_block + pos], local_rank); if (count == 0) { return find_basicblock(local_rank, s_block); } return kBlocksPerSuperBlock * s_block + pos + count - 1; } -public: + public: /** * Constructor from an external array of uint64_t. */ @@ -311,7 +315,7 @@ class BitVector { * */ class BitVectorInterleaved { -private: + private: constexpr static size_t kWordSize = 64; constexpr static size_t kSuperBlockRankIntSize = 64; constexpr static size_t kBasicBlockRankIntSize = 16; @@ -339,7 +343,7 @@ class BitVectorInterleaved { size_t offset_bits_ = 0; std::span bits_; - public: + public: BitReader(std::span bits) : bits_(bits) {} uint64_t ReadBits64(size_t num_bits) { if (num_bits > 64) { @@ -362,7 +366,7 @@ class BitVectorInterleaved { } }; -public: + public: /** * Constructor from an external array of uint64_t. */ @@ -418,14 +422,15 @@ class BitVectorInterleaved { static_cast(basic_block_sum) << 48; for (size_t j = 0; j < 7 && kWordSize * (i + j) < num_bits; ++j) { - bits_interleaved[i * (kWordsPerBlock) + j] = bit_reader.ReadBits64( - std::min(64ull, num_bits - i * kBasicBlockSize + j * kWordSize)); + bits_interleaved[i * (kWordsPerBlock) + j] = + bit_reader.ReadBits64(std::min( + 64ull, num_bits - i * kBasicBlockSize + j * kWordSize)); basic_block_sum += std::popcount(bits_interleaved[i * (kWordsPerBlock) + j]); } if ((i + 7) * kWordSize < num_bits) { - auto v = bit_reader.ReadBits64( - std::min(48ull, num_bits - (i * kBasicBlockSize + 7 * kWordSize))); + auto v = bit_reader.ReadBits64(std::min( + 48ull, num_bits - (i * kBasicBlockSize + 7 * kWordSize))); bits_interleaved[i * (kWordsPerBlock) + 7] ^= v; basic_block_sum += std::popcount(v); } @@ -471,4 +476,4 @@ class BitVectorInterleaved { } }; -} // namespace pixie +} // namespace pixie diff --git a/src/benchmarks.cpp b/src/benchmarks.cpp index 8aa9cc9..25daaa8 100644 --- a/src/benchmarks.cpp +++ b/src/benchmarks.cpp @@ -1,16 +1,15 @@ #include - #include + #include #include - -static void BM_RankNonInterleaved(benchmark::State &state) { +static void BM_RankNonInterleaved(benchmark::State& state) { size_t n = state.range(0); std::mt19937_64 rng(42); std::vector bits(1 + n / 64); - for (auto &x : bits) { + for (auto& x : bits) { x = rng(); } pixie::BitVector bv(bits, n); @@ -21,12 +20,12 @@ static void BM_RankNonInterleaved(benchmark::State &state) { } } -static void BM_RankInterleaved(benchmark::State &state) { +static void BM_RankInterleaved(benchmark::State& state) { size_t n = state.range(0); std::mt19937_64 rng(42); std::vector bits(1 + n / 64); - for (auto &x : bits) { + for (auto& x : bits) { x = rng(); } pixie::BitVectorInterleaved bv(bits, n); @@ -37,12 +36,12 @@ static void BM_RankInterleaved(benchmark::State &state) { } } -static void BM_SelectNonInterleaved(benchmark::State &state) { +static void BM_SelectNonInterleaved(benchmark::State& state) { size_t n = state.range(0); std::mt19937_64 rng(42); std::vector bits(1 + n / 64); - for (auto &x : bits) { + for (auto& x : bits) { x = rng(); } pixie::BitVector bv(bits, n); diff --git a/src/unittests.cpp b/src/unittests.cpp index ce423f4..bc0e038 100644 --- a/src/unittests.cpp +++ b/src/unittests.cpp @@ -1,9 +1,10 @@ +#include + #include #include #include "bits.h" #include "bitvector.h" -#include using pixie::BitVector; using pixie::BitVectorInterleaved; @@ -98,7 +99,7 @@ TEST(Select512, Random) { std::mt19937_64 rng(42); for (size_t t = 0; t < 1000; ++t) { - for (auto &x : a) { + for (auto& x : a) { x = rng(); } size_t rank = 0; @@ -123,7 +124,7 @@ TEST(Select512, RankCompativility) { std::mt19937_64 rng(42); for (size_t t = 0; t < 1000; ++t) { - for (auto &x : a) { + for (auto& x : a) { x = rng(); } size_t rank = 0; @@ -181,12 +182,12 @@ TEST(BitVectorTest, RankBasic) { std::vector bits = {0b10110}; BitVector bv(bits, 5); - EXPECT_EQ(bv.rank(0), 0); // No bits - EXPECT_EQ(bv.rank(1), 0); // 0 - EXPECT_EQ(bv.rank(2), 1); // 10 - EXPECT_EQ(bv.rank(3), 2); // 110 - EXPECT_EQ(bv.rank(4), 2); // 0110 - EXPECT_EQ(bv.rank(5), 3); // 10110 + EXPECT_EQ(bv.rank(0), 0); // No bits + EXPECT_EQ(bv.rank(1), 0); // 0 + EXPECT_EQ(bv.rank(2), 1); // 10 + EXPECT_EQ(bv.rank(3), 2); // 110 + EXPECT_EQ(bv.rank(4), 2); // 0110 + EXPECT_EQ(bv.rank(5), 3); // 10110 } TEST(BitVectorTest, RankWithZeros) { @@ -246,8 +247,6 @@ TEST(BitVectorTest, MainSelectTest) { } } - - TEST(BitVectorInterleavedTest, AtTest) { std::mt19937_64 rng(42); /* @@ -283,214 +282,106 @@ TEST(BitVectorInterleavedTest, MainTest) { } } -TEST(CMPLPREFLEN256, Random) { +TEST(LowerBound4x64, Random) { std::vector x(4); std::mt19937_64 rng(42); for (size_t i = 0; i < 1000; i++) { uint64_t y = rng(); uint16_t cnt = 0; bool fl = 1; - for (size_t j = 0; j < 4;j++) { + for (size_t j = 0; j < 4; j++) { x[j] = rng(); fl &= x[j] < y; cnt += fl; } - if (cnt < 4) - EXPECT_EQ(cmpl_pref_len_256(x.data(), y), cnt); - else - EXPECT_GE(cmpl_pref_len_256(x.data(), y), cnt); + if (cnt < 4) { + ASSERT_EQ(lower_bound_4x64(x.data(), y), cnt); + } else { + ASSERT_GE(lower_bound_4x64(x.data(), y), cnt); + } } } -TEST(CMPLPREFLEN512, Random) { +TEST(LowerBound8x64, Random) { std::vector x(8); std::mt19937_64 rng(42); for (size_t i = 0; i < 1000; i++) { uint64_t y = rng(); uint16_t cnt = 0; bool fl = 1; - for (size_t j = 0; j < 8;j++) { + for (size_t j = 0; j < 8; j++) { x[j] = rng(); fl &= x[j] < y; cnt += fl; } - if (cnt < 8) - EXPECT_EQ(cmpl_pref_len_512(x.data(), y), cnt); - else - EXPECT_GE(cmpl_pref_len_512(x.data(), y), cnt); + if (cnt < 8) { + ASSERT_EQ(lower_bound_8x64(x.data(), y), cnt); + } else { + ASSERT_GE(lower_bound_8x64(x.data(), y), cnt); + } } } -TEST(CMPLCOUNT512, Random) { +TEST(LowerBound32x16, Random) { std::vector x(32); std::mt19937 rng(42); for (size_t i = 0; i < 1000; i++) { uint16_t y = rng(); uint16_t cnt = 0; - for (size_t j = 0; j < 32;j++) { + for (size_t j = 0; j < 32; j++) { x[j] = rng(); cnt += x[j] < y; } - EXPECT_EQ(cmpl_count_512(x.data(), y), cnt); + ASSERT_EQ(lower_bound_32x16(x.data(), y), cnt); + } +} + +TEST(Popcount64x4, Random) { + std::vector x(32); + std::vector y(32); + std::mt19937 rng(42); + for (size_t i = 0; i < 1000; i++) { + for (size_t j = 0; j < 32; j++) { + x[j] = rng(); + } + popcount_64x4(x.data(), y.data()); + for (size_t j = 0; j < 32; ++j) { + uint8_t a = x[j] & 0x0F; + ASSERT_EQ(y[j] & 0x0F, std::popcount(a)); + a = x[j] >> 4; + ASSERT_EQ(y[j] >> 4, std::popcount(a)); + } } } -// TEST(BitVectorTest, SelectBasic) { -// std::vector bits = {0b10110}; -// BitVector bv(bits, 5); - -// EXPECT_EQ(bv.select(0), 0); // Position of the 1st 1-bit -// EXPECT_EQ(bv.select(1), 2); // Position of the 2nd 1-bit -// EXPECT_EQ(bv.select(2), 3); // Position of the 3rd 1-bit - -// EXPECT_THROW(bv.select(3), std::out_of_range); // Only 3 bits set -// } - -// TEST(BitVectorTest, SelectWithZeros) { -// std::vector bits = {0}; -// BitVector bv(bits, 5); - -// EXPECT_THROW(bv.select(0), std::out_of_range); // No bits set -// } - -// TEST(BitVectorTest, SelectWithAllOnes) { -// std::vector bits = {0b11111}; -// BitVector bv(bits, 5); - -// for (size_t i = 0; i < 5; i++) { -// EXPECT_EQ(bv.select(i), i); // i-th 1-bit is at position i -// } -// } - -// TEST(BitVectorTest, SelectMultipleWords) { -// // Create a bitvector with bits set at every third position -// std::vector bits(10, 0); // 10 words, 640 bits total -// size_t num_bits = 10 * 64; - -// for (size_t i = 0; i < num_bits; i++) { -// if (i % 3 == 0) { -// size_t word_idx = i / 64; -// size_t bit_offset = i % 64; -// bits[word_idx] |= (1ULL << bit_offset); -// } -// } - -// BitVector bv(bits, num_bits); - -// // Test select for positions where bits are set -// for (size_t i = 0; i < num_bits / 3; i++) { -// EXPECT_EQ(bv.select(i), i * 3); -// } -// } - -// TEST(BitVectorTest, CrossBlockBoundaries) { -// // Create a bitvector with specific bits set to test block boundaries -// std::vector bits(32, 0); // 32 words = 2048 bits -// size_t num_bits = 32 * 64; - -// // Set bits at basic block boundaries (multiples of 512 bits) -// for (size_t i = 0; i < num_bits; i += 512) { -// size_t word_idx = i / 64; -// size_t bit_offset = i % 64; -// bits[word_idx] |= (1ULL << bit_offset); -// } - -// // Set bits at the end of each basic block (511, 1023, etc.) -// for (size_t i = 511; i < num_bits; i += 512) { -// size_t word_idx = i / 64; -// size_t bit_offset = i % 64; -// bits[word_idx] |= (1ULL << bit_offset); -// } - -// BitVector bv(bits, num_bits); - -// // Test rank at basic block boundaries -// for (size_t i = 0; i <= 4; i++) { -// EXPECT_EQ(bv.rank(i * 512), i * 2); -// } - -// // Test select for these specific bits -// for (size_t i = 0; i < 8; i++) { -// if (i % 2 == 0) { -// EXPECT_EQ(bv.select(i), (i / 2) * 512); -// } else { -// EXPECT_EQ(bv.select(i), (i / 2) * 512 + 511); -// } -// } -// } - -// TEST(BitVectorTest, RandomPattern) { -// // Create a bitvector with a random pattern to test rank and select -// std::vector bits(16, 0); // 16 words = 1024 bits -// size_t num_bits = 1000; // Use 1000 bits (not the full 1024) - -// std::mt19937 rng(42); // Fixed seed for reproducibility - -// // Set random bits -// for (size_t i = 0; i < num_bits; i++) { -// if (rng() % 2 == 1) { // ~50% chance of setting each bit -// size_t word_idx = i / 64; -// size_t bit_offset = i % 64; -// bits[word_idx] |= (1ULL << bit_offset); -// } -// } - -// BitVector bv(bits, num_bits); - -// // Count the number of set bits up to each position -// std::vector expected_ranks(num_bits + 1, 0); -// for (size_t i = 0; i < num_bits; i++) { -// expected_ranks[i+1] = expected_ranks[i] + (bv[i] ? 1 : 0); -// } - -// // Test rank for all positions -// for (size_t i = 0; i <= num_bits; i++) { -// EXPECT_EQ(bv.rank(i), expected_ranks[i]); -// } - -// // Test select for all set bits -// size_t total_ones = expected_ranks[num_bits]; -// std::vector one_positions; -// for (size_t i = 0; i < num_bits; i++) { -// if (bv[i]) { -// one_positions.push_back(i); -// } -// } - -// for (size_t i = 0; i < total_ones; i++) { -// EXPECT_EQ(bv.select(i), one_positions[i]); -// } -// } - -// TEST(BitVectorTest, LargeBitVector) { -// // Test with a larger bitvector to ensure the two-level structure works -// correctly size_t num_words = 1024; // 1024 words = 65536 bits (matches -// kSuperBlockSize) std::vector bits(num_words, 0); size_t -// num_bits = num_words * 64; - -// // Set every 100th bit -// for (size_t i = 0; i < num_bits; i += 100) { -// size_t word_idx = i / 64; -// size_t bit_offset = i % 64; -// bits[word_idx] |= (1ULL << bit_offset); -// } - -// BitVector bv(bits, num_bits); - -// // Test rank at multiples of 100 -// for (size_t i = 0; i <= 650; i++) { -// EXPECT_EQ(bv.rank(i * 100), i); -// } - -// // Test select -// for (size_t i = 0; i < 650; i++) { -// EXPECT_EQ(bv.select(i), i * 100); -// } -// } - -// // Tests for our BitVector implementation -// TEST(BitVectorTest2, EmptyConstructor) { -// std::vector bits = {0b101010101}; -// BitVector bv(bits, 8); -// EXPECT_EQ(bv.size(), 8); -// } +TEST(Popcount32x8, Random) { + std::vector x(32); + std::vector y(32); + std::mt19937 rng(42); + for (size_t i = 0; i < 1000; i++) { + for (size_t j = 0; j < 32; j++) { + x[j] = rng(); + } + popcount_32x8(x.data(), y.data()); + for (size_t j = 0; j < 32; ++j) { + ASSERT_EQ(y[j], std::popcount(x[j])); + } + } +} + +TEST(RankParallel32x8, Random) { + std::vector x(32); + std::vector sum(32); + std::vector y(32); + std::mt19937 rng(42); + for (size_t i = 0; i < 1000; i++) { + x[0] = rng(); + sum[0] = std::popcount(x[0]); + for (size_t j = 1; j < 32; j++) { + x[j] = rng(); + sum[j] = sum[j - 1] + std::popcount(x[j]); + } + rank_32x8(x.data(), y.data()); + ASSERT_EQ(std::equal(y.begin(), y.end(), sum.begin()), true); + } +}