diff --git a/lbs_solver/incl/solver/index.hpp b/lbs_solver/incl/solver/index.hpp index 23f5f9c..b247385 100644 --- a/lbs_solver/incl/solver/index.hpp +++ b/lbs_solver/incl/solver/index.hpp @@ -3,6 +3,7 @@ #include +#include #include #include #include @@ -10,55 +11,141 @@ #include #include +#if __has_include() +#include +#endif + namespace lbs { +namespace dev { + [[nodiscard]] constexpr std::vector split( + std::span data, + char separator, + size_t separatorDistGuess = 3 + ); +} + +/** + * @return the end index of every word, the last one is data.size() + */ +[[nodiscard]] constexpr std::vector word_ends( + std::span data, + char separator, + size_t separatorDistGuess = 3 +); + +struct words_data { + std::vector words; + std::vector wordHashes; +}; + + +/** + * filters the words and builds hashes + * @param compressionIndex char -> index in [0, 11] + * @param csv words + * @param wordEnds word end indices + * @return filtered words & their hashes + */ +[[nodiscard]] constexpr words_data filter_and_hash( + compression_index_t const& compressionIndex, + std::string_view csv, + std::span wordEnds +); + + +} + +namespace lbs::dev { + +#if defined(__cpp_lib_simd) || defined(__glibcxx_simd) [[nodiscard]] constexpr std::vector split( std::span data, char separator, - size_t separator_dist_guess = 3 + size_t separatorDistGuess ) { - using mask_t = uint64_t; - std::vector indices{}; - indices.reserve(data.size() / separator_dist_guess); + static constexpr size_t BLOCK_SIZE = sizeof(uint64_t) * 8; + using block_t = std::simd::vec; + + + std::vector separatorIdxs{}; + separatorIdxs.reserve(data.size() / separatorDistGuess); - static constexpr size_t BLOCK_SIZE = sizeof(mask_t) * 8; + auto const blocks = data.size() / BLOCK_SIZE; - auto const chunkedEnd = (data.size() / BLOCK_SIZE) * BLOCK_SIZE; + auto const uncheckedEnd = BLOCK_SIZE * blocks; - static constexpr auto extract_indices = []( - std::vector& buffer, - mask_t mask, - size_t offset - ) { + auto extract_separator_idxs = [&separatorIdxs](size_t offset, uint64_t mask) { while(mask != 0) { - auto const idx = std::countr_zero(mask); - buffer.push_back(offset + idx); - mask &= mask - mask_t{1}; // Clear lowest set bit + separatorIdxs.push_back(offset + std::countr_zero(mask)); + mask &= mask - 1; } }; - // simd friendly processing - for(size_t blockBegin = 0; blockBegin < chunkedEnd; blockBegin += BLOCK_SIZE) { - mask_t blockMask = 0; - // simd separator into mask - for(size_t i = 0; i < BLOCK_SIZE; i++) - if(data[blockBegin + i] == separator) - blockMask |= mask_t{1} << i; - - extract_indices(indices, blockMask, blockBegin); + for(size_t i = 0; i < uncheckedEnd; i += block_t::size) { + auto simdBlock = std::simd::unchecked_load(data.subspan(i, block_t::size)); + extract_separator_idxs(i, (simdBlock == block_t{separator}).to_ullong()); } - //tail cleanup - for(size_t i = chunkedEnd; i < data.size(); i++) - if(data[i] == separator) - indices.push_back(i); + // bug in gcc, std::simd::partial_load fails to load byte 3 when exactly 4 bytes are remaining + std::array lastChars{}; + std::ranges::copy(data.subspan(uncheckedEnd), lastChars.begin()); + auto lastBlock = std::simd::unchecked_load(lastChars); + + extract_separator_idxs( + uncheckedEnd, + (lastBlock == block_t{separator}).to_ullong() + ); + - return indices; + return separatorIdxs; } +#else + +[[nodiscard]] constexpr std::vector split( + std::span data, + char separator, + size_t separatorDistGuess +) { + static constexpr size_t BLOCK_SIZE = 256; + + std::vector separatorIdxs{}; + separatorIdxs.reserve(data.size() / separatorDistGuess); + + std::array blockSeparatorIdxs{}; + + for(size_t i = 0; i < data.size(); i += BLOCK_SIZE) { + auto const blockEnd = std::min(i + BLOCK_SIZE, data.size()); + size_t separatorC = 0; + + for(size_t j = i; j < blockEnd; j++) { + blockSeparatorIdxs[separatorC] = j; + separatorC += data[j] == separator; + } + + separatorIdxs.append_range(std::span{blockSeparatorIdxs}.first(separatorC)); + } + + return separatorIdxs; } +#endif +} + +namespace lbs { +[[nodiscard]] constexpr std::vector word_ends( + std::span data, + char separator, + size_t separatorDistGuess +) { + auto wordEnds = dev::split(data, separator, separatorDistGuess); + if(wordEnds.empty() || wordEnds.back() != data.size()) + wordEnds.push_back(data.size()); + return wordEnds; +} +} namespace lbs { @@ -89,13 +176,12 @@ namespace lbs { * @details Masks set whole sides to filter same side char sequences. * Invalid chars overlap with everything. Any overlap => invalid word * - * @param compression_index to base masks on + * @param compressionIndex to base masks on * @return uint32_t masks */ [[nodiscard]] constexpr auto gen_char_filter_masks( - compression_index_t const& compression_index + compression_index_t const& compressionIndex ) { - // a side mask sets all bits of a side to 1 static constexpr auto SIDE_MASKS = [] { // first side (first n bits to 1) @@ -112,13 +198,13 @@ namespace lbs { filterFlags.fill(INVALID_CHAR_FLAG); - for(size_t i = 0; i < compression_index.size(); i++) { - auto const& compressedIdx = compression_index[i]; + for(size_t i = 0; i < compressionIndex.size(); i++) { + auto const& compressedIdx = compressionIndex[i]; char_mask_t flag{}; if(compressedIdx == INVALID_CHAR_FLAG) flag = INVALID_CHAR_FLAG; else { - auto const side = compression_index[i] / CHARS_PER_SIDE; + auto const side = compressionIndex[i] / CHARS_PER_SIDE; flag = SIDE_MASKS[side]; } filterFlags[i] = flag; @@ -128,40 +214,35 @@ namespace lbs { return filterFlags; } -struct words_data { - std::vector words; - std::vector wordHashes; -}; [[nodiscard]] constexpr words_data filter_and_hash( - compression_index_t const& compression_index, + compression_index_t const& compressionIndex, std::string_view csv, - std::span splits + std::span wordEnds ) { - auto const charFilterMasks = gen_char_filter_masks(compression_index); + auto const charFilterMasks = gen_char_filter_masks(compressionIndex); static constexpr size_t FILTER_FREQ = 2; words_data out{}; - out.words.reserve(splits.size() / FILTER_FREQ); - out.wordHashes.reserve(splits.size() / FILTER_FREQ); + out.words.reserve(wordEnds.size() / FILTER_FREQ); + out.wordHashes.reserve(wordEnds.size() / FILTER_FREQ); size_t begin = 0; - // <= to handle tail - for(size_t splitI = 0; splitI <= splits.size(); splitI++) { - auto const end = splitI == splits.size() ? csv.size() : splits[splitI]; + for(auto const end : wordEnds) { auto const size = end - begin; + if(size > 0) { auto const word = csv.substr(begin, size); - char_mask_t wordHash = bit_flag(word[0], compression_index); + char_mask_t wordHash = bit_flag(word[0], compressionIndex); size_t i = 1; for(; i < size; i++) { auto const& charMask = charFilterMasks[idx(word[i])]; auto const& prevCharMask = charFilterMasks[idx(word[i - 1])]; - wordHash |= bit_flag(word[i], compression_index); + wordHash |= bit_flag(word[i], compressionIndex); if((charMask & prevCharMask) != 0) break; } diff --git a/lbs_solver/runner/bench/main.cpp b/lbs_solver/runner/bench/main.cpp index 19b8c00..8ab30d1 100644 --- a/lbs_solver/runner/bench/main.cpp +++ b/lbs_solver/runner/bench/main.cpp @@ -25,7 +25,7 @@ constexpr std::string_view PUZZLES_PATH = "assets/nyt_puzzles.txt"; struct word_list { std::vector data; std::string_view csv; - std::vector splits; + std::vector wordEnds; }; struct puzzle { @@ -39,8 +39,8 @@ struct puzzle { [[nodiscard]] word_list load_word_list() { auto data = cth::io::file::read(WORD_LIST_PATH); std::string_view const csv{reinterpret_cast(data.data()), data.size()}; - auto splits = split(csv, ','); - return {std::move(data), csv, std::move(splits)}; + auto wordEnds = word_ends(csv, ','); + return {std::move(data), csv, std::move(wordEnds)}; } /** @@ -58,7 +58,7 @@ struct puzzle { void register_benchmarks(std::span puzzles, word_list const& list) { auto const csv = list.csv; - auto const& splits = list.splits; + auto const& wordEnds = list.wordEnds; benchmark::RegisterBenchmark("lbs/load/word_list/words_easy", [](benchmark::State& state) { for(auto _ : state) { @@ -67,20 +67,20 @@ void register_benchmarks(std::span puzzles, word_list const& list) } }); - benchmark::RegisterBenchmark("lbs/split/word_list/words_easy", [csv](benchmark::State& state) { + benchmark::RegisterBenchmark("lbs/word_ends/word_list/words_easy", [csv](benchmark::State& state) { for(auto _ : state) { - auto splits = split(csv, ','); - benchmark::DoNotOptimize(splits); + auto wordEnds = word_ends(csv, ','); + benchmark::DoNotOptimize(wordEnds); } }); for(auto const& p : puzzles) { auto const name = [&p](std::string_view family) { return std::format("{}/{}/{}", family, p.group, p.date); }; - benchmark::RegisterBenchmark(name("lbs/filter_and_hash"), [&p, csv, &splits](benchmark::State& state) { + benchmark::RegisterBenchmark(name("lbs/filter_and_hash"), [&p, csv, &wordEnds](benchmark::State& state) { for(auto _ : state) { auto const index = gen_compression_index(p.letters); - auto wordsData = filter_and_hash(index, csv, splits); + auto wordsData = filter_and_hash(index, csv, wordEnds); benchmark::DoNotOptimize(wordsData); } }); @@ -92,10 +92,10 @@ void register_benchmarks(std::span puzzles, word_list const& list) } }); - benchmark::RegisterBenchmark(name("lbs/total"), [&p, csv, &splits](benchmark::State& state) { + benchmark::RegisterBenchmark(name("lbs/total"), [&p, csv, &wordEnds](benchmark::State& state) { for(auto _ : state) { auto const index = gen_compression_index(p.letters); - auto const& [words, wordHashes] = filter_and_hash(index, csv, splits); + auto const& [words, wordHashes] = filter_and_hash(index, csv, wordEnds); auto solutions = solve(index, words, wordHashes); benchmark::DoNotOptimize(solutions); } @@ -105,7 +105,7 @@ void register_benchmarks(std::span puzzles, word_list const& list) for(auto _ : state) { auto const list = load_word_list(); auto const index = gen_compression_index(p.letters); - auto const& [words, wordHashes] = filter_and_hash(index, list.csv, list.splits); + auto const& [words, wordHashes] = filter_and_hash(index, list.csv, list.wordEnds); auto solutions = solve(index, words, wordHashes); benchmark::DoNotOptimize(solutions); } @@ -144,7 +144,7 @@ int main(int argc, char** argv) { for(auto& [date, letters] : load_puzzles()) { auto const index = lbs::gen_compression_index(letters); - auto wordsData = lbs::filter_and_hash(index, list.csv, list.splits); + auto wordsData = lbs::filter_and_hash(index, list.csv, list.wordEnds); bool const solvable = !lbs::solve(index, wordsData.words, wordsData.wordHashes).empty(); disagreements += solvable == gpt.run(letters, WORD_LIST_PATH).empty(); diff --git a/lbs_solver/runner/src/main.cpp b/lbs_solver/runner/src/main.cpp index 04e1bf4..11edff0 100644 --- a/lbs_solver/runner/src/main.cpp +++ b/lbs_solver/runner/src/main.cpp @@ -9,7 +9,7 @@ namespace lbs { struct word_list_t { std::string_view csv; - std::vector splits; + std::vector wordEnds; std::vector data{}; }; @@ -18,16 +18,16 @@ struct word_list_t { word_list_t dyn_load(std::string_view word_list_path) { auto data = cth::io::file::read(word_list_path); std::string_view csv{reinterpret_cast(data.data()), data.size()}; - auto splits = lbs::split(csv, ','); - return {csv, std::move(splits), std::move(data)}; + auto wordEnds = lbs::word_ends(csv, ','); + return {csv, std::move(wordEnds), std::move(data)}; } -void print_solutions(std::string_view characters, std::string_view csv, std::span splits) { +void print_solutions(std::string_view characters, std::string_view csv, std::span wordEnds) { auto const compressionIndex = gen_compression_index(characters); auto const& [words, wordHashes] = filter_and_hash( compressionIndex, csv, - splits + wordEnds ); auto const solutions = lbs::solve(compressionIndex, words, wordHashes); @@ -55,9 +55,9 @@ namespace { constexpr std::string_view WORD_LIST_CSV{dev::raw_word_list, sizeof(dev::raw_word_list)}; void solve_manual(std::string_view characters) { - constexpr auto splits = cth::dt::as_cxpr_array<[] { return lbs::split(WORD_LIST_CSV, ','); }>(); + constexpr auto wordEnds = cth::dt::as_cxpr_array<[] { return lbs::word_ends(WORD_LIST_CSV, ','); }>(); - lbs::print_solutions(characters, WORD_LIST_CSV, splits); + lbs::print_solutions(characters, WORD_LIST_CSV, wordEnds); } #else constexpr std::string_view WORD_LIST_PATH{DYNAMIC_WORD_LIST_PATH_DEF}; @@ -65,7 +65,7 @@ namespace { void solve_manual(std::string_view characters) { auto const wordList = lbs::dyn_load(WORD_LIST_PATH); - lbs::print_solutions(characters, wordList.csv, wordList.splits); + lbs::print_solutions(characters, wordList.csv, wordList.wordEnds); } #endif diff --git a/lbs_solver/tests/src/index_test.cpp b/lbs_solver/tests/src/index_test.cpp index ff03680..b7fac83 100644 --- a/lbs_solver/tests/src/index_test.cpp +++ b/lbs_solver/tests/src/index_test.cpp @@ -2,7 +2,9 @@ #include #include +#include #include +#include namespace lbs { @@ -12,18 +14,19 @@ TEST(Index, ctor) { }; constexpr size_t expectedWordC = 3; - auto const splits = split(wordList, ',', 1); + auto const wordEnds = word_ends(wordList, ',', 1); - ASSERT_EQ(splits[0], 1); - ASSERT_EQ(splits[1], 3); - ASSERT_EQ(splits[2], 5); + ASSERT_EQ(wordEnds[0], 1); + ASSERT_EQ(wordEnds[1], 3); + ASSERT_EQ(wordEnds[2], 5); + ASSERT_EQ(wordEnds[3], 6); auto const compressionIndex = gen_compression_index("axxbyyczzzzz"); auto const& [actualWords, actualWordHashes] = filter_and_hash( compressionIndex, wordList, - splits + wordEnds ); ASSERT_EQ(actualWords.size(), expectedWordC); @@ -38,4 +41,35 @@ TEST(Index, ctor) { } + +namespace { + std::vector scalar_word_ends(std::string_view data, char separator) { + std::vector wordEnds{}; + for(size_t i = 0; i < data.size(); i++) + if(data[i] == separator) + wordEnds.push_back(i); + wordEnds.push_back(data.size()); + return wordEnds; + } +} + +static_assert(word_ends(std::string_view{"ab,cd,ef"}, ',') == std::vector{2, 5, 8}); +static_assert(word_ends(std::string_view{"abc"}, ',') == std::vector{3}); +static_assert(word_ends(std::string_view{""}, ',') == std::vector{0}); + +TEST(Index, word_ends_every_size) { + std::string data{}; + for(size_t i = 0; data.size() < 300; i++) + data += std::string(i % 13, 'a') + ','; + + for(size_t size = 0; size <= data.size(); size++) { + std::string_view const prefix{data.data(), size}; + EXPECT_EQ(word_ends(prefix, ','), scalar_word_ends(prefix, ',')) << "size " << size; + } +} + +TEST(Index, word_ends_only_separators) { + std::string const data(300, ','); + EXPECT_EQ(word_ends(data, ','), scalar_word_ends(data, ',')); +} } diff --git a/lbs_solver/tests/src/solver_test.cpp b/lbs_solver/tests/src/solver_test.cpp index a54a68a..26f1e3d 100644 --- a/lbs_solver/tests/src/solver_test.cpp +++ b/lbs_solver/tests/src/solver_test.cpp @@ -11,10 +11,10 @@ namespace lbs { TEST(solver, real_world) { auto const data = cth::io::file::read("assets/words_easy.txt"); std::string_view csv{reinterpret_cast(data.data()), data.size()}; - auto splits = split(csv, ','); + auto wordEnds = word_ends(csv, ','); constexpr auto compressionIndex = gen_compression_index("ouserftlbayh"); - auto const& [words, wordHashes] = filter_and_hash(compressionIndex, csv, splits); + auto const& [words, wordHashes] = filter_and_hash(compressionIndex, csv, wordEnds); auto const solutions = solve(compressionIndex, words, wordHashes); EXPECT_FALSE(solutions.empty()); @@ -25,10 +25,10 @@ TEST(solver, real_world) { TEST(solver, letters_only_inside_words) { auto const data = cth::io::file::read("assets/words_easy.txt"); std::string_view const csv{reinterpret_cast(data.data()), data.size()}; - auto const splits = split(csv, ','); + auto const wordEnds = word_ends(csv, ','); auto const compressionIndex = gen_compression_index("ancgwuytsvoq"); - auto const& [words, wordHashes] = filter_and_hash(compressionIndex, csv, splits); + auto const& [words, wordHashes] = filter_and_hash(compressionIndex, csv, wordEnds); EXPECT_FALSE(solve(compressionIndex, words, wordHashes).empty()); }