From fc0fcebe0e73ce5cf074e57f83cc235ba7d206fd Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sat, 2 May 2026 05:42:15 +0300 Subject: [PATCH 01/29] feat: 3/5 queries (1, 2, 5) --- benchmarks/CMakeLists.txt | 1 + benchmarks/benchmark_queries.cpp | 115 ++++++++++++++++++++++++++----- benchmarks/queries/query_0.csv | 1 + benchmarks/queries/query_1.csv | 1 + benchmarks/queries/query_2.csv | 1 + benchmarks/queries/query_3.csv | 1 + benchmarks/queries/query_4.csv | 1 + benchmarks/query_runners.cpp | 24 +++++++ benchmarks/query_runners.h | 16 +++++ 9 files changed, 143 insertions(+), 18 deletions(-) create mode 100644 benchmarks/queries/query_0.csv create mode 100644 benchmarks/queries/query_1.csv create mode 100644 benchmarks/queries/query_2.csv create mode 100644 benchmarks/queries/query_3.csv create mode 100644 benchmarks/queries/query_4.csv create mode 100644 benchmarks/query_runners.cpp create mode 100644 benchmarks/query_runners.h diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 15f0bdc..6610dc2 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -9,6 +9,7 @@ target_link_libraries(benchmark_csv_to_columnar add_executable(benchmark_queries benchmark_queries.cpp + query_runners.cpp query_runners.h ) target_link_libraries(benchmark_queries diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index 90e8a79..5bea77e 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -1,46 +1,125 @@ +#include #include +#include #include #include +#include +#include "csv_batch_io.h" +#include "executor.h" #include "fileio.h" +#include "query_runners.h" #ifndef COLUMNAR_BENCHMARK_QUERIES_DIR #define COLUMNAR_BENCHMARK_QUERIES_DIR "benchmarks/queries" #endif -static constexpr int kFirstQueryId = 0; -static constexpr int kLastQueryId = 42; +#ifndef COLUMNAR_BENCHMARK_DATA_DIR +#define COLUMNAR_BENCHMARK_DATA_DIR "benchmarks" +#endif + +static constexpr int FirstQueryId = 0; +static constexpr int LastQueryId = 42; + +std::string BatchToString(const Batch& batch) { + std::vector> rows; + AppendBatchRows(batch, rows); + + std::string result; + + for (const auto& row : rows) { + for (size_t i = 0; i < row.size(); ++i) { + result += row[i]; + if (i + 1 < row.size()) { + result += ","; + } + } + result += "\n"; + } + + return result; +} + +void RunQuery(const Executor& executor, const std::filesystem::path& path, const int query_id, bool record = false) { + const auto& runners = GetQueryRunners(); + const auto it = runners.find(query_id); + + ExecuteExpected result; + const auto start = std::chrono::steady_clock::now(); + + if (it != runners.end()) { + result = it->second(executor); + } else if (std::filesystem::exists(path)) { + const std::string sql = ReadTextFile(path); + result = executor.Execute(sql); + } else { + std::cout << "query_" << query_id << ": missing, status=skipped" << std::endl; + return; + } -void RunQuerySkeleton(const std::filesystem::path& path, const int query_id) { - const std::string sql = ReadTextFile(path); - std::cout << "query_" << query_id << ": found, bytes=" << sql.size() << ", status=skeleton_only" << std::endl; + const auto end = std::chrono::steady_clock::now(); + const auto duration_ms = std::chrono::duration_cast(end - start).count(); + + if (!result.has_value()) { + std::cout << "query_" << query_id << ": error=\"" << result.error().GetMessage() << "\", status=failed" + << std::endl; + return; + } + + const std::string actual_result = BatchToString(*result); + const std::filesystem::path ref_path = + std::filesystem::path(COLUMNAR_BENCHMARK_QUERIES_DIR) / ("query_" + std::to_string(query_id) + ".csv"); + + std::string status; + + if (record) { + std::ofstream ref_file(ref_path); + ref_file << actual_result; + status = "recorded"; + } else if (std::filesystem::exists(ref_path)) { + const std::string expected_result = ReadTextFile(ref_path); + if (actual_result == expected_result) { + status = "passed"; + } else { + status = "failed"; + } + } else { + status = "no_reference"; + } + + std::cout << "query_" << query_id << ": rows=" << result->RowsCount() << ", time=" << duration_ms + << "ms, status=" << status << std::endl; } -int main() { +int main(const int argc, char** argv) { try { + bool record = false; + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "--record") { + record = true; + } + } + const std::filesystem::path queries_dir = COLUMNAR_BENCHMARK_QUERIES_DIR; + const std::filesystem::path data_dir = COLUMNAR_BENCHMARK_DATA_DIR; + + Executor executor; + executor.RegisterTable("hits", "hits_sample.columnar"); size_t found_queries = 0; - size_t missing_queries = 0; std::cout << "queries_dir=" << queries_dir << std::endl; - for (int query_id = kFirstQueryId; query_id <= kLastQueryId; ++query_id) { + for (int query_id = FirstQueryId; query_id <= LastQueryId; ++query_id) { const std::filesystem::path query_path = queries_dir / ("query_" + std::to_string(query_id) + ".sql"); - if (!std::filesystem::exists(query_path)) { - ++missing_queries; - std::cout << "query_" << query_id << ": missing, status=skipped" << std::endl; - continue; + if (GetQueryRunners().contains(query_id) || std::filesystem::exists(query_path)) { + ++found_queries; + RunQuery(executor, query_path, query_id, record); } - - ++found_queries; - - RunQuerySkeleton(query_path, query_id); } - std::cout << "found_queries=" << found_queries << std::endl; - std::cout << "missing_queries=" << missing_queries << std::endl; + std::cout << "total_executed=" << found_queries << std::endl; } catch (const std::exception& ex) { std::cerr << ex.what() << std::endl; return 1; diff --git a/benchmarks/queries/query_0.csv b/benchmarks/queries/query_0.csv new file mode 100644 index 0000000..4b9dfa7 --- /dev/null +++ b/benchmarks/queries/query_0.csv @@ -0,0 +1 @@ +999977 diff --git a/benchmarks/queries/query_1.csv b/benchmarks/queries/query_1.csv new file mode 100644 index 0000000..f8cb17a --- /dev/null +++ b/benchmarks/queries/query_1.csv @@ -0,0 +1 @@ +14174 diff --git a/benchmarks/queries/query_2.csv b/benchmarks/queries/query_2.csv new file mode 100644 index 0000000..49e722f --- /dev/null +++ b/benchmarks/queries/query_2.csv @@ -0,0 +1 @@ +80778,999977,1604 diff --git a/benchmarks/queries/query_3.csv b/benchmarks/queries/query_3.csv new file mode 100644 index 0000000..c7b6e3a --- /dev/null +++ b/benchmarks/queries/query_3.csv @@ -0,0 +1 @@ +1948165676197850120 diff --git a/benchmarks/queries/query_4.csv b/benchmarks/queries/query_4.csv new file mode 100644 index 0000000..d3d401f --- /dev/null +++ b/benchmarks/queries/query_4.csv @@ -0,0 +1 @@ +79842 diff --git a/benchmarks/query_runners.cpp b/benchmarks/query_runners.cpp new file mode 100644 index 0000000..b468e37 --- /dev/null +++ b/benchmarks/query_runners.cpp @@ -0,0 +1,24 @@ +#include "query_runners.h" + +ExecuteExpected RunQuery0(const Executor& executor) { return executor.Execute("SELECT COUNT(*) FROM hits;"); } + +ExecuteExpected RunQuery1(const Executor& executor) { + return executor.Execute("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); +} + +ExecuteExpected RunQuery2(const Executor& executor) { + return executor.Execute("SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;"); +} + +ExecuteExpected RunQuery3(const Executor& executor) { return executor.Execute("SELECT AVG(UserID) FROM hits;"); } + +ExecuteExpected RunQuery4(const Executor& executor) { + return executor.Execute("SELECT COUNT(DISTINCT UserID) FROM hits;"); +} + +const std::map& GetQueryRunners() { + static const std::map Runners = { + {0, RunQuery0}, {1, RunQuery1}, {2, RunQuery2}, {3, RunQuery3}, {4, RunQuery4}, + }; + return Runners; +} diff --git a/benchmarks/query_runners.h b/benchmarks/query_runners.h new file mode 100644 index 0000000..46b6f65 --- /dev/null +++ b/benchmarks/query_runners.h @@ -0,0 +1,16 @@ +#pragma once + +#include +#include + +#include "executor.h" + +using QueryRunner = std::function; + +ExecuteExpected RunQuery0(const Executor& executor); +ExecuteExpected RunQuery1(const Executor& executor); +ExecuteExpected RunQuery2(const Executor& executor); +ExecuteExpected RunQuery3(const Executor& executor); +ExecuteExpected RunQuery4(const Executor& executor); + +const std::map& GetQueryRunners(); From bb0fb8c97630238526abc0383794c357e6957ae6 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sat, 2 May 2026 15:14:19 +0300 Subject: [PATCH 02/29] fix: 5/5 queries done --- .clang-tidy | 8 +--- benchmarks/benchmark_queries.cpp | 13 +++--- src/executor/aggregate_states.cpp | 66 +------------------------------ tests/test_executor.cpp | 23 +++++------ 4 files changed, 22 insertions(+), 88 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 3d705b4..26f6e6d 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,5 @@ --- -Checks: '-*,cppcoreguidelines-avoid-goto,cppcoreguidelines-pro-type-const-cast, google-readability-casting, modernize-replace-random-shuffle, readability-braces-around-statements, readability-container-size-empty, readability-redundant-control-flow, readability-redundant-string-init, modernize-use-nullptr, readability-identifier-naming, google-build-using-namespace' +Checks: '-*,cppcoreguidelines-avoid-goto,cppcoreguidelines-pro-type-const-cast, google-readability-casting, modernize-replace-random-shuffle, readability-braces-around-statements, readability-container-size-empty, readability-redundant-control-flow, readability-redundant-string-init, modernize-use-nullptr, readability-identifier-naming, google-build-using-namespace' HeaderFilterRegex: '\.h$' WarningsAsErrors: '*' CheckOptions: @@ -25,17 +25,11 @@ CheckOptions: value: lower_case - key: readability-identifier-naming.ParameterCase value: lower_case - - key: readability-identifier-naming.GlobalConstantPrefix - value: k - key: readability-identifier-naming.GlobalConstantCase value: CamelCase - - key: readability-identifier-naming.StaticConstantPrefix - value: k - key: readability-identifier-naming.StaticConstantCase value: CamelCase - key: readability-identifier-naming.ConstexprVariableCase value: CamelCase - - key: readability-identifier-naming.ConstexprVariablePrefix - value: k - key: google-runtime-int.TypeSuffix value: _t \ No newline at end of file diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index 5bea77e..c3cedb9 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -14,10 +14,6 @@ #define COLUMNAR_BENCHMARK_QUERIES_DIR "benchmarks/queries" #endif -#ifndef COLUMNAR_BENCHMARK_DATA_DIR -#define COLUMNAR_BENCHMARK_DATA_DIR "benchmarks" -#endif - static constexpr int FirstQueryId = 0; static constexpr int LastQueryId = 42; @@ -82,6 +78,14 @@ void RunQuery(const Executor& executor, const std::filesystem::path& path, const status = "passed"; } else { status = "failed"; + std::cout << "query_" << query_id << ": expected:\n" << expected_result; + if (!expected_result.empty() && expected_result.back() != '\n') { + std::cout << std::endl; + } + std::cout << "query_" << query_id << ": actual:\n" << actual_result; + if (!actual_result.empty() && actual_result.back() != '\n') { + std::cout << std::endl; + } } } else { status = "no_reference"; @@ -101,7 +105,6 @@ int main(const int argc, char** argv) { } const std::filesystem::path queries_dir = COLUMNAR_BENCHMARK_QUERIES_DIR; - const std::filesystem::path data_dir = COLUMNAR_BENCHMARK_DATA_DIR; Executor executor; executor.RegisterTable("hits", "hits_sample.columnar"); diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index e9d00ef..b05e3ce 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -81,75 +81,11 @@ static Int128 ParseToInt128(const ColumnType type, const std::string_view value) throw Error::Unsupported("executor", "unsupported numeric conversion"); } -static UInt128 ToMagnitude(const Int128 value) { - if (value >= 0) { - return static_cast(value); - } - - UInt128 magnitude = static_cast(-(value + 1)); - ++magnitude; - - return magnitude; -} - -static std::string UInt128ToString(UInt128 value) { - if (value == 0) { - return "0"; - } - - std::string out; - - while (value > 0) { - out.push_back(static_cast('0' + value % 10)); - value /= 10; - } - - std::ranges::reverse(out); - - return out; -} - static std::string FormatAverage(const Int128 sum, const uint64_t count) { if (count == 0) { return "0"; } - - const bool negative = sum < 0; - const UInt128 magnitude = ToMagnitude(sum); - const UInt128 divisor = count; - const UInt128 integer = magnitude / divisor; - UInt128 remainder = magnitude % divisor; - - std::string output = UInt128ToString(integer); - - if (remainder != 0) { - output.push_back('.'); - - std::string fractional; - fractional.reserve(6); - - for (size_t i = 0; i < 6 && remainder != 0; ++i) { - remainder *= 10; - fractional.push_back(static_cast('0' + remainder / divisor)); - remainder %= divisor; - } - - while (!fractional.empty() && fractional.back() == '0') { - fractional.pop_back(); - } - - if (!fractional.empty()) { - output += fractional; - } else { - output.pop_back(); - } - } - - if (negative && output != "0") { - output.insert(output.begin(), '-'); - } - - return output; + return Int128ToString(sum / static_cast(count)); } class CountAS final : public AggState { diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index e44a044..f187705 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -38,45 +38,46 @@ static Batch BuildHitsTable(const std::string_view query) { if (!result.has_value()) { throw result.error(); } + return std::move(result.value()); } TEST(executor, executes_basic_aggregate_queries) { { const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(*)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"5"})); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"5"}); } { const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(*)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4"})); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { const Batch batch = BuildHitsTable("SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;"); EXPECT_EQ(BatchColumnNames(batch), (std::vector{"SUM(AdvEngineID)", "COUNT(*)", "AVG(ResolutionWidth)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"18", "5", "300.2"})); + EXPECT_EQ(SingleRowValues(batch), (std::vector{"18", "5", "300"})); } { const Batch batch = BuildHitsTable("SELECT AVG(UserID) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AVG(UserID)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4.2"})); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"AVG(UserID)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { const Batch batch = BuildHitsTable("SELECT COUNT(DISTINCT UserID) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(DISTINCT UserID)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4"})); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(DISTINCT UserID)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { const Batch batch = BuildHitsTable("SELECT COUNT(DISTINCT SearchPhrase) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(DISTINCT SearchPhrase)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4"})); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(DISTINCT SearchPhrase)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { From a8786c70c589112b6413e39d3368733f2631fb60 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Mon, 4 May 2026 17:44:14 +0300 Subject: [PATCH 03/29] feat: added job for github workflow --- .github/workflows/cmake-single-platform.yml | 13 +++++++++++++ benchmarks/benchmark_queries.cpp | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cmake-single-platform.yml b/.github/workflows/cmake-single-platform.yml index 5826612..f414afd 100644 --- a/.github/workflows/cmake-single-platform.yml +++ b/.github/workflows/cmake-single-platform.yml @@ -39,6 +39,11 @@ jobs: ./tests/test_stream ./tests/test_tokenizer + - name: Benchmark Queries + run: | + cd benchmarks + ../build/benchmarks/benchmark_queries + sanitizers: runs-on: ubuntu-latest @@ -70,3 +75,11 @@ jobs: run: | cd build-sanitizers ctest --output-on-failure + + - name: Benchmark Queries With Sanitizers + env: + ASAN_OPTIONS: detect_leaks=1:strict_string_checks=1:check_initialization_order=1 + UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 + run: | + cd benchmarks + ../build-sanitizers/benchmarks/benchmark_queries diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index c3cedb9..75d6da5 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -107,7 +107,7 @@ int main(const int argc, char** argv) { const std::filesystem::path queries_dir = COLUMNAR_BENCHMARK_QUERIES_DIR; Executor executor; - executor.RegisterTable("hits", "hits_sample.columnar"); + executor.RegisterTable("hits", "benchmarks/hits_sample.columnar"); size_t found_queries = 0; From 555dcf10b324a380fd017f38089293816aa1495b Mon Sep 17 00:00:00 2001 From: ik4rd Date: Mon, 4 May 2026 19:10:01 +0300 Subject: [PATCH 04/29] fix: f*ck this columnar shit... --- benchmarks/benchmark_csv_to_columnar.cpp | 4 ++-- benchmarks/benchmark_queries.cpp | 2 +- src/app/main.cpp | 2 +- src/model/schema.cpp | 7 +++++-- tests/test_columnar.cpp | 11 +++++++++++ 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/benchmarks/benchmark_csv_to_columnar.cpp b/benchmarks/benchmark_csv_to_columnar.cpp index 77632ba..d5fa0d6 100644 --- a/benchmarks/benchmark_csv_to_columnar.cpp +++ b/benchmarks/benchmark_csv_to_columnar.cpp @@ -28,7 +28,7 @@ #define COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA "benchmarks/schema_sample_new.csv" #endif -static constexpr size_t kRowsPerGroup = 50000; +static constexpr size_t RowsPerGroup = 50000; struct CsvCompareResult { bool equal = true; @@ -64,7 +64,7 @@ static CsvCompareResult CompareCsvFiles(const std::filesystem::path& lhs_path, c int main(const int argc, char** argv) { try { - size_t rows_per_group = kRowsPerGroup; + size_t rows_per_group = RowsPerGroup; if (argc == 2) { rows_per_group = std::stoull(argv[1]); } diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index 75d6da5..c3cedb9 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -107,7 +107,7 @@ int main(const int argc, char** argv) { const std::filesystem::path queries_dir = COLUMNAR_BENCHMARK_QUERIES_DIR; Executor executor; - executor.RegisterTable("hits", "benchmarks/hits_sample.columnar"); + executor.RegisterTable("hits", "hits_sample.columnar"); size_t found_queries = 0; diff --git a/src/app/main.cpp b/src/app/main.cpp index 31ed2c5..df9a2ba 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -21,7 +21,7 @@ void ConfigureConvertCommand(argparse::ArgumentParser& command) { command.add_argument("--schema").required(); command.add_argument("--input").required(); command.add_argument("--output").required(); - command.add_argument("--row-group-size").scan<'u', size_t>().default_value(50000); + command.add_argument("--row-group-size").scan<'u', size_t>().default_value(size_t{50000}); } void ConfigureRoundtripCommand(argparse::ArgumentParser& command) { diff --git a/src/model/schema.cpp b/src/model/schema.cpp index 3c0aae4..b1d37cb 100644 --- a/src/model/schema.cpp +++ b/src/model/schema.cpp @@ -45,12 +45,12 @@ static bool CanParseAs(const ColumnType type, const std::string& value) { } static ColumnType InferColumnType(const std::vector& values) { - static constexpr std::array kInferenceOrder = { + static constexpr std::array InferenceOrder = { ColumnType::Boolean, ColumnType::Int16, ColumnType::Int32, ColumnType::Int64, ColumnType::Int128, ColumnType::Date, ColumnType::Timestamp, ColumnType::Character, ColumnType::String, }; - for (const ColumnType type : kInferenceOrder) { + for (const ColumnType type : InferenceOrder) { bool matches = true; for (const auto& value : values) { if (!CanParseAs(type, value)) { @@ -76,6 +76,9 @@ Schema ReadSchemaCsv(const std::filesystem::path& path) { if (row.size() == 1 && row[0].empty()) { continue; } + while (row.size() > 2 && row.back().empty()) { + row.pop_back(); + } if (row.size() != 2) { throw Error::InvalidData("schema", "schema csv must have 2 columns", path.string()); } diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp index 4132268..d0a2f87 100644 --- a/tests/test_columnar.cpp +++ b/tests/test_columnar.cpp @@ -195,3 +195,14 @@ TEST(columnar, infer_schema_from_csv_is_convert_compatible) { EXPECT_EQ(ReadRows(data_out.Path()), data_rows); } + +TEST(columnar, schema_rows_with_trailing_empty_columns_are_accepted) { + const TempFile schema_in("schema_trailing_empty"); + const TempFile data_in("data_trailing_empty"); + const TempFile columnar_file("columnar_trailing_empty"); + + WriteRows(schema_in.Path(), {{"id", "int64", ""}, {"name", "string", ""}}); + WriteRows(data_in.Path(), {{"1", "alpha"}, {"2", "beta"}}); + + EXPECT_NO_THROW(ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 2)); +} From 132b2893dcab9a4e4c288f0cc2f06dd4ca9074e0 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Mon, 4 May 2026 23:33:12 +0300 Subject: [PATCH 05/29] fix: cosmetic stuff --- .editorconfig | 10 +++++ CMakeLists.txt | 11 +----- benchmarks/CMakeLists.txt | 2 +- benchmarks/benchmark_csv_to_columnar.cpp | 6 +-- benchmarks/benchmark_queries.cpp | 8 ++-- benchmarks/query_runners.cpp | 24 ------------ benchmarks/query_runners.h | 16 -------- benchmarks/support/query_runners.h | 35 +++++++++++++++++ include/convert/csv_columnar.h | 2 - include/executor/aggregate_function.h | 4 +- include/executor/executor.h | 12 +++--- include/io/batch_io.h | 4 +- include/io/columnar_batch_io.h | 13 ++++--- include/io/csv.h | 2 - include/io/csv_batch_io.h | 6 +-- include/io/fileio.h | 2 - include/io/stream.h | 4 +- include/model/batch.h | 6 +-- include/model/column.h | 4 +- include/model/column_boolean.h | 4 +- include/model/column_character.h | 4 +- include/model/column_date.h | 4 +- include/model/column_int128.h | 6 +-- include/model/column_int16.h | 4 +- include/model/column_int32.h | 4 +- include/model/column_int64.h | 4 +- include/model/column_string.h | 4 +- include/model/column_timestamp.h | 4 +- include/model/fixed_column.h | 8 ++-- include/model/metadata.h | 4 +- include/model/schema.h | 2 - include/model/variant_column.h | 8 ++-- include/sql_parser/tokenizer.h | 6 +-- include/support/error.h | 2 - include/support/int128.h | 2 - include/support/parsing.h | 6 +-- include/testing/executor_test_utils.h | 2 +- include/testing/temp_file.h | 2 - src/app/main.cpp | 12 +++--- src/convert/csv_columnar.cpp | 14 ++++--- src/executor/aggregate_states.cpp | 10 ++--- src/executor/executor.cpp | 4 +- src/executor/operators.cpp | 8 ++-- src/executor/query_parser.cpp | 6 +-- src/executor/query_planner.cpp | 6 +-- src/io/batch_io.cpp | 34 +++++++++-------- src/io/csv.cpp | 6 +-- src/io/fileio.cpp | 4 +- src/io/stream.cpp | 4 +- src/model/batch.cpp | 4 +- src/model/column.cpp | 22 +++++------ src/model/column_boolean.cpp | 4 +- src/model/column_character.cpp | 4 +- src/model/column_date.cpp | 4 +- src/model/column_int128.cpp | 4 +- src/model/column_int16.cpp | 4 +- src/model/column_int32.cpp | 4 +- src/model/column_int64.cpp | 4 +- src/model/column_string.cpp | 2 +- src/model/column_timestamp.cpp | 4 +- src/model/metadata.cpp | 6 +-- src/model/schema.cpp | 10 ++--- src/sql_parser/tokenizer.cpp | 6 +-- src/sql_parser/tokenizer_factory.cpp | 6 +-- src/sql_parser/tokenizer_tokens.cpp | 6 +-- src/support/ascii.cpp | 2 +- src/support/error.cpp | 2 +- src/support/parsing.cpp | 6 +-- tests/test_batch.cpp | 48 ++++++++++++++++++++---- tests/test_columnar.cpp | 16 ++++---- tests/test_columns.cpp | 10 ++--- tests/test_csv.cpp | 2 +- tests/test_executor.cpp | 22 +++++------ tests/test_fileio.cpp | 6 +-- tests/test_stream.cpp | 2 +- tests/test_tokenizer.cpp | 10 ++--- 76 files changed, 279 insertions(+), 300 deletions(-) create mode 100644 .editorconfig delete mode 100644 benchmarks/query_runners.cpp delete mode 100644 benchmarks/query_runners.h create mode 100644 benchmarks/support/query_runners.h diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4d761dd --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +trim_trailing_whitespace = false diff --git a/CMakeLists.txt b/CMakeLists.txt index aeeaf52..eba6976 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,18 +14,9 @@ include(cmake/dependencies_package.cmake) include(cmake/sanitizers.cmake) set(COLUMNAR_INCLUDE_DIRS - ${PROJECT_SOURCE_DIR}/include/convert - ${PROJECT_SOURCE_DIR}/include/io - ${PROJECT_SOURCE_DIR}/include/model - ${PROJECT_SOURCE_DIR}/include/support - ${PROJECT_SOURCE_DIR}/include/testing - ${PROJECT_SOURCE_DIR}/include/executor - ${PROJECT_SOURCE_DIR}/include/sql_parser - ${PROJECT_SOURCE_DIR}/include/tl + ${PROJECT_SOURCE_DIR}/include ) -include_directories(${COLUMNAR_INCLUDE_DIRS}) - add_compile_options(-Wall -Wextra -Werror -Wpedantic -O2) set(COLUMNAR_SANITIZERS "" CACHE STRING "Comma or semicolon separated list of sanitizers: address,undefined,leak,thread,memory") diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 6610dc2..9102491 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -9,7 +9,7 @@ target_link_libraries(benchmark_csv_to_columnar add_executable(benchmark_queries benchmark_queries.cpp - query_runners.cpp query_runners.h + support/query_runners.h ) target_link_libraries(benchmark_queries diff --git a/benchmarks/benchmark_csv_to_columnar.cpp b/benchmarks/benchmark_csv_to_columnar.cpp index d5fa0d6..db124f5 100644 --- a/benchmarks/benchmark_csv_to_columnar.cpp +++ b/benchmarks/benchmark_csv_to_columnar.cpp @@ -4,9 +4,9 @@ #include #include -#include "csv.h" -#include "csv_columnar.h" -#include "error.h" +#include "io/csv.h" +#include "convert/csv_columnar.h" +#include "support/error.h" #ifndef COLUMNAR_BENCHMARK_DEFAULT_DATA #define COLUMNAR_BENCHMARK_DEFAULT_DATA "benchmarks/hits_sample.csv" diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index c3cedb9..227a3f1 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -5,10 +5,10 @@ #include #include -#include "csv_batch_io.h" -#include "executor.h" -#include "fileio.h" -#include "query_runners.h" +#include "io/csv_batch_io.h" +#include "executor/executor.h" +#include "io/fileio.h" +#include "support/query_runners.h" #ifndef COLUMNAR_BENCHMARK_QUERIES_DIR #define COLUMNAR_BENCHMARK_QUERIES_DIR "benchmarks/queries" diff --git a/benchmarks/query_runners.cpp b/benchmarks/query_runners.cpp deleted file mode 100644 index b468e37..0000000 --- a/benchmarks/query_runners.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#include "query_runners.h" - -ExecuteExpected RunQuery0(const Executor& executor) { return executor.Execute("SELECT COUNT(*) FROM hits;"); } - -ExecuteExpected RunQuery1(const Executor& executor) { - return executor.Execute("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); -} - -ExecuteExpected RunQuery2(const Executor& executor) { - return executor.Execute("SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;"); -} - -ExecuteExpected RunQuery3(const Executor& executor) { return executor.Execute("SELECT AVG(UserID) FROM hits;"); } - -ExecuteExpected RunQuery4(const Executor& executor) { - return executor.Execute("SELECT COUNT(DISTINCT UserID) FROM hits;"); -} - -const std::map& GetQueryRunners() { - static const std::map Runners = { - {0, RunQuery0}, {1, RunQuery1}, {2, RunQuery2}, {3, RunQuery3}, {4, RunQuery4}, - }; - return Runners; -} diff --git a/benchmarks/query_runners.h b/benchmarks/query_runners.h deleted file mode 100644 index 46b6f65..0000000 --- a/benchmarks/query_runners.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include -#include - -#include "executor.h" - -using QueryRunner = std::function; - -ExecuteExpected RunQuery0(const Executor& executor); -ExecuteExpected RunQuery1(const Executor& executor); -ExecuteExpected RunQuery2(const Executor& executor); -ExecuteExpected RunQuery3(const Executor& executor); -ExecuteExpected RunQuery4(const Executor& executor); - -const std::map& GetQueryRunners(); diff --git a/benchmarks/support/query_runners.h b/benchmarks/support/query_runners.h new file mode 100644 index 0000000..3776cfa --- /dev/null +++ b/benchmarks/support/query_runners.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +#include "executor/executor.h" + +using QueryRunner = std::function; + +inline ExecuteExpected RunQuery0(const Executor& executor) { return executor.Execute("SELECT COUNT(*) FROM hits;"); } + +inline ExecuteExpected RunQuery1(const Executor& executor) { + return executor.Execute("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); +} + +inline ExecuteExpected RunQuery2(const Executor& executor) { + return executor.Execute("SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;"); +} + +inline ExecuteExpected RunQuery3(const Executor& executor) { return executor.Execute("SELECT AVG(UserID) FROM hits;"); } + +inline ExecuteExpected RunQuery4(const Executor& executor) { + return executor.Execute("SELECT COUNT(DISTINCT UserID) FROM hits;"); +} + +inline const std::map& GetQueryRunners() { + static const std::map Runners = { + {0, RunQuery0}, + {1, RunQuery1}, + {2, RunQuery2}, + {3, RunQuery3}, + {4, RunQuery4}, + }; + return Runners; +} diff --git a/include/convert/csv_columnar.h b/include/convert/csv_columnar.h index d58a734..cd2b5f5 100644 --- a/include/convert/csv_columnar.h +++ b/include/convert/csv_columnar.h @@ -1,5 +1,3 @@ -/* (What is this? — CSV <-> колоночный формат) */ - #pragma once #include diff --git a/include/executor/aggregate_function.h b/include/executor/aggregate_function.h index 0494d4e..69435ef 100644 --- a/include/executor/aggregate_function.h +++ b/include/executor/aggregate_function.h @@ -1,5 +1,3 @@ -/* (What is this? — Объявление функций агрегации) */ - #pragma once #include @@ -7,7 +5,7 @@ #include #include -#include "schema.h" +#include "model/schema.h" class AggState; struct PlannedAgg; diff --git a/include/executor/executor.h b/include/executor/executor.h index be10e0b..aa2cd1d 100644 --- a/include/executor/executor.h +++ b/include/executor/executor.h @@ -1,5 +1,3 @@ -/* (What is this? — Объявление классов и функций исполнителя запросов) */ - #pragma once #include @@ -10,11 +8,11 @@ #include #include -#include "aggregate_function.h" -#include "batch.h" -#include "error.h" -#include "expected.hpp" -#include "tokenizer.h" +#include "executor/aggregate_function.h" +#include "model/batch.h" +#include "support/error.h" +#include "tl/expected.hpp" +#include "sql_parser/tokenizer.h" using ExecuteExpected = tl::expected; diff --git a/include/io/batch_io.h b/include/io/batch_io.h index 3fea3d1..ff8b47b 100644 --- a/include/io/batch_io.h +++ b/include/io/batch_io.h @@ -1,10 +1,8 @@ -/* (What is this? — Потоковое чтение/запись батчей из CSV в колоночный формат) */ - #pragma once #include -#include "batch.h" +#include "model/batch.h" struct BatchSizing { std::optional max_rows; diff --git a/include/io/columnar_batch_io.h b/include/io/columnar_batch_io.h index d04f02b..6cafa3b 100644 --- a/include/io/columnar_batch_io.h +++ b/include/io/columnar_batch_io.h @@ -1,14 +1,12 @@ -/* (What is this? — Чтение/запись батчей из CSV в колоночный формат) */ - #pragma once #include #include #include -#include "batch_io.h" -#include "fileio.h" -#include "metadata.h" +#include "io/batch_io.h" +#include "io/fileio.h" +#include "model/metadata.h" class ColumnarBatchReader final : public BatchReader { public: @@ -49,10 +47,13 @@ class ColumnarBatchWriter final : public BatchWriter { void Write(const Batch& batch) override; void Flush() override; - void Finalize(); + void Finalize() &&; const ColumnarMetadata& GetMetadata() const { return metadata_; } + private: + void FinalizeImpl(); + private: std::filesystem::path path_; std::ofstream out_; diff --git a/include/io/csv.h b/include/io/csv.h index e01e998..1601a20 100644 --- a/include/io/csv.h +++ b/include/io/csv.h @@ -1,5 +1,3 @@ -/* (What is this? — Чтение/запись CSV) */ - #pragma once #include diff --git a/include/io/csv_batch_io.h b/include/io/csv_batch_io.h index 86c108a..881bd7c 100644 --- a/include/io/csv_batch_io.h +++ b/include/io/csv_batch_io.h @@ -1,13 +1,11 @@ -/* (What is this? — Чтение/запись CSV) */ - #pragma once #include #include #include -#include "batch_io.h" -#include "csv.h" +#include "io/batch_io.h" +#include "io/csv.h" class CsvBatchReader final : public BatchReader { public: diff --git a/include/io/fileio.h b/include/io/fileio.h index c2d9b36..fd07161 100644 --- a/include/io/fileio.h +++ b/include/io/fileio.h @@ -1,5 +1,3 @@ -/* (What is this? — Обертки над потоками, чтение и запись) */ - #pragma once #include diff --git a/include/io/stream.h b/include/io/stream.h index 676400a..04b11ba 100644 --- a/include/io/stream.h +++ b/include/io/stream.h @@ -1,5 +1,3 @@ -/* (What is this? — Обертки над потоками, чтение и запись) */ - #pragma once #include @@ -13,7 +11,7 @@ #include #include -#include "error.h" +#include "support/error.h" void ReadBytes(std::istream& in, char* dst, size_t size); void WriteBytes(std::ostream& out, std::string_view bytes); diff --git a/include/model/batch.h b/include/model/batch.h index 08d5667..1495aa5 100644 --- a/include/model/batch.h +++ b/include/model/batch.h @@ -1,12 +1,10 @@ -/* (What is this? — Класс батча) */ - #pragma once #include #include -#include "column.h" -#include "schema.h" +#include "model/column.h" +#include "model/schema.h" class Batch { public: diff --git a/include/model/column.h b/include/model/column.h index dc9c770..3a15866 100644 --- a/include/model/column.h +++ b/include/model/column.h @@ -1,5 +1,3 @@ -/* (What is this? — Абстракция колонок) */ - #pragma once #include @@ -8,7 +6,7 @@ #include #include -#include "schema.h" +#include "model/schema.h" class Column { public: diff --git a/include/model/column_boolean.h b/include/model/column_boolean.h index 4194b4a..33614e2 100644 --- a/include/model/column_boolean.h +++ b/include/model/column_boolean.h @@ -1,8 +1,6 @@ -/* (What is this? — Колонка логического типа) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class BooleanColumn final : public FixedColumn { public: diff --git a/include/model/column_character.h b/include/model/column_character.h index 9ba8684..b17baf7 100644 --- a/include/model/column_character.h +++ b/include/model/column_character.h @@ -1,8 +1,6 @@ -/* (What is this? — Колонка символьного типа) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class CharacterColumn final : public FixedColumn { public: diff --git a/include/model/column_date.h b/include/model/column_date.h index fa185a4..c7a9e73 100644 --- a/include/model/column_date.h +++ b/include/model/column_date.h @@ -1,8 +1,6 @@ -/* (What is this? — Колонка типа DATE) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class DateColumn final : public FixedColumn { public: diff --git a/include/model/column_int128.h b/include/model/column_int128.h index d24250d..282fe15 100644 --- a/include/model/column_int128.h +++ b/include/model/column_int128.h @@ -1,9 +1,7 @@ -/* (What is this? — Колонка типа INT128) */ - #pragma once -#include "fixed_column.h" -#include "int128.h" +#include "model/fixed_column.h" +#include "support/int128.h" class Int128Column final : public FixedColumn { public: diff --git a/include/model/column_int16.h b/include/model/column_int16.h index 14a0811..64fe72f 100644 --- a/include/model/column_int16.h +++ b/include/model/column_int16.h @@ -1,8 +1,6 @@ -/* (What is this? — Колонка типа INT16) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class Int16Column final : public FixedColumn { public: diff --git a/include/model/column_int32.h b/include/model/column_int32.h index 458e60a..7ffd398 100644 --- a/include/model/column_int32.h +++ b/include/model/column_int32.h @@ -1,8 +1,6 @@ -/* (What is this? — Колонка типа INT32) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class Int32Column final : public FixedColumn { public: diff --git a/include/model/column_int64.h b/include/model/column_int64.h index aa44865..f28adb4 100644 --- a/include/model/column_int64.h +++ b/include/model/column_int64.h @@ -1,8 +1,6 @@ -/* (What is this? — Колонка типа INT64) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class Int64Column final : public FixedColumn { public: diff --git a/include/model/column_string.h b/include/model/column_string.h index 60589f8..0d6542a 100644 --- a/include/model/column_string.h +++ b/include/model/column_string.h @@ -1,8 +1,6 @@ -/* (What is this? — Колонка типа STRING) */ - #pragma once -#include "variant_column.h" +#include "model/variant_column.h" class StringColumn final : public VariantColumn { public: diff --git a/include/model/column_timestamp.h b/include/model/column_timestamp.h index 6fb061b..d6be6c4 100644 --- a/include/model/column_timestamp.h +++ b/include/model/column_timestamp.h @@ -1,8 +1,6 @@ -/* (What is this? — Колонка типа TIMESTAMP) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class TimestampColumn final : public FixedColumn { public: diff --git a/include/model/fixed_column.h b/include/model/fixed_column.h index ce5b169..6a82881 100644 --- a/include/model/fixed_column.h +++ b/include/model/fixed_column.h @@ -1,12 +1,10 @@ -/* (What is this? — База для колонок фиксированного размера) */ - #pragma once #include -#include "column.h" -#include "error.h" -#include "stream.h" +#include "io/stream.h" +#include "model/column.h" +#include "support/error.h" template class FixedColumn : public Column { diff --git a/include/model/metadata.h b/include/model/metadata.h index f442ac0..9a9b339 100644 --- a/include/model/metadata.h +++ b/include/model/metadata.h @@ -1,12 +1,10 @@ -/* (What is this? — Сериализация метаданных) */ - #pragma once #include #include #include -#include "schema.h" +#include "model/schema.h" struct ColumnChunkMetadata { uint64_t offset = 0; diff --git a/include/model/schema.h b/include/model/schema.h index 2e40fd7..8cce2c9 100644 --- a/include/model/schema.h +++ b/include/model/schema.h @@ -1,5 +1,3 @@ -/* (What is this? — Чтение/запись схемы из CSV) */ - #pragma once #include diff --git a/include/model/variant_column.h b/include/model/variant_column.h index bbea74f..7db4599 100644 --- a/include/model/variant_column.h +++ b/include/model/variant_column.h @@ -1,14 +1,12 @@ -/* (What is this? — База для колонок переменной длины) */ - #pragma once #include #include #include -#include "column.h" -#include "error.h" -#include "stream.h" +#include "io/stream.h" +#include "model/column.h" +#include "support/error.h" template class VariantColumn : public Column { diff --git a/include/sql_parser/tokenizer.h b/include/sql_parser/tokenizer.h index 19bacb8..66adcb8 100644 --- a/include/sql_parser/tokenizer.h +++ b/include/sql_parser/tokenizer.h @@ -6,8 +6,8 @@ #include #include -#include "error.h" -#include "expected.hpp" +#include "support/error.h" +#include "tl/expected.hpp" enum class Tokens { NameToken, @@ -48,7 +48,7 @@ enum class Tokens { Greater, GreaterOrEqual, - EOI, + EndOfInput, }; std::string_view TokenTypeToString(Tokens type) noexcept; diff --git a/include/support/error.h b/include/support/error.h index 8cd7a87..034fc4f 100644 --- a/include/support/error.h +++ b/include/support/error.h @@ -1,5 +1,3 @@ -/* (What is this? — Модуль ошибок) */ - #pragma once #include diff --git a/include/support/int128.h b/include/support/int128.h index 7b9f28a..0b15cbf 100644 --- a/include/support/int128.h +++ b/include/support/int128.h @@ -1,5 +1,3 @@ -/* (What is this? — Алиасы для 128-битных целых) */ - #pragma once #if !defined(__SIZEOF_INT128__) diff --git a/include/support/parsing.h b/include/support/parsing.h index 97ec50d..6a4d585 100644 --- a/include/support/parsing.h +++ b/include/support/parsing.h @@ -1,13 +1,11 @@ -/* (What is this? — Парсеры) */ - #pragma once #include #include #include -#include "int128.h" -#include "schema.h" +#include "model/schema.h" +#include "support/int128.h" bool ParseBoolean(const std::string& value); int16_t ParseInt16(const std::string& value); diff --git a/include/testing/executor_test_utils.h b/include/testing/executor_test_utils.h index 55f3c9d..ee94d4b 100644 --- a/include/testing/executor_test_utils.h +++ b/include/testing/executor_test_utils.h @@ -3,8 +3,8 @@ #include #include -#include "csv_batch_io.h" #include "gtest/gtest.h" +#include "io/csv_batch_io.h" inline std::vector SingleRowValues(const Batch& batch) { EXPECT_EQ(batch.RowsCount(), 1u); diff --git a/include/testing/temp_file.h b/include/testing/temp_file.h index f4cea19..d729551 100644 --- a/include/testing/temp_file.h +++ b/include/testing/temp_file.h @@ -1,5 +1,3 @@ -/* (What is this? — Модуль временного файла) */ - #pragma once #include diff --git a/src/app/main.cpp b/src/app/main.cpp index df9a2ba..21b790f 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -3,12 +3,12 @@ #include #include -#include "csv_batch_io.h" -#include "csv_columnar.h" -#include "error.h" -#include "executor.h" -#include "fileio.h" -#include "schema.h" +#include "io/csv_batch_io.h" +#include "convert/csv_columnar.h" +#include "support/error.h" +#include "executor/executor.h" +#include "io/fileio.h" +#include "model/schema.h" void ConfigureInferSchemaCommand(argparse::ArgumentParser& command) { command.add_description("Infer a schema from a CSV file."); diff --git a/src/convert/csv_columnar.cpp b/src/convert/csv_columnar.cpp index 3f9d600..590ee89 100644 --- a/src/convert/csv_columnar.cpp +++ b/src/convert/csv_columnar.cpp @@ -1,9 +1,11 @@ -#include "csv_columnar.h" +#include -#include "columnar_batch_io.h" -#include "csv_batch_io.h" -#include "error.h" -#include "schema.h" +#include "convert/csv_columnar.h" + +#include "io/columnar_batch_io.h" +#include "io/csv_batch_io.h" +#include "support/error.h" +#include "model/schema.h" void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, const std::filesystem::path& output_path, size_t max_rows_per_group) { @@ -23,7 +25,7 @@ void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::f batch_writer.Write(*batch); } - batch_writer.Finalize(); + std::move(batch_writer).Finalize(); } void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index b05e3ce..bda9e89 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -4,11 +4,11 @@ #include #include -#include "aggregate_function.h" -#include "ascii.h" -#include "executor.h" -#include "int128.h" -#include "parsing.h" +#include "executor/aggregate_function.h" +#include "support/ascii.h" +#include "executor/executor.h" +#include "support/int128.h" +#include "support/parsing.h" static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view candidate, const std::string_view current, const bool is_min) { diff --git a/src/executor/executor.cpp b/src/executor/executor.cpp index 5f8bc82..91e222d 100644 --- a/src/executor/executor.cpp +++ b/src/executor/executor.cpp @@ -1,8 +1,8 @@ -#include "executor.h" +#include "executor/executor.h" #include -#include "ascii.h" +#include "support/ascii.h" void Executor::RegisterTable(const std::string& name, std::filesystem::path path) { tables_[ToLowerAscii(name)] = std::move(path); diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index d12602b..6da4aa7 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -2,10 +2,10 @@ #include #include -#include "columnar_batch_io.h" -#include "executor.h" -#include "fileio.h" -#include "parsing.h" +#include "io/columnar_batch_io.h" +#include "executor/executor.h" +#include "io/fileio.h" +#include "support/parsing.h" static void SeekRead(const std::filesystem::path& path, std::ifstream& in, const uint64_t offset) { in.seekg(static_cast(offset)); diff --git a/src/executor/query_parser.cpp b/src/executor/query_parser.cpp index 396b093..1c61a89 100644 --- a/src/executor/query_parser.cpp +++ b/src/executor/query_parser.cpp @@ -1,7 +1,7 @@ #include #include -#include "executor.h" +#include "executor/executor.h" class TokenCursor { public: @@ -52,7 +52,7 @@ class TokenCursor { case Tokens::LessOrEqual: case Tokens::Greater: case Tokens::GreaterOrEqual: - case Tokens::EOI: + case Tokens::EndOfInput: throw Error::InvalidArgument("query_parser", std::string(message)); default: break; @@ -169,7 +169,7 @@ ParsedQuery ParseQuery(const std::string_view query) { } cursor.TryConsume(Tokens::Semicolon); - cursor.Consume(Tokens::EOI, "unexpected trailing tokens"); + cursor.Consume(Tokens::EndOfInput, "unexpected trailing tokens"); return parsed; } diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index 80a245e..e2b3ef1 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -1,8 +1,8 @@ #include -#include "ascii.h" -#include "columnar_batch_io.h" -#include "executor.h" +#include "support/ascii.h" +#include "io/columnar_batch_io.h" +#include "executor/executor.h" static size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { const std::string needle = ToLowerAscii(column_name); diff --git a/src/io/batch_io.cpp b/src/io/batch_io.cpp index f66042a..06ba418 100644 --- a/src/io/batch_io.cpp +++ b/src/io/batch_io.cpp @@ -2,14 +2,14 @@ #include #include -#include "columnar_batch_io.h" -#include "csv.h" -#include "csv_batch_io.h" -#include "error.h" -#include "metadata.h" -#include "stream.h" +#include "io/columnar_batch_io.h" +#include "io/csv.h" +#include "io/csv_batch_io.h" +#include "io/stream.h" +#include "model/metadata.h" +#include "support/error.h" -static constexpr std::string_view kColumnarMagic = "CLMN"; +static constexpr std::string_view ColumnarMagic = "CLMN"; static void SeekRead(const std::filesystem::path& path, std::ifstream& in, const uint64_t pos) { in.seekg(pos); @@ -211,28 +211,28 @@ ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::pa } const uint64_t file_size = file_metadata->size; - constexpr uint64_t kFooterSize = sizeof(uint64_t) + kColumnarMagic.size(); + constexpr uint64_t FooterSize = sizeof(uint64_t) + ColumnarMagic.size(); - if (file_size < kFooterSize) { + if (file_size < FooterSize) { throw Error::InvalidData("batch_io", "columnar file is too small", path.string()); } std::ifstream in = OpenInputFile(path); - SeekRead(path, in, file_size - kFooterSize); + SeekRead(path, in, file_size - FooterSize); const uint64_t metadata_size = ReadStream(in); - std::string magic_read(kColumnarMagic.size(), '\0'); + std::string magic_read(ColumnarMagic.size(), '\0'); ReadBytes(in, magic_read.data(), magic_read.size()); - if (magic_read != kColumnarMagic) { + if (magic_read != ColumnarMagic) { throw Error::InvalidData("batch_io", "invalid columnar magic", path.string()); } - if (metadata_size > file_size - kFooterSize) { + if (metadata_size > file_size - FooterSize) { throw Error::InvalidData("batch_io", "metadata size exceeds file size", path.string()); } - SeekRead(path, in, file_size - kFooterSize - metadata_size); + SeekRead(path, in, file_size - FooterSize - metadata_size); return ReadMetadata(in); } @@ -310,7 +310,9 @@ void ColumnarBatchWriter::Write(const Batch& batch) { void ColumnarBatchWriter::Flush() { FlushWrite(path_, out_); } -void ColumnarBatchWriter::Finalize() { +void ColumnarBatchWriter::Finalize() && { FinalizeImpl(); } + +void ColumnarBatchWriter::FinalizeImpl() { if (finalized_) { throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); } @@ -321,7 +323,7 @@ void ColumnarBatchWriter::Finalize() { const uint64_t metadata_size = metadata_end - metadata_start; WriteStream(out_, metadata_size); - WriteBytes(out_, kColumnarMagic); + WriteBytes(out_, ColumnarMagic); FlushWrite(path_, out_); finalized_ = true; diff --git a/src/io/csv.cpp b/src/io/csv.cpp index 408d572..0291c4a 100644 --- a/src/io/csv.cpp +++ b/src/io/csv.cpp @@ -1,9 +1,9 @@ -#include "csv.h" +#include "io/csv.h" #include -#include "error.h" -#include "fileio.h" +#include "io/fileio.h" +#include "support/error.h" static bool NeedsCsvQuotes(const std::string_view value) { return value.find_first_of(",\"\n\r") != std::string_view::npos; diff --git a/src/io/fileio.cpp b/src/io/fileio.cpp index a65ac38..3c58721 100644 --- a/src/io/fileio.cpp +++ b/src/io/fileio.cpp @@ -1,8 +1,8 @@ -#include "fileio.h" +#include "io/fileio.h" #include -#include "error.h" +#include "support/error.h" std::optional TryGetFileMetadata(const std::filesystem::path& path) { std::error_code ec; diff --git a/src/io/stream.cpp b/src/io/stream.cpp index 5193bfb..41d85f2 100644 --- a/src/io/stream.cpp +++ b/src/io/stream.cpp @@ -1,6 +1,6 @@ -#include "stream.h" +#include "io/stream.h" -#include "error.h" +#include "support/error.h" void ReadBytes(std::istream& in, char* dst, const size_t size) { if (size == 0) { diff --git a/src/model/batch.cpp b/src/model/batch.cpp index bad890d..09fe5ca 100644 --- a/src/model/batch.cpp +++ b/src/model/batch.cpp @@ -1,8 +1,8 @@ -#include "batch.h" +#include "model/batch.h" #include -#include "error.h" +#include "support/error.h" static std::vector> CloneColumns(const std::vector>& columns) { std::vector> clones; diff --git a/src/model/column.cpp b/src/model/column.cpp index 7eded02..ab11742 100644 --- a/src/model/column.cpp +++ b/src/model/column.cpp @@ -1,17 +1,17 @@ -#include "column.h" +#include "model/column.h" #include -#include "column_boolean.h" -#include "column_character.h" -#include "column_date.h" -#include "column_int128.h" -#include "column_int16.h" -#include "column_int32.h" -#include "column_int64.h" -#include "column_string.h" -#include "column_timestamp.h" -#include "error.h" +#include "model/column_boolean.h" +#include "model/column_character.h" +#include "model/column_date.h" +#include "model/column_int128.h" +#include "model/column_int16.h" +#include "model/column_int32.h" +#include "model/column_int64.h" +#include "model/column_string.h" +#include "model/column_timestamp.h" +#include "support/error.h" std::unique_ptr CreateColumn(const ColumnType type) { switch (type) { diff --git a/src/model/column_boolean.cpp b/src/model/column_boolean.cpp index 4959ec5..c58ee59 100644 --- a/src/model/column_boolean.cpp +++ b/src/model/column_boolean.cpp @@ -1,6 +1,6 @@ -#include "column_boolean.h" +#include "model/column_boolean.h" -#include "parsing.h" +#include "support/parsing.h" void BooleanColumn::AppendFromString(const std::string& value) { AppendValue(ParseBoolean(value) ? 1 : 0); } diff --git a/src/model/column_character.cpp b/src/model/column_character.cpp index 5184b18..e7312e4 100644 --- a/src/model/column_character.cpp +++ b/src/model/column_character.cpp @@ -1,6 +1,6 @@ -#include "column_character.h" +#include "model/column_character.h" -#include "parsing.h" +#include "support/parsing.h" void CharacterColumn::AppendFromString(const std::string& value) { AppendValue(ParseCharacter(value)); } diff --git a/src/model/column_date.cpp b/src/model/column_date.cpp index 4bfcc9e..e9cc250 100644 --- a/src/model/column_date.cpp +++ b/src/model/column_date.cpp @@ -1,6 +1,6 @@ -#include "column_date.h" +#include "model/column_date.h" -#include "parsing.h" +#include "support/parsing.h" void DateColumn::AppendFromString(const std::string& value) { AppendValue(ParseDate(value)); } diff --git a/src/model/column_int128.cpp b/src/model/column_int128.cpp index 161ab39..2102e6f 100644 --- a/src/model/column_int128.cpp +++ b/src/model/column_int128.cpp @@ -1,6 +1,6 @@ -#include "column_int128.h" +#include "model/column_int128.h" -#include "parsing.h" +#include "support/parsing.h" void Int128Column::AppendFromString(const std::string& value) { AppendValue(ParseInt128(value)); } diff --git a/src/model/column_int16.cpp b/src/model/column_int16.cpp index 94415e8..95c6cd5 100644 --- a/src/model/column_int16.cpp +++ b/src/model/column_int16.cpp @@ -1,6 +1,6 @@ -#include "column_int16.h" +#include "model/column_int16.h" -#include "parsing.h" +#include "support/parsing.h" void Int16Column::AppendFromString(const std::string& value) { AppendValue(ParseInt16(value)); } diff --git a/src/model/column_int32.cpp b/src/model/column_int32.cpp index b9353b9..d0d3639 100644 --- a/src/model/column_int32.cpp +++ b/src/model/column_int32.cpp @@ -1,6 +1,6 @@ -#include "column_int32.h" +#include "model/column_int32.h" -#include "parsing.h" +#include "support/parsing.h" void Int32Column::AppendFromString(const std::string& value) { AppendValue(ParseInt32(value)); } diff --git a/src/model/column_int64.cpp b/src/model/column_int64.cpp index f6c2399..df9be61 100644 --- a/src/model/column_int64.cpp +++ b/src/model/column_int64.cpp @@ -1,6 +1,6 @@ -#include "column_int64.h" +#include "model/column_int64.h" -#include "parsing.h" +#include "support/parsing.h" void Int64Column::AppendFromString(const std::string& value) { AppendValue(ParseInt64(value)); } diff --git a/src/model/column_string.cpp b/src/model/column_string.cpp index 31cea45..1a94bc2 100644 --- a/src/model/column_string.cpp +++ b/src/model/column_string.cpp @@ -1,4 +1,4 @@ -#include "column_string.h" +#include "model/column_string.h" void StringColumn::AppendFromString(const std::string& value) { AppendValue(value); } diff --git a/src/model/column_timestamp.cpp b/src/model/column_timestamp.cpp index 42867fe..c69a98a 100644 --- a/src/model/column_timestamp.cpp +++ b/src/model/column_timestamp.cpp @@ -1,6 +1,6 @@ -#include "column_timestamp.h" +#include "model/column_timestamp.h" -#include "parsing.h" +#include "support/parsing.h" void TimestampColumn::AppendFromString(const std::string& value) { AppendValue(ParseTimestamp(value)); } diff --git a/src/model/metadata.cpp b/src/model/metadata.cpp index 33736bc..17e1209 100644 --- a/src/model/metadata.cpp +++ b/src/model/metadata.cpp @@ -1,10 +1,10 @@ -#include "metadata.h" +#include "model/metadata.h" #include #include -#include "error.h" -#include "stream.h" +#include "support/error.h" +#include "io/stream.h" static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { switch (static_cast(type_byte)) { diff --git a/src/model/schema.cpp b/src/model/schema.cpp index b1d37cb..c5344f1 100644 --- a/src/model/schema.cpp +++ b/src/model/schema.cpp @@ -1,11 +1,11 @@ -#include "schema.h" +#include "model/schema.h" #include -#include "csv.h" -#include "error.h" -#include "fileio.h" -#include "parsing.h" +#include "io/csv.h" +#include "support/error.h" +#include "io/fileio.h" +#include "support/parsing.h" static bool CanParseAs(const ColumnType type, const std::string& value) { try { diff --git a/src/sql_parser/tokenizer.cpp b/src/sql_parser/tokenizer.cpp index 04c5038..c8980b1 100644 --- a/src/sql_parser/tokenizer.cpp +++ b/src/sql_parser/tokenizer.cpp @@ -1,4 +1,4 @@ -#include "tokenizer.h" +#include "sql_parser/tokenizer.h" #include #include @@ -24,7 +24,7 @@ Expected Tokenizer::GetNext() { } if (pos_ >= query_.size()) { - return MakeToken(Tokens::EOI, {}, query_.size()); + return MakeToken(Tokens::EndOfInput, {}, query_.size()); } const char ch = query_[pos_]; @@ -165,7 +165,7 @@ Expected> TokenizeSql(const std::string_view input) { } tokens.push_back(token.value()); - if (token.value()->GetType() == Tokens::EOI) { + if (token.value()->GetType() == Tokens::EndOfInput) { break; } } diff --git a/src/sql_parser/tokenizer_factory.cpp b/src/sql_parser/tokenizer_factory.cpp index 7ee46eb..dd99921 100644 --- a/src/sql_parser/tokenizer_factory.cpp +++ b/src/sql_parser/tokenizer_factory.cpp @@ -4,8 +4,8 @@ #include #include -#include "ascii.h" -#include "tokenizer.h" +#include "support/ascii.h" +#include "sql_parser/tokenizer.h" static const std::unordered_map kKeywords = { {"AS", Tokens::As}, {"AND", Tokens::And}, {"AVG", Tokens::Avg}, {"BY", Tokens::By}, @@ -113,7 +113,7 @@ TokenPtr MakeToken(const Tokens type, std::string text, const size_t offset) { return std::make_shared(std::move(text), offset); case Tokens::GreaterOrEqual: return std::make_shared(std::move(text), offset); - case Tokens::EOI: + case Tokens::EndOfInput: return std::make_shared(std::move(text), offset); } return nullptr; diff --git a/src/sql_parser/tokenizer_tokens.cpp b/src/sql_parser/tokenizer_tokens.cpp index b1bc6b5..0d07844 100644 --- a/src/sql_parser/tokenizer_tokens.cpp +++ b/src/sql_parser/tokenizer_tokens.cpp @@ -1,6 +1,6 @@ #include -#include "tokenizer.h" +#include "sql_parser/tokenizer.h" std::string_view TokenTypeToString(const Tokens type) noexcept { switch (type) { @@ -72,7 +72,7 @@ std::string_view TokenTypeToString(const Tokens type) noexcept { return "greater"; case Tokens::GreaterOrEqual: return "greater_or_equal"; - case Tokens::EOI: + case Tokens::EndOfInput: return "end_of_input"; } return "unknown"; @@ -126,7 +126,7 @@ COLUMNAR_DEFINE_TOKEN_TYPE(TLessToken, Less) COLUMNAR_DEFINE_TOKEN_TYPE(TLessOrEqualToken, LessOrEqual) COLUMNAR_DEFINE_TOKEN_TYPE(TGreaterToken, Greater) COLUMNAR_DEFINE_TOKEN_TYPE(TGreaterOrEqualToken, GreaterOrEqual) -COLUMNAR_DEFINE_TOKEN_TYPE(TEndOfInputToken, EOI) +COLUMNAR_DEFINE_TOKEN_TYPE(TEndOfInputToken, EndOfInput) #undef COLUMNAR_DEFINE_TOKEN_TYPE diff --git a/src/support/ascii.cpp b/src/support/ascii.cpp index 19246ec..779323f 100644 --- a/src/support/ascii.cpp +++ b/src/support/ascii.cpp @@ -1,4 +1,4 @@ -#include "ascii.h" +#include "support/ascii.h" #include diff --git a/src/support/error.cpp b/src/support/error.cpp index f374ff2..0867b49 100644 --- a/src/support/error.cpp +++ b/src/support/error.cpp @@ -1,4 +1,4 @@ -#include "error.h" +#include "support/error.h" #include diff --git a/src/support/parsing.cpp b/src/support/parsing.cpp index 2ddda39..5737ef2 100644 --- a/src/support/parsing.cpp +++ b/src/support/parsing.cpp @@ -1,4 +1,4 @@ -#include "parsing.h" +#include "support/parsing.h" #include #include @@ -6,8 +6,8 @@ #include #include -#include "ascii.h" -#include "error.h" +#include "support/ascii.h" +#include "support/error.h" template T ParseInteger(const std::string& value, const std::string_view type_name) { diff --git a/tests/test_batch.cpp b/tests/test_batch.cpp index 90ddefa..cfbfb5a 100644 --- a/tests/test_batch.cpp +++ b/tests/test_batch.cpp @@ -1,19 +1,51 @@ #include #include +#include #include -#include "columnar_batch_io.h" -#include "csv.h" -#include "csv_batch_io.h" -#include "error.h" +#include "io/columnar_batch_io.h" +#include "io/csv.h" +#include "io/csv_batch_io.h" +#include "support/error.h" #include "gtest/gtest.h" -#include "temp_file.h" +#include "testing/temp_file.h" static_assert(std::is_copy_constructible_v); static_assert(std::is_copy_assignable_v); static_assert(std::is_move_constructible_v); static_assert(std::is_move_assignable_v); +TEST(batch, usage_example_csv_to_columnar_roundtrip) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + const TempFile csv_file("batch_usage_csv"); + const TempFile columnar_file("batch_usage_columnar"); + WriteRows(csv_file.Path(), { + {"1", "alpha"}, + {"2", "beta"}, + }); + + CsvBatchReader csv_reader(csv_file.Path(), schema, {}); + ColumnarBatchWriter columnar_writer(columnar_file.Path(), schema); + + while (auto batch = csv_reader.ReadNext()) { + columnar_writer.Write(*batch); + } + std::move(columnar_writer).Finalize(); + + ColumnarBatchReader columnar_reader(columnar_file.Path()); + auto batch = columnar_reader.ReadNext(); + ASSERT_TRUE(batch.has_value()); + EXPECT_EQ(batch->RowsCount(), 2u); + EXPECT_EQ(batch->ColumnAt(0).ValueAsString(0), "1"); + EXPECT_EQ(batch->ColumnAt(1).ValueAsString(1), "beta"); + EXPECT_FALSE(columnar_reader.ReadNext().has_value()); +} + TEST(batch, csv_reader_respects_max_rows) { Schema schema; schema.columns = { @@ -97,7 +129,7 @@ TEST(batch, columnar_roundtrip) { while (auto batch = csv_reader.ReadNext()) { columnar_writer.Write(*batch); } - columnar_writer.Finalize(); + std::move(columnar_writer).Finalize(); ColumnarBatchReader columnar_reader(columnar_file.Path()); EXPECT_EQ(columnar_reader.GetMetadata().row_groups.size(), 2u); @@ -237,9 +269,9 @@ TEST(batch, columnar_writer_rejects_after_finalize) { const TempFile columnar_file("batch_finalize_columnar"); ColumnarBatchWriter writer(columnar_file.Path(), schema); - writer.Finalize(); + std::move(writer).Finalize(); Batch batch(schema); EXPECT_THROW(writer.Write(batch), Error); - EXPECT_THROW(writer.Finalize(), Error); + EXPECT_THROW(std::move(writer).Finalize(), Error); } diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp index d0a2f87..f6051f5 100644 --- a/tests/test_columnar.cpp +++ b/tests/test_columnar.cpp @@ -2,15 +2,15 @@ #include #include -#include "columnar_batch_io.h" -#include "csv_columnar.h" -#include "csv.h" -#include "error.h" -#include "fileio.h" +#include "io/columnar_batch_io.h" +#include "convert/csv_columnar.h" +#include "io/csv.h" +#include "support/error.h" +#include "io/fileio.h" #include "gtest/gtest.h" -#include "parsing.h" -#include "schema.h" -#include "temp_file.h" +#include "support/parsing.h" +#include "model/schema.h" +#include "testing/temp_file.h" TEST(columnar, csv_to_columnar_and_back) { const TempFile schema_in("schema_in"); diff --git a/tests/test_columns.cpp b/tests/test_columns.cpp index b83e667..8c32a40 100644 --- a/tests/test_columns.cpp +++ b/tests/test_columns.cpp @@ -2,12 +2,12 @@ #include #include -#include "column.h" -#include "column_int64.h" -#include "column_string.h" -#include "error.h" +#include "model/column.h" +#include "model/column_int64.h" +#include "model/column_string.h" +#include "support/error.h" #include "gtest/gtest.h" -#include "parsing.h" +#include "support/parsing.h" static void ExpectColumnRoundtrip(const ColumnType type, const std::vector& values, const std::vector& expected_values = {}) { diff --git a/tests/test_csv.cpp b/tests/test_csv.cpp index 7e9c4c6..c892da5 100644 --- a/tests/test_csv.cpp +++ b/tests/test_csv.cpp @@ -3,7 +3,7 @@ #include #include -#include "csv.h" +#include "io/csv.h" #include "gtest/gtest.h" static_assert(!std::is_copy_constructible_v); diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index f187705..567ecd6 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -1,12 +1,12 @@ #include #include -#include "csv.h" -#include "csv_columnar.h" -#include "executor.h" -#include "executor_test_utils.h" +#include "io/csv.h" +#include "convert/csv_columnar.h" +#include "executor/executor.h" +#include "testing/executor_test_utils.h" #include "gtest/gtest.h" -#include "temp_file.h" +#include "testing/temp_file.h" static Batch BuildHitsTable(const std::string_view query) { const TempFile schema_file("executor_schema"); @@ -42,13 +42,13 @@ static Batch BuildHitsTable(const std::string_view query) { return std::move(result.value()); } -TEST(executor, executes_basic_aggregate_queries) { - { - const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); - EXPECT_EQ(SingleRowValues(batch), std::vector{"5"}); - } +TEST(executor, usage_example_register_table_and_execute_query) { + const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"5"}); +} +TEST(executor, executes_basic_aggregate_queries) { { const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); diff --git a/tests/test_fileio.cpp b/tests/test_fileio.cpp index c919fcd..5427f41 100644 --- a/tests/test_fileio.cpp +++ b/tests/test_fileio.cpp @@ -3,10 +3,10 @@ #include #include -#include "error.h" -#include "fileio.h" +#include "support/error.h" +#include "io/fileio.h" #include "gtest/gtest.h" -#include "temp_file.h" +#include "testing/temp_file.h" TEST(fileio, write_and_read_roundtrip) { const TempFile temp("fileio_roundtrip"); diff --git a/tests/test_stream.cpp b/tests/test_stream.cpp index a3617c8..89a7d80 100644 --- a/tests/test_stream.cpp +++ b/tests/test_stream.cpp @@ -3,7 +3,7 @@ #include #include "gtest/gtest.h" -#include "stream.h" +#include "io/stream.h" TEST(stream, roundtrip_signed_integrals) { std::stringstream buffer; diff --git a/tests/test_tokenizer.cpp b/tests/test_tokenizer.cpp index ed9f019..ca11a74 100644 --- a/tests/test_tokenizer.cpp +++ b/tests/test_tokenizer.cpp @@ -3,7 +3,7 @@ #include #include "gtest/gtest.h" -#include "tokenizer.h" +#include "sql_parser/tokenizer.h" static std::vector CollectTypes(const std::vector& tokens) { std::vector types; @@ -30,7 +30,7 @@ TEST(sql_tokenizer, tokenizes_select_query) { Tokens::OpenBracket, Tokens::Asterisk, Tokens::CloseBracket, Tokens::Comma, Tokens::Length, Tokens::OpenBracket, Tokens::NameToken, Tokens::CloseBracket, Tokens::From, Tokens::NameToken, Tokens::Group, Tokens::By, Tokens::NameToken, Tokens::Order, Tokens::By, - Tokens::NameToken, Tokens::Limit, Tokens::NumericLiteral, Tokens::Semicolon, Tokens::EOI, + Tokens::NameToken, Tokens::Limit, Tokens::NumericLiteral, Tokens::Semicolon, Tokens::EndOfInput, })); ASSERT_EQ(tokens.size(), 25U); @@ -63,7 +63,7 @@ TEST(sql_tokenizer, tokenizes_create_with_literals_and_case_insensitive_keywords Tokens::NumericLiteral, Tokens::From, Tokens::NameToken, - Tokens::EOI, + Tokens::EndOfInput, })); ASSERT_EQ(tokens.size(), 12U); @@ -84,7 +84,7 @@ TEST(sql_tokenizer, tokenizer_returns_tokens_one_by_one) { ASSERT_TRUE(token.has_value()); types.push_back(token.value()->GetType()); - if (token.value()->GetType() == Tokens::EOI) { + if (token.value()->GetType() == Tokens::EndOfInput) { break; } } @@ -102,7 +102,7 @@ TEST(sql_tokenizer, tokenizer_returns_tokens_one_by_one) { Tokens::NameToken, Tokens::NotEqual, Tokens::NumericLiteral, - Tokens::EOI, + Tokens::EndOfInput, })); } From 2ff40ecdf8270e57e9bd2dd9485a19ca87ce00a3 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Mon, 4 May 2026 23:39:59 +0300 Subject: [PATCH 06/29] feat: all 43 queries --- benchmarks/queries/queries.sql | 43 +++++++++++++++++++++++++++++++++ benchmarks/queries/query_10.sql | 1 + benchmarks/queries/query_11.sql | 1 + benchmarks/queries/query_12.sql | 1 + benchmarks/queries/query_13.sql | 1 + benchmarks/queries/query_14.sql | 1 + benchmarks/queries/query_15.sql | 1 + benchmarks/queries/query_16.sql | 1 + benchmarks/queries/query_17.sql | 1 + benchmarks/queries/query_18.sql | 1 + benchmarks/queries/query_19.sql | 1 + benchmarks/queries/query_20.sql | 1 + benchmarks/queries/query_21.sql | 1 + benchmarks/queries/query_22.sql | 1 + benchmarks/queries/query_23.sql | 1 + benchmarks/queries/query_24.sql | 1 + benchmarks/queries/query_25.sql | 1 + benchmarks/queries/query_26.sql | 1 + benchmarks/queries/query_27.sql | 1 + benchmarks/queries/query_28.sql | 1 + benchmarks/queries/query_29.sql | 1 + benchmarks/queries/query_30.sql | 1 + benchmarks/queries/query_31.sql | 1 + benchmarks/queries/query_32.sql | 1 + benchmarks/queries/query_33.sql | 1 + benchmarks/queries/query_34.sql | 1 + benchmarks/queries/query_35.sql | 1 + benchmarks/queries/query_36.sql | 1 + benchmarks/queries/query_37.sql | 1 + benchmarks/queries/query_38.sql | 1 + benchmarks/queries/query_39.sql | 1 + benchmarks/queries/query_40.sql | 1 + benchmarks/queries/query_41.sql | 1 + benchmarks/queries/query_42.sql | 1 + benchmarks/queries/query_5.sql | 1 + benchmarks/queries/query_6.sql | 1 + benchmarks/queries/query_7.sql | 1 + benchmarks/queries/query_8.sql | 1 + benchmarks/queries/query_9.sql | 1 + 39 files changed, 81 insertions(+) create mode 100644 benchmarks/queries/queries.sql create mode 100644 benchmarks/queries/query_10.sql create mode 100644 benchmarks/queries/query_11.sql create mode 100644 benchmarks/queries/query_12.sql create mode 100644 benchmarks/queries/query_13.sql create mode 100644 benchmarks/queries/query_14.sql create mode 100644 benchmarks/queries/query_15.sql create mode 100644 benchmarks/queries/query_16.sql create mode 100644 benchmarks/queries/query_17.sql create mode 100644 benchmarks/queries/query_18.sql create mode 100644 benchmarks/queries/query_19.sql create mode 100644 benchmarks/queries/query_20.sql create mode 100644 benchmarks/queries/query_21.sql create mode 100644 benchmarks/queries/query_22.sql create mode 100644 benchmarks/queries/query_23.sql create mode 100644 benchmarks/queries/query_24.sql create mode 100644 benchmarks/queries/query_25.sql create mode 100644 benchmarks/queries/query_26.sql create mode 100644 benchmarks/queries/query_27.sql create mode 100644 benchmarks/queries/query_28.sql create mode 100644 benchmarks/queries/query_29.sql create mode 100644 benchmarks/queries/query_30.sql create mode 100644 benchmarks/queries/query_31.sql create mode 100644 benchmarks/queries/query_32.sql create mode 100644 benchmarks/queries/query_33.sql create mode 100644 benchmarks/queries/query_34.sql create mode 100644 benchmarks/queries/query_35.sql create mode 100644 benchmarks/queries/query_36.sql create mode 100644 benchmarks/queries/query_37.sql create mode 100644 benchmarks/queries/query_38.sql create mode 100644 benchmarks/queries/query_39.sql create mode 100644 benchmarks/queries/query_40.sql create mode 100644 benchmarks/queries/query_41.sql create mode 100644 benchmarks/queries/query_42.sql create mode 100644 benchmarks/queries/query_5.sql create mode 100644 benchmarks/queries/query_6.sql create mode 100644 benchmarks/queries/query_7.sql create mode 100644 benchmarks/queries/query_8.sql create mode 100644 benchmarks/queries/query_9.sql diff --git a/benchmarks/queries/queries.sql b/benchmarks/queries/queries.sql new file mode 100644 index 0000000..49dd47c --- /dev/null +++ b/benchmarks/queries/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; +SELECT AVG(UserID) FROM hits; +SELECT COUNT(DISTINCT UserID) FROM hits; +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; +SELECT MIN(EventDate), MAX(EventDate) FROM hits; +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID FROM hits WHERE UserID = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; +SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; \ No newline at end of file diff --git a/benchmarks/queries/query_10.sql b/benchmarks/queries/query_10.sql new file mode 100644 index 0000000..d0ea7e3 --- /dev/null +++ b/benchmarks/queries/query_10.sql @@ -0,0 +1 @@ +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_11.sql b/benchmarks/queries/query_11.sql new file mode 100644 index 0000000..2a316d1 --- /dev/null +++ b/benchmarks/queries/query_11.sql @@ -0,0 +1 @@ +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_12.sql b/benchmarks/queries/query_12.sql new file mode 100644 index 0000000..7a02546 --- /dev/null +++ b/benchmarks/queries/query_12.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_13.sql b/benchmarks/queries/query_13.sql new file mode 100644 index 0000000..4ce0fee --- /dev/null +++ b/benchmarks/queries/query_13.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_14.sql b/benchmarks/queries/query_14.sql new file mode 100644 index 0000000..8e85255 --- /dev/null +++ b/benchmarks/queries/query_14.sql @@ -0,0 +1 @@ +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_15.sql b/benchmarks/queries/query_15.sql new file mode 100644 index 0000000..f959e98 --- /dev/null +++ b/benchmarks/queries/query_15.sql @@ -0,0 +1 @@ +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/benchmarks/queries/query_16.sql b/benchmarks/queries/query_16.sql new file mode 100644 index 0000000..50b1f38 --- /dev/null +++ b/benchmarks/queries/query_16.sql @@ -0,0 +1 @@ +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/benchmarks/queries/query_17.sql b/benchmarks/queries/query_17.sql new file mode 100644 index 0000000..454cdb5 --- /dev/null +++ b/benchmarks/queries/query_17.sql @@ -0,0 +1 @@ +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; diff --git a/benchmarks/queries/query_18.sql b/benchmarks/queries/query_18.sql new file mode 100644 index 0000000..6520cc5 --- /dev/null +++ b/benchmarks/queries/query_18.sql @@ -0,0 +1 @@ +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/benchmarks/queries/query_19.sql b/benchmarks/queries/query_19.sql new file mode 100644 index 0000000..3f84066 --- /dev/null +++ b/benchmarks/queries/query_19.sql @@ -0,0 +1 @@ +SELECT UserID FROM hits WHERE UserID = 435090932899640449; diff --git a/benchmarks/queries/query_20.sql b/benchmarks/queries/query_20.sql new file mode 100644 index 0000000..4426afa --- /dev/null +++ b/benchmarks/queries/query_20.sql @@ -0,0 +1 @@ +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; diff --git a/benchmarks/queries/query_21.sql b/benchmarks/queries/query_21.sql new file mode 100644 index 0000000..bd28609 --- /dev/null +++ b/benchmarks/queries/query_21.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_22.sql b/benchmarks/queries/query_22.sql new file mode 100644 index 0000000..4ee87ac --- /dev/null +++ b/benchmarks/queries/query_22.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_23.sql b/benchmarks/queries/query_23.sql new file mode 100644 index 0000000..935169e --- /dev/null +++ b/benchmarks/queries/query_23.sql @@ -0,0 +1 @@ +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; diff --git a/benchmarks/queries/query_24.sql b/benchmarks/queries/query_24.sql new file mode 100644 index 0000000..1bcfd4c --- /dev/null +++ b/benchmarks/queries/query_24.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; diff --git a/benchmarks/queries/query_25.sql b/benchmarks/queries/query_25.sql new file mode 100644 index 0000000..58ea761 --- /dev/null +++ b/benchmarks/queries/query_25.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; diff --git a/benchmarks/queries/query_26.sql b/benchmarks/queries/query_26.sql new file mode 100644 index 0000000..88ed7ba --- /dev/null +++ b/benchmarks/queries/query_26.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; diff --git a/benchmarks/queries/query_27.sql b/benchmarks/queries/query_27.sql new file mode 100644 index 0000000..8f89977 --- /dev/null +++ b/benchmarks/queries/query_27.sql @@ -0,0 +1 @@ +SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/benchmarks/queries/query_28.sql b/benchmarks/queries/query_28.sql new file mode 100644 index 0000000..f55fac1 --- /dev/null +++ b/benchmarks/queries/query_28.sql @@ -0,0 +1 @@ +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/benchmarks/queries/query_29.sql b/benchmarks/queries/query_29.sql new file mode 100644 index 0000000..630450b --- /dev/null +++ b/benchmarks/queries/query_29.sql @@ -0,0 +1 @@ +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; diff --git a/benchmarks/queries/query_30.sql b/benchmarks/queries/query_30.sql new file mode 100644 index 0000000..f8a80d9 --- /dev/null +++ b/benchmarks/queries/query_30.sql @@ -0,0 +1 @@ +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_31.sql b/benchmarks/queries/query_31.sql new file mode 100644 index 0000000..ba8ac2f --- /dev/null +++ b/benchmarks/queries/query_31.sql @@ -0,0 +1 @@ +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_32.sql b/benchmarks/queries/query_32.sql new file mode 100644 index 0000000..893773a --- /dev/null +++ b/benchmarks/queries/query_32.sql @@ -0,0 +1 @@ +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_33.sql b/benchmarks/queries/query_33.sql new file mode 100644 index 0000000..f00a3e8 --- /dev/null +++ b/benchmarks/queries/query_33.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_34.sql b/benchmarks/queries/query_34.sql new file mode 100644 index 0000000..2137530 --- /dev/null +++ b/benchmarks/queries/query_34.sql @@ -0,0 +1 @@ +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_35.sql b/benchmarks/queries/query_35.sql new file mode 100644 index 0000000..581e5e3 --- /dev/null +++ b/benchmarks/queries/query_35.sql @@ -0,0 +1 @@ +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_36.sql b/benchmarks/queries/query_36.sql new file mode 100644 index 0000000..d6e06ce --- /dev/null +++ b/benchmarks/queries/query_36.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; diff --git a/benchmarks/queries/query_37.sql b/benchmarks/queries/query_37.sql new file mode 100644 index 0000000..f63f7a8 --- /dev/null +++ b/benchmarks/queries/query_37.sql @@ -0,0 +1 @@ +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; diff --git a/benchmarks/queries/query_38.sql b/benchmarks/queries/query_38.sql new file mode 100644 index 0000000..498db57 --- /dev/null +++ b/benchmarks/queries/query_38.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; diff --git a/benchmarks/queries/query_39.sql b/benchmarks/queries/query_39.sql new file mode 100644 index 0000000..9d3d319 --- /dev/null +++ b/benchmarks/queries/query_39.sql @@ -0,0 +1 @@ +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; diff --git a/benchmarks/queries/query_40.sql b/benchmarks/queries/query_40.sql new file mode 100644 index 0000000..f54f6db --- /dev/null +++ b/benchmarks/queries/query_40.sql @@ -0,0 +1 @@ +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; diff --git a/benchmarks/queries/query_41.sql b/benchmarks/queries/query_41.sql new file mode 100644 index 0000000..dc1b7e6 --- /dev/null +++ b/benchmarks/queries/query_41.sql @@ -0,0 +1 @@ +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; diff --git a/benchmarks/queries/query_42.sql b/benchmarks/queries/query_42.sql new file mode 100644 index 0000000..4ac2815 --- /dev/null +++ b/benchmarks/queries/query_42.sql @@ -0,0 +1 @@ +SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; diff --git a/benchmarks/queries/query_5.sql b/benchmarks/queries/query_5.sql new file mode 100644 index 0000000..e9615f8 --- /dev/null +++ b/benchmarks/queries/query_5.sql @@ -0,0 +1 @@ +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; diff --git a/benchmarks/queries/query_6.sql b/benchmarks/queries/query_6.sql new file mode 100644 index 0000000..03fbb82 --- /dev/null +++ b/benchmarks/queries/query_6.sql @@ -0,0 +1 @@ +SELECT MIN(EventDate), MAX(EventDate) FROM hits; diff --git a/benchmarks/queries/query_7.sql b/benchmarks/queries/query_7.sql new file mode 100644 index 0000000..25a10b1 --- /dev/null +++ b/benchmarks/queries/query_7.sql @@ -0,0 +1 @@ +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; diff --git a/benchmarks/queries/query_8.sql b/benchmarks/queries/query_8.sql new file mode 100644 index 0000000..a6bf47e --- /dev/null +++ b/benchmarks/queries/query_8.sql @@ -0,0 +1 @@ +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_9.sql b/benchmarks/queries/query_9.sql new file mode 100644 index 0000000..f4a9ee3 --- /dev/null +++ b/benchmarks/queries/query_9.sql @@ -0,0 +1 @@ +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; From 7e32fedc5005f242d6c1868c16d60ae08168c03f Mon Sep 17 00:00:00 2001 From: ik4rd Date: Mon, 4 May 2026 23:46:44 +0300 Subject: [PATCH 07/29] feat: 7/10 queries done --- benchmarks/queries/query_5.csv | 1 + benchmarks/queries/query_6.csv | 1 + benchmarks/queries/query_7.csv | 5 +++++ benchmarks/queries/query_8.csv | 10 ++++++++++ benchmarks/queries/query_9.csv | 10 ++++++++++ 5 files changed, 27 insertions(+) create mode 100644 benchmarks/queries/query_5.csv create mode 100644 benchmarks/queries/query_6.csv create mode 100644 benchmarks/queries/query_7.csv create mode 100644 benchmarks/queries/query_8.csv create mode 100644 benchmarks/queries/query_9.csv diff --git a/benchmarks/queries/query_5.csv b/benchmarks/queries/query_5.csv new file mode 100644 index 0000000..d2c24b0 --- /dev/null +++ b/benchmarks/queries/query_5.csv @@ -0,0 +1 @@ +18316 diff --git a/benchmarks/queries/query_6.csv b/benchmarks/queries/query_6.csv new file mode 100644 index 0000000..b72aa99 --- /dev/null +++ b/benchmarks/queries/query_6.csv @@ -0,0 +1 @@ +2013-07-15,2013-07-15 diff --git a/benchmarks/queries/query_7.csv b/benchmarks/queries/query_7.csv new file mode 100644 index 0000000..0e1c4d4 --- /dev/null +++ b/benchmarks/queries/query_7.csv @@ -0,0 +1,5 @@ +2,9543 +13,4592 +52,34 +50,4 +28,1 diff --git a/benchmarks/queries/query_8.csv b/benchmarks/queries/query_8.csv new file mode 100644 index 0000000..5f52dd6 --- /dev/null +++ b/benchmarks/queries/query_8.csv @@ -0,0 +1,10 @@ +229,27961 +2,10413 +208,3073 +1,1720 +34,1428 +158,1110 +184,987 +107,966 +42,956 +47,943 diff --git a/benchmarks/queries/query_9.csv b/benchmarks/queries/query_9.csv new file mode 100644 index 0000000..355f967 --- /dev/null +++ b/benchmarks/queries/query_9.csv @@ -0,0 +1,10 @@ +229,38044,426412,1612,27961 +2,12801,148193,1593,10413 +208,2673,30614,1490,3073 +1,1802,28577,1623,1720 +34,508,14329,1592,1428 +47,1041,13661,1637,943 +158,78,13294,1576,1110 +7,1166,11679,1627,647 +42,642,11547,1625,956 +184,30,10157,1614,987 From 48c15a19a5e895d076f10ed1576b2377a516b394 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Tue, 5 May 2026 00:22:46 +0300 Subject: [PATCH 08/29] feat: +1 query + dashboard --- README.md | 18 +- benchmarks/benchmark_queries.cpp | 7 +- benchmarks/results/readme_benchmarks.csv | 9 + include/executor/executor.h | 43 +++- include/executor/operator_utils.h | 9 + include/io/fileio.h | 1 + include/testing/executor_test_utils.h | 7 + scripts/dashboard.py | 300 +++++++++++++++++++++++ src/CMakeLists.txt | 1 + src/executor/operator_utils.cpp | 83 +++++++ src/executor/operators.cpp | 218 +++++++++++----- src/executor/query_parser.cpp | 80 +++++- src/executor/query_planner.cpp | 97 +++++++- src/io/batch_io.cpp | 13 +- src/io/fileio.cpp | 7 + tests/test_executor.cpp | 16 +- 16 files changed, 821 insertions(+), 88 deletions(-) create mode 100644 benchmarks/results/readme_benchmarks.csv create mode 100644 include/executor/operator_utils.h create mode 100644 scripts/dashboard.py create mode 100644 src/executor/operator_utils.cpp diff --git a/README.md b/README.md index 94cd7d0..ac9007f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,19 @@ # columnar-engine -Колононый движок с нуля (in progress) \ No newline at end of file +Колононый движок с нуля (in progress) + + +## Dashboard + + +| Query | SQL | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | +| --- | --- | ---: | ---: | ---: | ---: | ---: | --- | +| [query_0.sql](benchmarks/queries/query_0.sql) | `SELECT COUNT(*) FROM hits;` | 53.43 | 9.63 | 9.23 | 8.77 | 11.28 | ok | +| [query_1.sql](benchmarks/queries/query_1.sql) | `SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;` | 41.22 | 27.88 | 27.79 | 27.43 | 28.51 | ok | +| [query_2.sql](benchmarks/queries/query_2.sql) | `SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;` | 62.18 | 54.10 | 53.49 | 50.64 | 58.77 | ok | +| [query_3.sql](benchmarks/queries/query_3.sql) | `SELECT AVG(UserID) FROM hits;` | 85.12 | 58.40 | 54.68 | 54.23 | 70.00 | ok | +| [query_4.sql](benchmarks/queries/query_4.sql) | `SELECT COUNT(DISTINCT UserID) FROM hits;` | 46.61 | 47.05 | 47.30 | 43.96 | 49.62 | ok | +| [query_5.sql](benchmarks/queries/query_5.sql) | `SELECT COUNT(DISTINCT SearchPhrase) FROM hits;` | 61.44 | 45.14 | 44.92 | 44.58 | 46.12 | ok | +| [query_6.sql](benchmarks/queries/query_6.sql) | `SELECT MIN(EventDate), MAX(EventDate) FROM hits;` | 685.92 | 665.72 | 665.60 | 665.26 | 666.43 | ok | +| [query_7.sql](benchmarks/queries/query_7.sql) | `SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDE…` | 28.69 | 28.12 | 28.13 | 28.01 | 28.20 | ok | + diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index 227a3f1..71d417d 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -5,8 +5,8 @@ #include #include -#include "io/csv_batch_io.h" #include "executor/executor.h" +#include "io/csv_batch_io.h" #include "io/fileio.h" #include "support/query_runners.h" @@ -15,7 +15,7 @@ #endif static constexpr int FirstQueryId = 0; -static constexpr int LastQueryId = 42; +static constexpr int LastQueryId = 7; std::string BatchToString(const Batch& batch) { std::vector> rows; @@ -105,9 +105,10 @@ int main(const int argc, char** argv) { } const std::filesystem::path queries_dir = COLUMNAR_BENCHMARK_QUERIES_DIR; + const std::filesystem::path data_path = queries_dir.parent_path() / "hits_sample.columnar"; Executor executor; - executor.RegisterTable("hits", "hits_sample.columnar"); + executor.RegisterTable("hits", data_path); size_t found_queries = 0; diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv new file mode 100644 index 0000000..777f677 --- /dev/null +++ b/benchmarks/results/readme_benchmarks.csv @@ -0,0 +1,9 @@ +query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,status,error +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,53.43,9.63,9.23,8.77,11.28,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,41.22,27.88,27.79,27.43,28.51,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",62.18,54.10,53.49,50.64,58.77,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,85.12,58.40,54.68,54.23,70.00,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,46.61,47.05,47.30,43.96,49.62,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,61.44,45.14,44.92,44.58,46.12,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",685.92,665.72,665.60,665.26,666.43,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",28.69,28.12,28.13,28.01,28.20,ok, diff --git a/include/executor/executor.h b/include/executor/executor.h index aa2cd1d..e1a0692 100644 --- a/include/executor/executor.h +++ b/include/executor/executor.h @@ -69,10 +69,29 @@ struct AggSpec { std::string output_name; }; +enum class SelectItemKind { + GroupKey, + Aggregate, +}; + +struct SelectItemSpec { + SelectItemKind kind = SelectItemKind::GroupKey; + std::string column_name; + AggSpec aggregate; + std::string output_name; +}; + +struct OrderBySpec { + SelectItemSpec item; + bool descending = false; +}; + struct ParsedQuery { std::string table_name; - std::vector aggregates; + std::vector select_items; + std::vector group_by; std::optional filter; + std::vector order_by; }; struct PlannedAgg { @@ -84,6 +103,25 @@ struct PlannedAgg { std::string output_name; }; +struct PlannedGroupKey { + size_t column_index = 0; + ColumnType column_type = ColumnType::String; + std::string output_name; +}; + +struct PlannedSelectItem { + SelectItemKind kind = SelectItemKind::GroupKey; + size_t index = 0; + ColumnType output_type = ColumnType::String; + std::string output_name; +}; + +struct PlannedOrderBy { + size_t result_column_index = 0; + bool descending = false; + ColumnType value_type = ColumnType::String; +}; + struct PlannedFilter { size_t column_index = 0; ColumnType column_type = ColumnType::String; @@ -95,7 +133,10 @@ struct PlannedQuery { std::filesystem::path table_path; std::vector projection_indexes; std::optional filter; + std::vector group_keys; std::vector aggregates; + std::vector select_items; + std::vector order_by; }; class AggState { diff --git a/include/executor/operator_utils.h b/include/executor/operator_utils.h new file mode 100644 index 0000000..7a26006 --- /dev/null +++ b/include/executor/operator_utils.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +#include "executor/executor.h" + +int CompareValues(ColumnType type, std::string_view lhs, std::string_view rhs); +bool MatchesComparison(ColumnType type, std::string_view lhs, std::string_view rhs, ComparisonKind kind); +ColumnType AggregateOutputType(const PlannedAgg& aggregate); diff --git a/include/io/fileio.h b/include/io/fileio.h index fd07161..2e62c62 100644 --- a/include/io/fileio.h +++ b/include/io/fileio.h @@ -19,6 +19,7 @@ bool FileExists(const std::filesystem::path& path); void EnsureParentDirectory(const std::filesystem::path& path); std::ifstream OpenInputFile(const std::filesystem::path& path); +void SeekInputFile(std::istream& in, const std::filesystem::path& path, uint64_t offset); enum class FileOpenMode { Truncate, diff --git a/include/testing/executor_test_utils.h b/include/testing/executor_test_utils.h index ee94d4b..bf8c3e3 100644 --- a/include/testing/executor_test_utils.h +++ b/include/testing/executor_test_utils.h @@ -21,6 +21,13 @@ inline std::vector SingleRowValues(const Batch& batch) { return std::move(rows.front()); } +inline std::vector> BatchRows(const Batch& batch) { + std::vector> rows; + rows.reserve(batch.RowsCount()); + AppendBatchRows(batch, rows); + return rows; +} + inline std::vector BatchColumnNames(const Batch& batch) { std::vector names; names.reserve(batch.ColumnsCount()); diff --git a/scripts/dashboard.py b/scripts/dashboard.py new file mode 100644 index 0000000..53e87a8 --- /dev/null +++ b/scripts/dashboard.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import pathlib +import statistics +import subprocess +import sys +import tempfile +import time + +README_START = "" +README_END = "" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run the first N benchmark queries, write CSV stats, and refresh the README benchmark table." + ) + parser.add_argument("--count", type=int, default=8, help="Number of first queries to benchmark.") + parser.add_argument("--warm-runs", type=int, default=4, help="Number of warm runs after the first measured run.") + parser.add_argument( + "--query-dir", + type=pathlib.Path, + default=pathlib.Path("benchmarks/queries"), + help="Directory with query_.sql files.", + ) + parser.add_argument( + "--input", + type=pathlib.Path, + default=pathlib.Path("benchmarks/hits_sample.columnar"), + help="Columnar input file.", + ) + parser.add_argument( + "--binary", + type=pathlib.Path, + default=pathlib.Path("build/src/columnar_engine"), + help="Path to the columnar_engine binary.", + ) + parser.add_argument( + "--csv-out", + type=pathlib.Path, + default=pathlib.Path("benchmarks/results/readme_benchmarks.csv"), + help="Where to write the benchmark CSV.", + ) + parser.add_argument( + "--readme", + type=pathlib.Path, + default=pathlib.Path("README.md"), + help="README file to update.", + ) + parser.add_argument( + "--table-name", + default="hits", + help="Logical table name passed to run-query.", + ) + return parser.parse_args() + + +def repo_root() -> pathlib.Path: + return pathlib.Path(__file__).resolve().parent.parent + + +def resolve_path(path: pathlib.Path, root: pathlib.Path) -> pathlib.Path: + return path if path.is_absolute() else root / path + + +def format_ms(value: float) -> str: + return f"{value:.2f}" + + +def compact_sql(sql: str, max_len: int = 88) -> str: + normalized = " ".join(sql.split()) + if len(normalized) <= max_len: + return normalized + return normalized[: max_len - 1] + "…" + + +def markdown_escape(text: str) -> str: + return text.replace("|", "\\|") + + +def run_query( + binary: pathlib.Path, + input_path: pathlib.Path, + query_path: pathlib.Path, + output_path: pathlib.Path, + table_name: str, +) -> float: + command = [ + str(binary), + "run-query", + "--input", + str(input_path), + "--output", + str(output_path), + "--table-name", + table_name, + "--query-file", + str(query_path), + ] + + started_at = time.perf_counter() + completed = subprocess.run(command, capture_output=True, text=True) + elapsed_ms = (time.perf_counter() - started_at) * 1000.0 + + if completed.returncode != 0: + stderr = completed.stderr.strip() + stdout = completed.stdout.strip() + message = stderr or stdout or f"exit code {completed.returncode}" + raise RuntimeError(message) + + return elapsed_ms + + +def collect_rows( + *, + root: pathlib.Path, + binary: pathlib.Path, + input_path: pathlib.Path, + query_dir: pathlib.Path, + count: int, + warm_runs: int, + table_name: str, +) -> list[dict[str, str]]: + rows: list[dict[str, str]] = [] + + with tempfile.TemporaryDirectory(prefix="readme_benchmarks_") as temp_dir_name: + temp_dir = pathlib.Path(temp_dir_name) + + for query_id in range(count): + query_path = query_dir / f"query_{query_id}.sql" + if not query_path.exists(): + break + + sql = query_path.read_text(encoding="utf-8").strip() + output_path = temp_dir / f"query_{query_id}.csv" + + first_error = "" + status = "ok" + first_run_ms = "" + warm_avg_ms = "" + warm_min_ms = "" + warm_max_ms = "" + warm_median_ms = "" + + try: + first_run_value = run_query(binary, input_path, query_path, output_path, table_name) + first_run_ms = format_ms(first_run_value) + + warm_values: list[float] = [] + for _ in range(warm_runs): + warm_values.append(run_query(binary, input_path, query_path, output_path, table_name)) + + if warm_values: + warm_avg_ms = format_ms(sum(warm_values) / len(warm_values)) + warm_min_ms = format_ms(min(warm_values)) + warm_max_ms = format_ms(max(warm_values)) + warm_median_ms = format_ms(statistics.median(warm_values)) + except RuntimeError as error: + status = "failed" + first_error = str(error) + + rows.append( + { + "query_id": str(query_id), + "query_file": str(query_path.relative_to(root)), + "sql": sql, + "first_run_ms": first_run_ms, + "warm_avg_ms": warm_avg_ms, + "warm_median_ms": warm_median_ms, + "warm_min_ms": warm_min_ms, + "warm_max_ms": warm_max_ms, + "status": status, + "error": first_error, + } + ) + + return rows + + +def write_csv(csv_path: pathlib.Path, rows: list[dict[str, str]]) -> None: + csv_path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = [ + "query_id", + "query_file", + "sql", + "first_run_ms", + "warm_avg_ms", + "warm_median_ms", + "warm_min_ms", + "warm_max_ms", + "status", + "error", + ] + with csv_path.open("w", encoding="utf-8", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def render_markdown(csv_path: pathlib.Path, rows: list[dict[str, str]], count: int, warm_runs: int) -> str: + generated_at = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + lines = [ + "## Dashboard", + "", + "", + "| Query | SQL | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | --- |", + ] + + for row in rows: + query_file = row["query_file"] + query_label = f"[query_{row['query_id']}.sql]({query_file})" + sql = f"`{markdown_escape(compact_sql(row['sql']))}`" + status = row["status"] + if row["error"]: + status = f"{status}: {markdown_escape(compact_sql(row['error'], max_len=48))}" + + lines.append( + "| " + + " | ".join( + [ + query_label, + sql, + row["first_run_ms"] or "—", + row["warm_avg_ms"] or "—", + row["warm_median_ms"] or "—", + row["warm_min_ms"] or "—", + row["warm_max_ms"] or "—", + status, + ] + ) + + " |" + ) + + return "\n".join(lines) + + +def update_readme(readme_path: pathlib.Path, markdown_block: str) -> None: + if readme_path.exists(): + content = readme_path.read_text(encoding="utf-8") + else: + content = "" + + wrapped = f"{README_START}\n{markdown_block}\n{README_END}" + + if README_START in content and README_END in content: + start = content.index(README_START) + end = content.index(README_END) + len(README_END) + updated = content[:start] + wrapped + content[end:] + else: + suffix = "" if not content or content.endswith("\n") else "\n" + updated = content + suffix + "\n" + wrapped + "\n" + + readme_path.write_text(updated, encoding="utf-8") + + +def main() -> int: + args = parse_args() + if args.count <= 0: + raise SystemExit("--count must be > 0") + if args.warm_runs < 0: + raise SystemExit("--warm-runs must be >= 0") + + root = repo_root() + binary = resolve_path(args.binary, root) + input_path = resolve_path(args.input, root) + query_dir = resolve_path(args.query_dir, root) + csv_path = resolve_path(args.csv_out, root) + readme_path = resolve_path(args.readme, root) + + if not binary.exists(): + raise SystemExit(f"binary not found: {binary}") + if not input_path.exists(): + raise SystemExit(f"input file not found: {input_path}") + if not query_dir.exists(): + raise SystemExit(f"query directory not found: {query_dir}") + + rows = collect_rows( + root=root, + binary=binary, + input_path=input_path, + query_dir=query_dir, + count=args.count, + warm_runs=args.warm_runs, + table_name=args.table_name, + ) + write_csv(csv_path, rows) + markdown_block = render_markdown(csv_path.relative_to(root), rows, args.count, args.warm_runs) + update_readme(readme_path, markdown_block) + print(f"updated {readme_path.relative_to(root)} from {csv_path.relative_to(root)} with {len(rows)} row(s)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2e0e551..34d09d2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -24,6 +24,7 @@ add_library(columnar_engine_lib executor/query_parser.cpp executor/query_planner.cpp executor/aggregate_states.cpp + executor/operator_utils.cpp executor/operators.cpp sql_parser/tokenizer_factory.cpp sql_parser/tokenizer_tokens.cpp diff --git a/src/executor/operator_utils.cpp b/src/executor/operator_utils.cpp new file mode 100644 index 0000000..6b047f4 --- /dev/null +++ b/src/executor/operator_utils.cpp @@ -0,0 +1,83 @@ +#include "executor/operator_utils.h" + +#include + +#include "support/ascii.h" +#include "support/parsing.h" + +template +static bool EvaluateComparison(const T left, const T right, const ComparisonKind kind) { + if (kind == ComparisonKind::Equal) { + return left == right; + } + if (kind == ComparisonKind::NotEqual) { + return left != right; + } + if (kind == ComparisonKind::Less) { + return left < right; + } + if (kind == ComparisonKind::LessOrEqual) { + return left <= right; + } + if (kind == ComparisonKind::Greater) { + return left > right; + } + if (kind == ComparisonKind::GreaterOrEqual) { + return left >= right; + } + throw Error::Unsupported("executor", "unsupported comparison kind"); +} + +template +static int CompareValuesImpl(const T left, const T right) { + if (left < right) { + return -1; + } + if (left > right) { + return 1; + } + return 0; +} + +int CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { + switch (type) { + case ColumnType::Boolean: + return CompareValuesImpl(ParseBoolean(std::string(lhs)), ParseBoolean(std::string(rhs))); + case ColumnType::Int16: + return CompareValuesImpl(ParseInt16(std::string(lhs)), ParseInt16(std::string(rhs))); + case ColumnType::Int32: + return CompareValuesImpl(ParseInt32(std::string(lhs)), ParseInt32(std::string(rhs))); + case ColumnType::Int64: + return CompareValuesImpl(ParseInt64(std::string(lhs)), ParseInt64(std::string(rhs))); + case ColumnType::Int128: + return CompareValuesImpl(ParseInt128(std::string(lhs)), ParseInt128(std::string(rhs))); + case ColumnType::String: + return CompareValuesImpl(std::string(lhs), std::string(rhs)); + case ColumnType::Date: + return CompareValuesImpl(ParseDate(std::string(lhs)), ParseDate(std::string(rhs))); + case ColumnType::Timestamp: + return CompareValuesImpl(ParseTimestamp(std::string(lhs)), ParseTimestamp(std::string(rhs))); + case ColumnType::Character: + return CompareValuesImpl(ParseCharacter(std::string(lhs)), ParseCharacter(std::string(rhs))); + } + throw Error::Unsupported("executor", "unsupported column type in comparison"); +} + +bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, + const ComparisonKind kind) { + return EvaluateComparison(CompareValues(type, lhs, rhs), 0, kind); +} + +ColumnType AggregateOutputType(const PlannedAgg& aggregate) { + const std::string name = ToUpperAscii(aggregate.function->canonical_name); + if (name == "COUNT") { + return ColumnType::Int64; + } + if (name == "SUM" || name == "AVG") { + return ColumnType::Int128; + } + if (name == "MIN" || name == "MAX") { + return aggregate.input_type; + } + throw Error::Unsupported("executor", "unsupported aggregate result type"); +} diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index 6da4aa7..dea2af1 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -1,66 +1,13 @@ +#include #include +#include #include #include -#include "io/columnar_batch_io.h" #include "executor/executor.h" +#include "executor/operator_utils.h" +#include "io/columnar_batch_io.h" #include "io/fileio.h" -#include "support/parsing.h" - -static void SeekRead(const std::filesystem::path& path, std::ifstream& in, const uint64_t offset) { - in.seekg(static_cast(offset)); - if (!in) { - throw Error::PathIo("operators", path, "seek file"); - } -} - -template -static bool EvaluateComparison(const T left, const T right, const ComparisonKind kind) { - if (kind == ComparisonKind::Equal) { - return left == right; - } - if (kind == ComparisonKind::NotEqual) { - return left != right; - } - if (kind == ComparisonKind::Less) { - return left < right; - } - if (kind == ComparisonKind::LessOrEqual) { - return left <= right; - } - if (kind == ComparisonKind::Greater) { - return left > right; - } - if (kind == ComparisonKind::GreaterOrEqual) { - return left >= right; - } - throw Error::Unsupported("operators", "unsupported comparison kind"); -} - -static bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, - const ComparisonKind kind) { - switch (type) { - case ColumnType::Boolean: - return EvaluateComparison(ParseBoolean(std::string(lhs)), ParseBoolean(std::string(rhs)), kind); - case ColumnType::Int16: - return EvaluateComparison(ParseInt16(std::string(lhs)), ParseInt16(std::string(rhs)), kind); - case ColumnType::Int32: - return EvaluateComparison(ParseInt32(std::string(lhs)), ParseInt32(std::string(rhs)), kind); - case ColumnType::Int64: - return EvaluateComparison(ParseInt64(std::string(lhs)), ParseInt64(std::string(rhs)), kind); - case ColumnType::Int128: - return EvaluateComparison(ParseInt128(std::string(lhs)), ParseInt128(std::string(rhs)), kind); - case ColumnType::String: - return EvaluateComparison(std::string(lhs), std::string(rhs), kind); - case ColumnType::Date: - return EvaluateComparison(ParseDate(std::string(lhs)), ParseDate(std::string(rhs)), kind); - case ColumnType::Timestamp: - return EvaluateComparison(ParseTimestamp(std::string(lhs)), ParseTimestamp(std::string(rhs)), kind); - case ColumnType::Character: - return EvaluateComparison(ParseCharacter(std::string(lhs)), ParseCharacter(std::string(rhs)), kind); - } - throw Error::Unsupported("operators", "unsupported column type in comparison"); -} class ScanOperator final : public Operator { public: @@ -93,7 +40,7 @@ class ScanOperator final : public Operator { for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { const size_t source_index = projection_indexes_[projected_index]; const auto& chunk = row_group.columns[source_index]; - SeekRead(path_, in_, chunk.offset); + SeekInputFile(in_, path_, chunk.offset); batch.ColumnAt(projected_index).ReadFrom(in_, row_group.row_count, chunk.size); } @@ -188,7 +135,7 @@ class AggOperator final : public Operator { for (const auto& binding : bindings_) { schema.columns.push_back(ColumnSchema{ .name = binding.aggregate.output_name, - .type = ColumnType::String, + .type = AggregateOutputType(binding.aggregate), }); } @@ -207,11 +154,162 @@ class AggOperator final : public Operator { bool returned_ = false; }; +class GroupAggOperator final : public Operator { + public: + GroupAggOperator(std::unique_ptr child, std::vector group_keys, + std::vector aggregates, std::vector select_items, + std::vector order_by) + : child_(std::move(child)), + group_keys_(std::move(group_keys)), + aggregates_(std::move(aggregates)), + select_items_(std::move(select_items)), + order_by_(std::move(order_by)) {} + + public: + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + returned_ = true; + + while (auto batch = child_->Next()) { + for (size_t row = 0; row < batch->RowsCount(); ++row) { + std::vector key_values; + key_values.reserve(group_keys_.size()); + + std::string group_id; + for (const auto& group_key : group_keys_) { + std::string value = batch->ColumnAt(group_key.column_index).ValueAsString(row); + group_id += std::to_string(value.size()); + group_id.push_back(':'); + group_id += value; + group_id.push_back('|'); + key_values.push_back(std::move(value)); + } + + size_t group_index = 0; + const auto it = group_indexes_.find(group_id); + if (it == group_indexes_.end()) { + group_index = groups_.size(); + group_indexes_.emplace(std::move(group_id), group_index); + groups_.push_back(GroupState{ + .key_values = std::move(key_values), + .states = CreateStates(), + }); + } else { + group_index = it->second; + } + + GroupState& group = groups_[group_index]; + for (size_t i = 0; i < aggregates_.size(); ++i) { + const PlannedAgg& aggregate = aggregates_[i]; + if (!aggregate.star) { + std::string materialized = batch->ColumnAt(*aggregate.column_index).ValueAsString(row); + group.states[i]->Consume(materialized); + } else { + group.states[i]->Consume(std::nullopt); + } + } + } + } + + Schema schema; + schema.columns.reserve(select_items_.size()); + for (const auto& item : select_items_) { + schema.columns.push_back(ColumnSchema{ + .name = item.output_name, + .type = item.output_type, + }); + } + + std::vector> rows; + rows.reserve(groups_.size()); + for (const auto& group : groups_) { + rows.push_back(MaterializeRow(group)); + } + + if (!order_by_.empty()) { + std::ranges::sort(rows, [&](const std::vector& lhs, const std::vector& rhs) { + for (const auto& order : order_by_) { + const int comparison = + CompareValues(order.value_type, lhs[order.result_column_index], rhs[order.result_column_index]); + if (comparison != 0) { + return order.descending ? comparison > 0 : comparison < 0; + } + } + + for (size_t i = 0; i < select_items_.size(); ++i) { + const int comparison = CompareValues(select_items_[i].output_type, lhs[i], rhs[i]); + if (comparison != 0) { + return comparison < 0; + } + } + + return false; + }); + } + + Batch result(schema, rows.size()); + for (const auto& row : rows) { + for (size_t i = 0; i < row.size(); ++i) { + result.ColumnAt(i).AppendFromString(row[i]); + } + } + + return result; + } + + private: + struct GroupState { + std::vector key_values; + std::vector> states; + }; + + std::vector> CreateStates() const { + std::vector> states; + states.reserve(aggregates_.size()); + for (const auto& aggregate : aggregates_) { + states.push_back(CreateAggState(aggregate)); + } + return states; + } + + std::vector MaterializeRow(const GroupState& group) const { + std::vector row; + row.reserve(select_items_.size()); + + for (const auto& item : select_items_) { + if (item.kind == SelectItemKind::GroupKey) { + row.push_back(group.key_values[item.index]); + } else { + row.push_back(group.states[item.index]->Finalize()); + } + } + + return row; + } + + private: + std::unique_ptr child_; + std::vector group_keys_; + std::vector aggregates_; + std::vector select_items_; + std::vector order_by_; + std::unordered_map group_indexes_; + std::vector groups_; + bool returned_ = false; +}; + std::unique_ptr BuildPlan(const PlannedQuery& planned) { std::unique_ptr root = std::make_unique(planned.table_path, planned.projection_indexes); if (planned.filter.has_value()) { root = std::make_unique(std::move(root), *planned.filter); } - root = std::make_unique(std::move(root), planned.aggregates); + if (!planned.group_keys.empty()) { + root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, + planned.select_items, planned.order_by); + } else { + root = std::make_unique(std::move(root), planned.aggregates); + } return root; } diff --git a/src/executor/query_parser.cpp b/src/executor/query_parser.cpp index 1c61a89..1e50d6d 100644 --- a/src/executor/query_parser.cpp +++ b/src/executor/query_parser.cpp @@ -2,20 +2,23 @@ #include #include "executor/executor.h" +#include "support/ascii.h" class TokenCursor { public: explicit TokenCursor(std::vector tokens) : tokens_(std::move(tokens)) {} public: - const Token& Peek() const { - if (pos_ >= tokens_.size()) { + const Token& Peek(const size_t lookahead = 0) const { + if (pos_ + lookahead >= tokens_.size()) { throw Error::InvalidArgument("query_parser", "unexpected end of query"); } - return *tokens_[pos_]; + return *tokens_[pos_ + lookahead]; } - bool Match(const Tokens type) const { return pos_ < tokens_.size() && tokens_[pos_]->GetType() == type; } + bool Match(const Tokens type, const size_t lookahead = 0) const { + return pos_ + lookahead < tokens_.size() && tokens_[pos_ + lookahead]->GetType() == type; + } const Token& Consume(const Tokens type, const std::string_view message) { if (!Match(type)) { @@ -115,6 +118,48 @@ static AggSpec ParseAggregate(TokenCursor& cursor) { return spec; } +static SelectItemSpec ParseSelectItem(TokenCursor& cursor) { + if (cursor.Match(Tokens::NameToken) && !cursor.Match(Tokens::OpenBracket, 1)) { + const Token& token = cursor.Consume(Tokens::NameToken, "expected column name in SELECT"); + return SelectItemSpec{ + .kind = SelectItemKind::GroupKey, + .column_name = std::string(token.GetText()), + .aggregate = {}, + .output_name = std::string(token.GetText()), + }; + } + + return SelectItemSpec{ + .kind = SelectItemKind::Aggregate, + .column_name = {}, + .aggregate = ParseAggregate(cursor), + .output_name = {}, + }; +} + +static OrderBySpec ParseOrderByItem(TokenCursor& cursor) { + SelectItemSpec item = ParseSelectItem(cursor); + if (item.kind == SelectItemKind::Aggregate) { + item.output_name = item.aggregate.output_name; + } + + bool descending = false; + if (cursor.Match(Tokens::NameToken)) { + const std::string direction = ToUpperAscii(cursor.Peek().GetText()); + if (direction == "DESC") { + cursor.Consume(Tokens::NameToken, "expected DESC"); + descending = true; + } else if (direction == "ASC") { + cursor.Consume(Tokens::NameToken, "expected ASC"); + } + } + + return OrderBySpec{ + .item = std::move(item), + .descending = descending, + }; +} + ParsedQuery ParseQuery(const std::string_view query) { auto tokenized = TokenizeSql(query); if (!tokenized.has_value()) { @@ -126,9 +171,15 @@ ParsedQuery ParseQuery(const std::string_view query) { ParsedQuery parsed; cursor.Consume(Tokens::Select, "expected SELECT"); - parsed.aggregates.push_back(ParseAggregate(cursor)); + parsed.select_items.push_back(ParseSelectItem(cursor)); while (cursor.TryConsume(Tokens::Comma)) { - parsed.aggregates.push_back(ParseAggregate(cursor)); + parsed.select_items.push_back(ParseSelectItem(cursor)); + } + + for (auto& item : parsed.select_items) { + if (item.kind == SelectItemKind::Aggregate) { + item.output_name = item.aggregate.output_name; + } } cursor.Consume(Tokens::From, "expected FROM"); @@ -168,6 +219,23 @@ ParsedQuery ParseQuery(const std::string_view query) { parsed.filter = std::move(filter); } + if (cursor.TryConsume(Tokens::Group)) { + cursor.Consume(Tokens::By, "expected BY after GROUP"); + parsed.group_by.push_back(std::string(cursor.Consume(Tokens::NameToken, "expected column name in GROUP BY").GetText())); + while (cursor.TryConsume(Tokens::Comma)) { + parsed.group_by.push_back( + std::string(cursor.Consume(Tokens::NameToken, "expected column name in GROUP BY").GetText())); + } + } + + if (cursor.TryConsume(Tokens::Order)) { + cursor.Consume(Tokens::By, "expected BY after ORDER"); + parsed.order_by.push_back(ParseOrderByItem(cursor)); + while (cursor.TryConsume(Tokens::Comma)) { + parsed.order_by.push_back(ParseOrderByItem(cursor)); + } + } + cursor.TryConsume(Tokens::Semicolon); cursor.Consume(Tokens::EndOfInput, "unexpected trailing tokens"); diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index e2b3ef1..d25e43b 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -1,8 +1,9 @@ #include -#include "support/ascii.h" -#include "io/columnar_batch_io.h" #include "executor/executor.h" +#include "executor/operator_utils.h" +#include "io/columnar_batch_io.h" +#include "support/ascii.h" static size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { const std::string needle = ToLowerAscii(column_name); @@ -49,6 +50,22 @@ static std::string NormalizeLiteral(const ParsedLiteral& literal, const ColumnTy } } +static bool SameColumnName(const std::string_view lhs, const std::string_view rhs) { + return ToLowerAscii(lhs) == ToLowerAscii(rhs); +} + +static bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { + if (lhs.kind != rhs.kind) { + return false; + } + + if (lhs.kind == SelectItemKind::GroupKey) { + return SameColumnName(lhs.column_name, rhs.column_name); + } + + return ToLowerAscii(lhs.aggregate.output_name) == ToLowerAscii(rhs.aggregate.output_name); +} + PlannedQuery PlanQuery(const ParsedQuery& parsed, const std::unordered_map& tables) { PlannedQuery planned; @@ -60,7 +77,7 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, planned.table_path = table_it->second; - ColumnarBatchReader reader(planned.table_path); + const ColumnarBatchReader reader(planned.table_path); const Schema& schema = reader.GetSchema(); const auto register_projection = [&](const size_t source_index) -> size_t { @@ -84,9 +101,51 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, }; } - planned.aggregates.reserve(parsed.aggregates.size()); + planned.group_keys.reserve(parsed.group_by.size()); + for (const std::string& group_column : parsed.group_by) { + const size_t source_index = FindColumnIndex(schema, group_column); + planned.group_keys.push_back(PlannedGroupKey{ + .column_index = register_projection(source_index), + .column_type = schema.columns[source_index].type, + .output_name = schema.columns[source_index].name, + }); + } + + bool has_aggregate = false; + for (const auto& item : parsed.select_items) { + if (item.kind == SelectItemKind::Aggregate) { + has_aggregate = true; + break; + } + } + + planned.select_items.reserve(parsed.select_items.size()); + planned.aggregates.reserve(parsed.select_items.size()); + + for (const auto& item : parsed.select_items) { + if (item.kind == SelectItemKind::GroupKey) { + if (parsed.group_by.empty()) { + throw Error::Unsupported("query_planner", "plain column SELECT items require GROUP BY"); + } + + const auto it = std::ranges::find_if(planned.group_keys, [&](const PlannedGroupKey& group_key) { + return SameColumnName(group_key.output_name, item.column_name); + }); + if (it == planned.group_keys.end()) { + throw Error::InvalidArgument("query_planner", + "column '" + item.column_name + "' must appear in GROUP BY"); + } + + planned.select_items.push_back(PlannedSelectItem{ + .kind = SelectItemKind::GroupKey, + .index = static_cast(std::distance(planned.group_keys.begin(), it)), + .output_type = it->column_type, + .output_name = it->output_name, + }); + continue; + } - for (const auto& aggregate : parsed.aggregates) { + const AggSpec& aggregate = item.aggregate; PlannedAgg planned_aggregate{ .function = aggregate.function, .distinct = aggregate.distinct, @@ -107,7 +166,35 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, } } + const size_t aggregate_index = planned.aggregates.size(); planned.aggregates.push_back(std::move(planned_aggregate)); + planned.select_items.push_back(PlannedSelectItem{ + .kind = SelectItemKind::Aggregate, + .index = aggregate_index, + .output_type = AggregateOutputType(planned.aggregates.back()), + .output_name = aggregate.output_name, + }); + } + + if (parsed.group_by.empty() && !has_aggregate) { + throw Error::Unsupported("query_planner", "plain SELECT without aggregates is not supported"); + } + + planned.order_by.reserve(parsed.order_by.size()); + for (const auto& order_item : parsed.order_by) { + const auto it = std::ranges::find_if(parsed.select_items, [&](const SelectItemSpec& select_item) { + return SameSelectItem(select_item, order_item.item); + }); + if (it == parsed.select_items.end()) { + throw Error::InvalidArgument("query_planner", "ORDER BY expression must appear in SELECT"); + } + + const size_t result_column_index = static_cast(std::distance(parsed.select_items.begin(), it)); + planned.order_by.push_back(PlannedOrderBy{ + .result_column_index = result_column_index, + .descending = order_item.descending, + .value_type = planned.select_items[result_column_index].output_type, + }); } if (planned.projection_indexes.empty() && !schema.columns.empty()) { diff --git a/src/io/batch_io.cpp b/src/io/batch_io.cpp index 06ba418..786caf5 100644 --- a/src/io/batch_io.cpp +++ b/src/io/batch_io.cpp @@ -11,13 +11,6 @@ static constexpr std::string_view ColumnarMagic = "CLMN"; -static void SeekRead(const std::filesystem::path& path, std::ifstream& in, const uint64_t pos) { - in.seekg(pos); - if (!in) { - throw Error::PathIo("batch_io", path, "seek file"); - } -} - static uint64_t TellWrite(const std::filesystem::path& path, std::ofstream& out) { const auto pos = out.tellp(); if (pos == -1) { @@ -218,7 +211,7 @@ ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::pa } std::ifstream in = OpenInputFile(path); - SeekRead(path, in, file_size - FooterSize); + SeekInputFile(in, path, file_size - FooterSize); const uint64_t metadata_size = ReadStream(in); std::string magic_read(ColumnarMagic.size(), '\0'); @@ -232,7 +225,7 @@ ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::pa throw Error::InvalidData("batch_io", "metadata size exceeds file size", path.string()); } - SeekRead(path, in, file_size - FooterSize - metadata_size); + SeekInputFile(in, path, file_size - FooterSize - metadata_size); return ReadMetadata(in); } @@ -260,7 +253,7 @@ std::optional ColumnarBatchReader::ReadNext() { for (size_t col = 0; col < columns.size(); ++col) { const auto& [offset, size] = row_group_columns[col]; - SeekRead(path_, in_, offset); + SeekInputFile(in_, path_, offset); columns[col]->ReadFrom(in_, row_count, size); } diff --git a/src/io/fileio.cpp b/src/io/fileio.cpp index 3c58721..b332524 100644 --- a/src/io/fileio.cpp +++ b/src/io/fileio.cpp @@ -66,6 +66,13 @@ std::ifstream OpenInputFile(const std::filesystem::path& path) { return file; } +void SeekInputFile(std::istream& in, const std::filesystem::path& path, const uint64_t offset) { + in.seekg(static_cast(offset)); + if (!in) { + throw Error::PathIo("fileio", path, "seek for read"); + } +} + std::ofstream OpenOutputFile(const std::filesystem::path& path, const FileOpenMode mode) { std::ofstream file(path, std::ios::binary | (mode == FileOpenMode::Append ? std::ios::app : std::ios::trunc)); if (!file.is_open()) { diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index 567ecd6..3a04eba 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -1,11 +1,11 @@ #include #include -#include "io/csv.h" #include "convert/csv_columnar.h" #include "executor/executor.h" -#include "testing/executor_test_utils.h" #include "gtest/gtest.h" +#include "io/csv.h" +#include "testing/executor_test_utils.h" #include "testing/temp_file.h" static Batch BuildHitsTable(const std::string_view query) { @@ -48,6 +48,18 @@ TEST(executor, usage_example_register_table_and_execute_query) { EXPECT_EQ(SingleRowValues(batch), std::vector{"5"}); } +TEST(executor, groups_rows_and_orders_by_aggregate_descending) { + const Batch batch = BuildHitsTable( + "SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + EXPECT_EQ(BatchRows(batch), (std::vector>{ + {"7", "2"}, + {"1", "1"}, + {"3", "1"}, + })); +} + TEST(executor, executes_basic_aggregate_queries) { { const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); From cd9e507fcdd118141f8c6b69438aac9953d35e3f Mon Sep 17 00:00:00 2001 From: ik4rd Date: Tue, 5 May 2026 00:25:42 +0300 Subject: [PATCH 09/29] feat: 8/10 queries done --- README.md | 20 ++++++++++---------- benchmarks/results/readme_benchmarks.csv | 16 ++++++++-------- scripts/dashboard.py | 6 ++---- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ac9007f..a562f9a 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,14 @@ ## Dashboard -| Query | SQL | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | -| --- | --- | ---: | ---: | ---: | ---: | ---: | --- | -| [query_0.sql](benchmarks/queries/query_0.sql) | `SELECT COUNT(*) FROM hits;` | 53.43 | 9.63 | 9.23 | 8.77 | 11.28 | ok | -| [query_1.sql](benchmarks/queries/query_1.sql) | `SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;` | 41.22 | 27.88 | 27.79 | 27.43 | 28.51 | ok | -| [query_2.sql](benchmarks/queries/query_2.sql) | `SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;` | 62.18 | 54.10 | 53.49 | 50.64 | 58.77 | ok | -| [query_3.sql](benchmarks/queries/query_3.sql) | `SELECT AVG(UserID) FROM hits;` | 85.12 | 58.40 | 54.68 | 54.23 | 70.00 | ok | -| [query_4.sql](benchmarks/queries/query_4.sql) | `SELECT COUNT(DISTINCT UserID) FROM hits;` | 46.61 | 47.05 | 47.30 | 43.96 | 49.62 | ok | -| [query_5.sql](benchmarks/queries/query_5.sql) | `SELECT COUNT(DISTINCT SearchPhrase) FROM hits;` | 61.44 | 45.14 | 44.92 | 44.58 | 46.12 | ok | -| [query_6.sql](benchmarks/queries/query_6.sql) | `SELECT MIN(EventDate), MAX(EventDate) FROM hits;` | 685.92 | 665.72 | 665.60 | 665.26 | 666.43 | ok | -| [query_7.sql](benchmarks/queries/query_7.sql) | `SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDE…` | 28.69 | 28.12 | 28.13 | 28.01 | 28.20 | ok | +| Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| [query_0.sql](benchmarks/queries/query_0.sql) | 28.55 | 8.02 | 8.02 | 7.94 | 8.09 | ok | +| [query_1.sql](benchmarks/queries/query_1.sql) | 40.47 | 28.38 | 28.29 | 28.07 | 28.89 | ok | +| [query_2.sql](benchmarks/queries/query_2.sql) | 64.85 | 52.41 | 51.88 | 50.87 | 55.01 | ok | +| [query_3.sql](benchmarks/queries/query_3.sql) | 71.56 | 52.63 | 52.51 | 52.09 | 53.42 | ok | +| [query_4.sql](benchmarks/queries/query_4.sql) | 43.76 | 47.17 | 45.71 | 43.59 | 53.66 | ok | +| [query_5.sql](benchmarks/queries/query_5.sql) | 65.48 | 48.34 | 46.16 | 45.11 | 55.94 | ok | +| [query_6.sql](benchmarks/queries/query_6.sql) | 679.19 | 673.33 | 668.61 | 664.56 | 691.56 | ok | +| [query_7.sql](benchmarks/queries/query_7.sql) | 30.22 | 29.44 | 29.44 | 28.81 | 30.07 | ok | diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index 777f677..8051df3 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,9 +1,9 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,53.43,9.63,9.23,8.77,11.28,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,41.22,27.88,27.79,27.43,28.51,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",62.18,54.10,53.49,50.64,58.77,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,85.12,58.40,54.68,54.23,70.00,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,46.61,47.05,47.30,43.96,49.62,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,61.44,45.14,44.92,44.58,46.12,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",685.92,665.72,665.60,665.26,666.43,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",28.69,28.12,28.13,28.01,28.20,ok, +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,28.55,8.02,8.02,7.94,8.09,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,40.47,28.38,28.29,28.07,28.89,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",64.85,52.41,51.88,50.87,55.01,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,71.56,52.63,52.51,52.09,53.42,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,43.76,47.17,45.71,43.59,53.66,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,65.48,48.34,46.16,45.11,55.94,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",679.19,673.33,668.61,664.56,691.56,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",30.22,29.44,29.44,28.81,30.07,ok, diff --git a/scripts/dashboard.py b/scripts/dashboard.py index 53e87a8..b6faf47 100644 --- a/scripts/dashboard.py +++ b/scripts/dashboard.py @@ -208,14 +208,13 @@ def render_markdown(csv_path: pathlib.Path, rows: list[dict[str, str]], count: i "## Dashboard", "", "", - "| Query | SQL | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status |", - "| --- | --- | ---: | ---: | ---: | ---: | ---: | --- |", + "| Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status |", + "| --- | ---: | ---: | ---: | ---: | ---: | --- |", ] for row in rows: query_file = row["query_file"] query_label = f"[query_{row['query_id']}.sql]({query_file})" - sql = f"`{markdown_escape(compact_sql(row['sql']))}`" status = row["status"] if row["error"]: status = f"{status}: {markdown_escape(compact_sql(row['error'], max_len=48))}" @@ -225,7 +224,6 @@ def render_markdown(csv_path: pathlib.Path, rows: list[dict[str, str]], count: i + " | ".join( [ query_label, - sql, row["first_run_ms"] or "—", row["warm_avg_ms"] or "—", row["warm_median_ms"] or "—", From ff6c6204d52190ba6b6a8b10191d1a29b5fff35d Mon Sep 17 00:00:00 2001 From: ik4rd Date: Tue, 5 May 2026 00:31:31 +0300 Subject: [PATCH 10/29] fix: add missed queries runners --- README.md | 16 ++++++++-------- benchmarks/results/readme_benchmarks.csv | 16 ++++++++-------- benchmarks/support/query_runners.h | 16 ++++++++++++++++ 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index a562f9a..168a204 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,12 @@ | Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | | --- | ---: | ---: | ---: | ---: | ---: | --- | -| [query_0.sql](benchmarks/queries/query_0.sql) | 28.55 | 8.02 | 8.02 | 7.94 | 8.09 | ok | -| [query_1.sql](benchmarks/queries/query_1.sql) | 40.47 | 28.38 | 28.29 | 28.07 | 28.89 | ok | -| [query_2.sql](benchmarks/queries/query_2.sql) | 64.85 | 52.41 | 51.88 | 50.87 | 55.01 | ok | -| [query_3.sql](benchmarks/queries/query_3.sql) | 71.56 | 52.63 | 52.51 | 52.09 | 53.42 | ok | -| [query_4.sql](benchmarks/queries/query_4.sql) | 43.76 | 47.17 | 45.71 | 43.59 | 53.66 | ok | -| [query_5.sql](benchmarks/queries/query_5.sql) | 65.48 | 48.34 | 46.16 | 45.11 | 55.94 | ok | -| [query_6.sql](benchmarks/queries/query_6.sql) | 679.19 | 673.33 | 668.61 | 664.56 | 691.56 | ok | -| [query_7.sql](benchmarks/queries/query_7.sql) | 30.22 | 29.44 | 29.44 | 28.81 | 30.07 | ok | +| [query_0.sql](benchmarks/queries/query_0.sql) | 29.81 | 8.51 | 8.45 | 8.31 | 8.82 | ok | +| [query_1.sql](benchmarks/queries/query_1.sql) | 42.13 | 30.55 | 30.61 | 29.61 | 31.37 | ok | +| [query_2.sql](benchmarks/queries/query_2.sql) | 81.45 | 55.04 | 54.61 | 52.69 | 58.24 | ok | +| [query_3.sql](benchmarks/queries/query_3.sql) | 73.69 | 57.07 | 57.04 | 55.59 | 58.59 | ok | +| [query_4.sql](benchmarks/queries/query_4.sql) | 49.07 | 58.40 | 55.45 | 55.17 | 67.53 | ok | +| [query_5.sql](benchmarks/queries/query_5.sql) | 65.59 | 53.05 | 52.50 | 46.93 | 60.27 | ok | +| [query_6.sql](benchmarks/queries/query_6.sql) | 768.21 | 698.64 | 685.19 | 668.31 | 755.87 | ok | +| [query_7.sql](benchmarks/queries/query_7.sql) | 28.96 | 28.49 | 28.49 | 28.28 | 28.70 | ok | diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index 8051df3..2dea935 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,9 +1,9 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,28.55,8.02,8.02,7.94,8.09,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,40.47,28.38,28.29,28.07,28.89,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",64.85,52.41,51.88,50.87,55.01,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,71.56,52.63,52.51,52.09,53.42,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,43.76,47.17,45.71,43.59,53.66,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,65.48,48.34,46.16,45.11,55.94,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",679.19,673.33,668.61,664.56,691.56,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",30.22,29.44,29.44,28.81,30.07,ok, +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,29.81,8.51,8.45,8.31,8.82,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,42.13,30.55,30.61,29.61,31.37,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",81.45,55.04,54.61,52.69,58.24,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,73.69,57.07,57.04,55.59,58.59,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,49.07,58.40,55.45,55.17,67.53,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,65.59,53.05,52.50,46.93,60.27,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",768.21,698.64,685.19,668.31,755.87,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",28.96,28.49,28.49,28.28,28.70,ok, diff --git a/benchmarks/support/query_runners.h b/benchmarks/support/query_runners.h index 3776cfa..f0717bd 100644 --- a/benchmarks/support/query_runners.h +++ b/benchmarks/support/query_runners.h @@ -23,6 +23,19 @@ inline ExecuteExpected RunQuery4(const Executor& executor) { return executor.Execute("SELECT COUNT(DISTINCT UserID) FROM hits;"); } +inline ExecuteExpected RunQuery5(const Executor& executor) { + return executor.Execute("SELECT COUNT(DISTINCT SearchPhrase) FROM hits;"); +} + +inline ExecuteExpected RunQuery6(const Executor& executor) { + return executor.Execute("SELECT MIN(EventDate), MAX(EventDate) FROM hits;"); +} + +inline ExecuteExpected RunQuery7(const Executor& executor) { + return executor.Execute( + "SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;"); +} + inline const std::map& GetQueryRunners() { static const std::map Runners = { {0, RunQuery0}, @@ -30,6 +43,9 @@ inline const std::map& GetQueryRunners() { {2, RunQuery2}, {3, RunQuery3}, {4, RunQuery4}, + {5, RunQuery5}, + {6, RunQuery6}, + {7, RunQuery7}, }; return Runners; } From 18f96083555029b6b44baa051eaeb99c8f40a751 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Tue, 5 May 2026 00:34:27 +0300 Subject: [PATCH 11/29] feat: dashboard for all queries --- README.md | 51 ++++++++++++++++++++---- benchmarks/benchmark_queries.cpp | 2 +- benchmarks/results/readme_benchmarks.csv | 51 ++++++++++++++++++++---- scripts/dashboard.py | 2 +- 4 files changed, 88 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 168a204..44ea475 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,47 @@ | Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | | --- | ---: | ---: | ---: | ---: | ---: | --- | -| [query_0.sql](benchmarks/queries/query_0.sql) | 29.81 | 8.51 | 8.45 | 8.31 | 8.82 | ok | -| [query_1.sql](benchmarks/queries/query_1.sql) | 42.13 | 30.55 | 30.61 | 29.61 | 31.37 | ok | -| [query_2.sql](benchmarks/queries/query_2.sql) | 81.45 | 55.04 | 54.61 | 52.69 | 58.24 | ok | -| [query_3.sql](benchmarks/queries/query_3.sql) | 73.69 | 57.07 | 57.04 | 55.59 | 58.59 | ok | -| [query_4.sql](benchmarks/queries/query_4.sql) | 49.07 | 58.40 | 55.45 | 55.17 | 67.53 | ok | -| [query_5.sql](benchmarks/queries/query_5.sql) | 65.59 | 53.05 | 52.50 | 46.93 | 60.27 | ok | -| [query_6.sql](benchmarks/queries/query_6.sql) | 768.21 | 698.64 | 685.19 | 668.31 | 755.87 | ok | -| [query_7.sql](benchmarks/queries/query_7.sql) | 28.96 | 28.49 | 28.49 | 28.28 | 28.70 | ok | +| [query_0.sql](benchmarks/queries/query_0.sql) | 28.86 | 12.14 | 11.25 | 9.60 | 16.44 | ok | +| [query_1.sql](benchmarks/queries/query_1.sql) | 51.92 | 30.69 | 30.18 | 29.77 | 32.64 | ok | +| [query_2.sql](benchmarks/queries/query_2.sql) | 72.78 | 58.04 | 58.82 | 54.30 | 60.23 | ok | +| [query_3.sql](benchmarks/queries/query_3.sql) | 74.22 | 77.95 | 67.37 | 52.85 | 124.19 | ok | +| [query_4.sql](benchmarks/queries/query_4.sql) | 68.50 | 52.63 | 51.18 | 48.12 | 60.06 | ok | +| [query_5.sql](benchmarks/queries/query_5.sql) | 60.41 | 44.76 | 44.68 | 44.65 | 45.02 | ok | +| [query_6.sql](benchmarks/queries/query_6.sql) | 693.40 | 675.22 | 665.84 | 663.16 | 706.04 | ok | +| [query_7.sql](benchmarks/queries/query_7.sql) | 31.60 | 29.68 | 29.62 | 28.63 | 30.84 | ok | +| [query_8.sql](benchmarks/queries/query_8.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_9.sql](benchmarks/queries/query_9.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_10.sql](benchmarks/queries/query_10.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_11.sql](benchmarks/queries/query_11.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_12.sql](benchmarks/queries/query_12.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_13.sql](benchmarks/queries/query_13.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_14.sql](benchmarks/queries/query_14.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_15.sql](benchmarks/queries/query_15.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | +| [query_16.sql](benchmarks/queries/query_16.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | +| [query_17.sql](benchmarks/queries/query_17.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | +| [query_18.sql](benchmarks/queries/query_18.sql) | — | — | — | — | — | failed: executor: aggregate 'extract' is not registered | +| [query_19.sql](benchmarks/queries/query_19.sql) | — | — | — | — | — | failed: query_planner: plain column SELECT items requir… | +| [query_20.sql](benchmarks/queries/query_20.sql) | — | — | — | — | — | failed: query_parser: expected comparison operator in W… | +| [query_21.sql](benchmarks/queries/query_21.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_22.sql](benchmarks/queries/query_22.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_23.sql](benchmarks/queries/query_23.sql) | — | — | — | — | — | failed: executor: aggregate '*' is not registered | +| [query_24.sql](benchmarks/queries/query_24.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | +| [query_25.sql](benchmarks/queries/query_25.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | +| [query_26.sql](benchmarks/queries/query_26.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | +| [query_27.sql](benchmarks/queries/query_27.sql) | — | — | — | — | — | failed: query_parser: expected ')' after aggregate argu… | +| [query_28.sql](benchmarks/queries/query_28.sql) | — | — | — | — | — | failed: executor: aggregate 'REGEXP_REPLACE' is not reg… | +| [query_29.sql](benchmarks/queries/query_29.sql) | — | — | — | — | — | failed: query_parser: expected ')' after aggregate argu… | +| [query_30.sql](benchmarks/queries/query_30.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_31.sql](benchmarks/queries/query_31.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_32.sql](benchmarks/queries/query_32.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_33.sql](benchmarks/queries/query_33.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_34.sql](benchmarks/queries/query_34.sql) | — | — | — | — | — | failed: query_parser: expected aggregate function | +| [query_35.sql](benchmarks/queries/query_35.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_36.sql](benchmarks/queries/query_36.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_37.sql](benchmarks/queries/query_37.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_38.sql](benchmarks/queries/query_38.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_39.sql](benchmarks/queries/query_39.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_40.sql](benchmarks/queries/query_40.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_41.sql](benchmarks/queries/query_41.sql) | — | — | — | — | — | failed: query_parser: expected FROM | +| [query_42.sql](benchmarks/queries/query_42.sql) | — | — | — | — | — | failed: executor: aggregate 'DATE_TRUNC' is not registe… | diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index 71d417d..9e8fb4d 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -15,7 +15,7 @@ #endif static constexpr int FirstQueryId = 0; -static constexpr int LastQueryId = 7; +static constexpr int LastQueryId = 42; std::string BatchToString(const Batch& batch) { std::vector> rows; diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index 2dea935..c56ffb3 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,9 +1,44 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,29.81,8.51,8.45,8.31,8.82,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,42.13,30.55,30.61,29.61,31.37,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",81.45,55.04,54.61,52.69,58.24,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,73.69,57.07,57.04,55.59,58.59,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,49.07,58.40,55.45,55.17,67.53,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,65.59,53.05,52.50,46.93,60.27,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",768.21,698.64,685.19,668.31,755.87,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",28.96,28.49,28.49,28.28,28.70,ok, +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,28.86,12.14,11.25,9.60,16.44,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,51.92,30.69,30.18,29.77,32.64,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",72.78,58.04,58.82,54.30,60.23,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,74.22,77.95,67.37,52.85,124.19,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,68.50,52.63,51.18,48.12,60.06,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,60.41,44.76,44.68,44.65,45.02,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",693.40,675.22,665.84,663.16,706.04,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",31.60,29.68,29.62,28.63,30.84,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",,,,,,failed,query_parser: unexpected trailing tokens +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",,,,,,failed,query_parser: unexpected trailing tokens +17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",,,,,,failed,query_parser: unexpected trailing tokens +18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",,,,,,failed,executor: aggregate 'extract' is not registered +19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,,,,,,failed,query_planner: plain column SELECT items require GROUP BY +20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,,,,,,failed,query_parser: expected comparison operator in WHERE +21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,,,,,,failed,executor: aggregate '*' is not registered +24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,,,,,,failed,query_parser: unexpected trailing tokens +25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,,,,,,failed,query_parser: unexpected trailing tokens +26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",,,,,,failed,query_parser: unexpected trailing tokens +27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",,,,,,failed,query_parser: expected ')' after aggregate arguments +28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",,,,,,failed,executor: aggregate 'REGEXP_REPLACE' is not registered +29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",,,,,,failed,query_parser: expected ')' after aggregate arguments +30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected aggregate function +35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM +38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",,,,,,failed,query_parser: expected FROM +39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",,,,,,failed,query_parser: expected FROM +40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",,,,,,failed,query_parser: expected FROM +41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",,,,,,failed,query_parser: expected FROM +42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",,,,,,failed,executor: aggregate 'DATE_TRUNC' is not registered diff --git a/scripts/dashboard.py b/scripts/dashboard.py index b6faf47..df789d7 100644 --- a/scripts/dashboard.py +++ b/scripts/dashboard.py @@ -20,7 +20,7 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Run the first N benchmark queries, write CSV stats, and refresh the README benchmark table." ) - parser.add_argument("--count", type=int, default=8, help="Number of first queries to benchmark.") + parser.add_argument("--count", type=int, default=43, help="Number of first queries to benchmark.") parser.add_argument("--warm-runs", type=int, default=4, help="Number of warm runs after the first measured run.") parser.add_argument( "--query-dir", From aa4e6255ccf868488da95e59c076e9b540d48c09 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Tue, 5 May 2026 01:50:40 +0300 Subject: [PATCH 12/29] feat: 17/43 queries done --- README.md | 60 ++++--------- benchmarks/CMakeLists.txt | 2 + benchmarks/benchmark_queries.cpp | 33 ++------ benchmarks/queries/query_10.csv | 10 +++ benchmarks/queries/query_11.csv | 10 +++ benchmarks/queries/query_12.csv | 10 +++ benchmarks/queries/query_13.csv | 10 +++ benchmarks/queries/query_14.csv | 10 +++ benchmarks/queries/query_15.csv | 10 +++ benchmarks/queries/query_16.csv | 10 +++ benchmarks/results/readme_benchmarks.csv | 60 ++++--------- benchmarks/support/query_result_checks.cpp | 60 +++++++++++++ benchmarks/support/query_result_checks.h | 10 +++ benchmarks/support/query_runners.h | 63 ++++++++++++-- include/executor/executor.h | 16 +++- include/executor/planner_utils.h | 14 ++++ scripts/dashboard.py | 4 +- src/CMakeLists.txt | 1 + src/executor/operators.cpp | 71 ++++++++++++---- src/executor/planner_utils.cpp | 95 +++++++++++++++++++++ src/executor/query_parser.cpp | 98 +++++++++++++++++----- src/executor/query_planner.cpp | 90 ++++---------------- tests/test_executor.cpp | 96 ++++++++++++++++++--- 23 files changed, 599 insertions(+), 244 deletions(-) create mode 100644 benchmarks/queries/query_10.csv create mode 100644 benchmarks/queries/query_11.csv create mode 100644 benchmarks/queries/query_12.csv create mode 100644 benchmarks/queries/query_13.csv create mode 100644 benchmarks/queries/query_14.csv create mode 100644 benchmarks/queries/query_15.csv create mode 100644 benchmarks/queries/query_16.csv create mode 100644 benchmarks/support/query_result_checks.cpp create mode 100644 benchmarks/support/query_result_checks.h create mode 100644 include/executor/planner_utils.h create mode 100644 src/executor/planner_utils.cpp diff --git a/README.md b/README.md index 44ea475..be0e72b 100644 --- a/README.md +++ b/README.md @@ -8,47 +8,21 @@ | Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | | --- | ---: | ---: | ---: | ---: | ---: | --- | -| [query_0.sql](benchmarks/queries/query_0.sql) | 28.86 | 12.14 | 11.25 | 9.60 | 16.44 | ok | -| [query_1.sql](benchmarks/queries/query_1.sql) | 51.92 | 30.69 | 30.18 | 29.77 | 32.64 | ok | -| [query_2.sql](benchmarks/queries/query_2.sql) | 72.78 | 58.04 | 58.82 | 54.30 | 60.23 | ok | -| [query_3.sql](benchmarks/queries/query_3.sql) | 74.22 | 77.95 | 67.37 | 52.85 | 124.19 | ok | -| [query_4.sql](benchmarks/queries/query_4.sql) | 68.50 | 52.63 | 51.18 | 48.12 | 60.06 | ok | -| [query_5.sql](benchmarks/queries/query_5.sql) | 60.41 | 44.76 | 44.68 | 44.65 | 45.02 | ok | -| [query_6.sql](benchmarks/queries/query_6.sql) | 693.40 | 675.22 | 665.84 | 663.16 | 706.04 | ok | -| [query_7.sql](benchmarks/queries/query_7.sql) | 31.60 | 29.68 | 29.62 | 28.63 | 30.84 | ok | -| [query_8.sql](benchmarks/queries/query_8.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_9.sql](benchmarks/queries/query_9.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_10.sql](benchmarks/queries/query_10.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_11.sql](benchmarks/queries/query_11.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_12.sql](benchmarks/queries/query_12.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_13.sql](benchmarks/queries/query_13.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_14.sql](benchmarks/queries/query_14.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_15.sql](benchmarks/queries/query_15.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | -| [query_16.sql](benchmarks/queries/query_16.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | -| [query_17.sql](benchmarks/queries/query_17.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | -| [query_18.sql](benchmarks/queries/query_18.sql) | — | — | — | — | — | failed: executor: aggregate 'extract' is not registered | -| [query_19.sql](benchmarks/queries/query_19.sql) | — | — | — | — | — | failed: query_planner: plain column SELECT items requir… | -| [query_20.sql](benchmarks/queries/query_20.sql) | — | — | — | — | — | failed: query_parser: expected comparison operator in W… | -| [query_21.sql](benchmarks/queries/query_21.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_22.sql](benchmarks/queries/query_22.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_23.sql](benchmarks/queries/query_23.sql) | — | — | — | — | — | failed: executor: aggregate '*' is not registered | -| [query_24.sql](benchmarks/queries/query_24.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | -| [query_25.sql](benchmarks/queries/query_25.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | -| [query_26.sql](benchmarks/queries/query_26.sql) | — | — | — | — | — | failed: query_parser: unexpected trailing tokens | -| [query_27.sql](benchmarks/queries/query_27.sql) | — | — | — | — | — | failed: query_parser: expected ')' after aggregate argu… | -| [query_28.sql](benchmarks/queries/query_28.sql) | — | — | — | — | — | failed: executor: aggregate 'REGEXP_REPLACE' is not reg… | -| [query_29.sql](benchmarks/queries/query_29.sql) | — | — | — | — | — | failed: query_parser: expected ')' after aggregate argu… | -| [query_30.sql](benchmarks/queries/query_30.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_31.sql](benchmarks/queries/query_31.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_32.sql](benchmarks/queries/query_32.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_33.sql](benchmarks/queries/query_33.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_34.sql](benchmarks/queries/query_34.sql) | — | — | — | — | — | failed: query_parser: expected aggregate function | -| [query_35.sql](benchmarks/queries/query_35.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_36.sql](benchmarks/queries/query_36.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_37.sql](benchmarks/queries/query_37.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_38.sql](benchmarks/queries/query_38.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_39.sql](benchmarks/queries/query_39.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_40.sql](benchmarks/queries/query_40.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_41.sql](benchmarks/queries/query_41.sql) | — | — | — | — | — | failed: query_parser: expected FROM | -| [query_42.sql](benchmarks/queries/query_42.sql) | — | — | — | — | — | failed: executor: aggregate 'DATE_TRUNC' is not registe… | +| [0](benchmarks/queries/query_0.sql) | 31.90 | 10.09 | 9.93 | 9.63 | 10.88 | ok | +| [1](benchmarks/queries/query_1.sql) | 47.47 | 29.97 | 29.35 | 28.25 | 32.94 | ok | +| [2](benchmarks/queries/query_2.sql) | 65.71 | 99.09 | 88.86 | 52.07 | 166.55 | ok | +| [3](benchmarks/queries/query_3.sql) | 102.02 | 63.90 | 64.27 | 59.52 | 67.55 | ok | +| [4](benchmarks/queries/query_4.sql) | 53.78 | 72.75 | 57.36 | 55.74 | 120.53 | ok | +| [5](benchmarks/queries/query_5.sql) | 64.92 | 47.45 | 47.18 | 44.87 | 50.55 | ok | +| [6](benchmarks/queries/query_6.sql) | 689.46 | 689.40 | 681.74 | 669.80 | 724.31 | ok | +| [7](benchmarks/queries/query_7.sql) | 32.64 | 28.93 | 28.98 | 28.51 | 29.27 | ok | +| [8](benchmarks/queries/query_8.sql) | 137.21 | 125.35 | 126.21 | 118.63 | 130.37 | ok | +| [9](benchmarks/queries/query_9.sql) | 181.60 | 185.20 | 182.91 | 177.70 | 197.28 | ok | +| [10](benchmarks/queries/query_10.sql) | 48.49 | 36.32 | 36.14 | 35.74 | 37.27 | ok | +| [11](benchmarks/queries/query_11.sql) | 52.50 | 40.40 | 40.20 | 38.94 | 42.24 | ok | +| [12](benchmarks/queries/query_12.sql) | 91.76 | 88.94 | 88.69 | 86.63 | 91.77 | ok | +| [13](benchmarks/queries/query_13.sql) | 100.89 | 97.84 | 96.94 | 94.84 | 102.63 | ok | +| [14](benchmarks/queries/query_14.sql) | 107.95 | 95.51 | 95.43 | 94.33 | 96.84 | ok | +| [15](benchmarks/queries/query_15.sql) | 175.84 | 169.96 | 169.44 | 164.33 | 176.64 | ok | +| [16](benchmarks/queries/query_16.sql) | 259.40 | 267.56 | 267.52 | 266.79 | 268.43 | ok | diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 9102491..5559b8d 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -9,6 +9,8 @@ target_link_libraries(benchmark_csv_to_columnar add_executable(benchmark_queries benchmark_queries.cpp + support/query_result_checks.cpp + support/query_result_checks.h support/query_runners.h ) diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index 9e8fb4d..2d5469f 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -1,13 +1,13 @@ +#include #include #include #include #include #include -#include #include "executor/executor.h" -#include "io/csv_batch_io.h" #include "io/fileio.h" +#include "support/query_result_checks.h" #include "support/query_runners.h" #ifndef COLUMNAR_BENCHMARK_QUERIES_DIR @@ -15,30 +15,13 @@ #endif static constexpr int FirstQueryId = 0; -static constexpr int LastQueryId = 42; - -std::string BatchToString(const Batch& batch) { - std::vector> rows; - AppendBatchRows(batch, rows); - - std::string result; - - for (const auto& row : rows) { - for (size_t i = 0; i < row.size(); ++i) { - result += row[i]; - if (i + 1 < row.size()) { - result += ","; - } - } - result += "\n"; - } - - return result; -} +static constexpr int LastQueryId = 16; void RunQuery(const Executor& executor, const std::filesystem::path& path, const int query_id, bool record = false) { const auto& runners = GetQueryRunners(); const auto it = runners.find(query_id); + const std::string query_sql = std::filesystem::exists(path) ? ReadTextFile(path) : ""; + const bool compare_unordered = QueryUsesGroupBy(query_sql); ExecuteExpected result; const auto start = std::chrono::steady_clock::now(); @@ -46,8 +29,7 @@ void RunQuery(const Executor& executor, const std::filesystem::path& path, const if (it != runners.end()) { result = it->second(executor); } else if (std::filesystem::exists(path)) { - const std::string sql = ReadTextFile(path); - result = executor.Execute(sql); + result = executor.Execute(query_sql); } else { std::cout << "query_" << query_id << ": missing, status=skipped" << std::endl; return; @@ -74,7 +56,8 @@ void RunQuery(const Executor& executor, const std::filesystem::path& path, const status = "recorded"; } else if (std::filesystem::exists(ref_path)) { const std::string expected_result = ReadTextFile(ref_path); - if (actual_result == expected_result) { + if (compare_unordered ? EqualAsRowMultisets(actual_result, expected_result) + : actual_result == expected_result) { status = "passed"; } else { status = "failed"; diff --git a/benchmarks/queries/query_10.csv b/benchmarks/queries/query_10.csv new file mode 100644 index 0000000..4734f82 --- /dev/null +++ b/benchmarks/queries/query_10.csv @@ -0,0 +1,10 @@ +iPad,2303 +iPhone,107 +A500,34 +N8-00,12 +GT-P7300B,12 +iPho,11 +3110000,6 +IQ245Plus,5 +eagle75,4 +GT-S5830,3 diff --git a/benchmarks/queries/query_11.csv b/benchmarks/queries/query_11.csv new file mode 100644 index 0000000..3f24c76 --- /dev/null +++ b/benchmarks/queries/query_11.csv @@ -0,0 +1,10 @@ +1,iPad,1967 +5,iPad,97 +7,iPad,79 +6,iPad,55 +6,iPhone,37 +26,iPhone,36 +118,A500,34 +32,iPad,29 +60,iPad,22 +13,iPad,12 diff --git a/benchmarks/queries/query_12.csv b/benchmarks/queries/query_12.csv new file mode 100644 index 0000000..2dbd091 --- /dev/null +++ b/benchmarks/queries/query_12.csv @@ -0,0 +1,10 @@ +ведомосквы вместу,4943 +ведомосквы вы из,2471 +ведомосквиталия страции,2026 +ведомосковский,1686 +покеты рецепт засня,961 +рецепты сбербан,788 +авторий,705 +ведомосква,446 +ведомосквы новые водительная болгарин,411 +инстанец жизнь,391 diff --git a/benchmarks/queries/query_13.csv b/benchmarks/queries/query_13.csv new file mode 100644 index 0000000..0dc3e1d --- /dev/null +++ b/benchmarks/queries/query_13.csv @@ -0,0 +1,10 @@ +ведомосквы вместу,1381 +ведомосквы вы из,678 +ведомосквиталия страции,658 +рецепты сбербан,594 +ведомосковский,407 +инстанец жизнь,292 +покеты рецепт засня,281 +авторий,196 +рецепт блиноленские,135 +ведомосква,129 diff --git a/benchmarks/queries/query_14.csv b/benchmarks/queries/query_14.csv new file mode 100644 index 0000000..c40691f --- /dev/null +++ b/benchmarks/queries/query_14.csv @@ -0,0 +1,10 @@ +2,ведомосквы вместу,3478 +2,ведомосквы вы из,1857 +2,ведомосковский,1682 +2,ведомосквиталия страции,1434 +4,покеты рецепт засня,959 +2,рецепты сбербан,737 +3,ведомосквы вместу,660 +2,авторий,576 +3,ведомосквиталия страции,494 +4,ведомосквы вместу,442 diff --git a/benchmarks/queries/query_15.csv b/benchmarks/queries/query_15.csv new file mode 100644 index 0000000..32d99e7 --- /dev/null +++ b/benchmarks/queries/query_15.csv @@ -0,0 +1,10 @@ +1508127196834704092,1303 +3205616454965152970,949 +502693359570399458,893 +873022393995828557,876 +2256536417172705921,695 +340634745528635910,610 +72709437341035504,560 +5705194083846317709,532 +1257144732630861346,524 +4885305169967046117,516 diff --git a/benchmarks/queries/query_16.csv b/benchmarks/queries/query_16.csv new file mode 100644 index 0000000..43d129b --- /dev/null +++ b/benchmarks/queries/query_16.csv @@ -0,0 +1,10 @@ +1508127196834704092,,1303 +3205616454965152970,,949 +502693359570399458,,893 +873022393995828557,,876 +2256536417172705921,,695 +340634745528635910,,610 +72709437341035504,,560 +5705194083846317709,,532 +614605011960296602,,506 +775643969820522877,,483 diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index c56ffb3..2595e32 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,44 +1,18 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,28.86,12.14,11.25,9.60,16.44,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,51.92,30.69,30.18,29.77,32.64,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",72.78,58.04,58.82,54.30,60.23,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,74.22,77.95,67.37,52.85,124.19,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,68.50,52.63,51.18,48.12,60.06,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,60.41,44.76,44.68,44.65,45.02,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",693.40,675.22,665.84,663.16,706.04,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",31.60,29.68,29.62,28.63,30.84,ok, -8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",,,,,,failed,query_parser: unexpected trailing tokens -16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",,,,,,failed,query_parser: unexpected trailing tokens -17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",,,,,,failed,query_parser: unexpected trailing tokens -18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",,,,,,failed,executor: aggregate 'extract' is not registered -19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,,,,,,failed,query_planner: plain column SELECT items require GROUP BY -20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,,,,,,failed,query_parser: expected comparison operator in WHERE -21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,,,,,,failed,executor: aggregate '*' is not registered -24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,,,,,,failed,query_parser: unexpected trailing tokens -25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,,,,,,failed,query_parser: unexpected trailing tokens -26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",,,,,,failed,query_parser: unexpected trailing tokens -27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",,,,,,failed,query_parser: expected ')' after aggregate arguments -28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",,,,,,failed,executor: aggregate 'REGEXP_REPLACE' is not registered -29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",,,,,,failed,query_parser: expected ')' after aggregate arguments -30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected aggregate function -35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",,,,,,failed,query_parser: expected FROM -38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",,,,,,failed,query_parser: expected FROM -39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",,,,,,failed,query_parser: expected FROM -40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",,,,,,failed,query_parser: expected FROM -41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",,,,,,failed,query_parser: expected FROM -42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",,,,,,failed,executor: aggregate 'DATE_TRUNC' is not registered +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,31.90,10.09,9.93,9.63,10.88,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,47.47,29.97,29.35,28.25,32.94,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",65.71,99.09,88.86,52.07,166.55,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,102.02,63.90,64.27,59.52,67.55,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,53.78,72.75,57.36,55.74,120.53,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,64.92,47.45,47.18,44.87,50.55,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",689.46,689.40,681.74,669.80,724.31,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",32.64,28.93,28.98,28.51,29.27,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",137.21,125.35,126.21,118.63,130.37,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",181.60,185.20,182.91,177.70,197.28,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",48.49,36.32,36.14,35.74,37.27,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",52.50,40.40,40.20,38.94,42.24,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",91.76,88.94,88.69,86.63,91.77,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",100.89,97.84,96.94,94.84,102.63,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",107.95,95.51,95.43,94.33,96.84,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",175.84,169.96,169.44,164.33,176.64,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",259.40,267.56,267.52,266.79,268.43,ok, diff --git a/benchmarks/support/query_result_checks.cpp b/benchmarks/support/query_result_checks.cpp new file mode 100644 index 0000000..da89f53 --- /dev/null +++ b/benchmarks/support/query_result_checks.cpp @@ -0,0 +1,60 @@ +#include "query_result_checks.h" + +#include +#include +#include + +#include "io/csv_batch_io.h" +#include "support/ascii.h" + +std::string BatchToString(const Batch& batch) { + std::vector> rows; + AppendBatchRows(batch, rows); + + std::string result; + + for (const auto& row : rows) { + for (size_t i = 0; i < row.size(); ++i) { + result += row[i]; + if (i + 1 < row.size()) { + result += ","; + } + } + result += "\n"; + } + + return result; +} + +bool QueryUsesGroupBy(const std::string_view sql) { return ToUpperAscii(sql).find("GROUP BY") != std::string::npos; } + +static std::vector SplitRows(const std::string_view text) { + std::vector rows; + size_t start = 0; + + while (start < text.size()) { + const size_t end = text.find('\n', start); + if (end == std::string_view::npos) { + rows.emplace_back(text.substr(start)); + break; + } + rows.emplace_back(text.substr(start, end - start)); + start = end + 1; + } + + if (!rows.empty() && rows.back().empty()) { + rows.pop_back(); + } + + return rows; +} + +bool EqualAsRowMultisets(const std::string_view lhs, const std::string_view rhs) { + std::vector lhs_rows = SplitRows(lhs); + std::vector rhs_rows = SplitRows(rhs); + + std::ranges::sort(lhs_rows); + std::ranges::sort(rhs_rows); + + return lhs_rows == rhs_rows; +} diff --git a/benchmarks/support/query_result_checks.h b/benchmarks/support/query_result_checks.h new file mode 100644 index 0000000..ceea1b7 --- /dev/null +++ b/benchmarks/support/query_result_checks.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +#include "model/batch.h" + +std::string BatchToString(const Batch& batch); +bool QueryUsesGroupBy(std::string_view sql); +bool EqualAsRowMultisets(std::string_view lhs, std::string_view rhs); diff --git a/benchmarks/support/query_runners.h b/benchmarks/support/query_runners.h index f0717bd..b1ca5cf 100644 --- a/benchmarks/support/query_runners.h +++ b/benchmarks/support/query_runners.h @@ -36,16 +36,63 @@ inline ExecuteExpected RunQuery7(const Executor& executor) { "SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;"); } +inline ExecuteExpected RunQuery8(const Executor& executor) { + return executor.Execute( + "SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC " + "LIMIT 10;"); +} + +inline ExecuteExpected RunQuery9(const Executor& executor) { + return executor.Execute( + "SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), " + "COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery10(const Executor& executor) { + return executor.Execute( + "SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE " + "MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery11(const Executor& executor) { + return executor.Execute( + "SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE " + "MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery12(const Executor& executor) { + return executor.Execute( + "SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY " + "SearchPhrase ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery13(const Executor& executor) { + return executor.Execute( + "SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' " + "GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery14(const Executor& executor) { + return executor.Execute( + "SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' " + "GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery15(const Executor& executor) { + return executor.Execute("SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery16(const Executor& executor) { + return executor.Execute( + "SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC " + "LIMIT 10;"); +} + inline const std::map& GetQueryRunners() { static const std::map Runners = { - {0, RunQuery0}, - {1, RunQuery1}, - {2, RunQuery2}, - {3, RunQuery3}, - {4, RunQuery4}, - {5, RunQuery5}, - {6, RunQuery6}, - {7, RunQuery7}, + {0, RunQuery0}, {1, RunQuery1}, {2, RunQuery2}, {3, RunQuery3}, {4, RunQuery4}, {5, RunQuery5}, + {6, RunQuery6}, {7, RunQuery7}, {8, RunQuery8}, {9, RunQuery9}, {10, RunQuery10}, {11, RunQuery11}, + {12, RunQuery12}, {13, RunQuery13}, {14, RunQuery14}, {15, RunQuery15}, {16, RunQuery16}, }; return Runners; } diff --git a/include/executor/executor.h b/include/executor/executor.h index e1a0692..53da8b7 100644 --- a/include/executor/executor.h +++ b/include/executor/executor.h @@ -55,8 +55,13 @@ struct ParsedLiteral { Tokens token_type = Tokens::StringLiteral; }; +struct ColumnRef { + std::string qualifier; + std::string name; +}; + struct FilterSpec { - std::string column_name; + ColumnRef column; ComparisonKind comparison = ComparisonKind::Equal; ParsedLiteral literal; }; @@ -65,7 +70,7 @@ struct AggSpec { const AggFuncDefinition* function = nullptr; bool distinct = false; bool star = false; - std::string column_name; + ColumnRef column; std::string output_name; }; @@ -76,7 +81,7 @@ enum class SelectItemKind { struct SelectItemSpec { SelectItemKind kind = SelectItemKind::GroupKey; - std::string column_name; + ColumnRef column; AggSpec aggregate; std::string output_name; }; @@ -88,10 +93,12 @@ struct OrderBySpec { struct ParsedQuery { std::string table_name; + std::string table_alias; std::vector select_items; - std::vector group_by; + std::vector group_by; std::optional filter; std::vector order_by; + std::optional limit; }; struct PlannedAgg { @@ -137,6 +144,7 @@ struct PlannedQuery { std::vector aggregates; std::vector select_items; std::vector order_by; + std::optional limit; }; class AggState { diff --git a/include/executor/planner_utils.h b/include/executor/planner_utils.h new file mode 100644 index 0000000..f5b4b90 --- /dev/null +++ b/include/executor/planner_utils.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +#include "executor/executor.h" + +size_t FindColumnIndex(const Schema& schema, std::string_view column_name); +void ValidateColumnQualifier(const ParsedQuery& parsed, const ColumnRef& column); +std::string NormalizeLiteral(const ParsedLiteral& literal, ColumnType type); +bool SameColumnName(std::string_view lhs, std::string_view rhs); +bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs); +bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs); +bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs); diff --git a/scripts/dashboard.py b/scripts/dashboard.py index df789d7..9d4f052 100644 --- a/scripts/dashboard.py +++ b/scripts/dashboard.py @@ -20,7 +20,7 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Run the first N benchmark queries, write CSV stats, and refresh the README benchmark table." ) - parser.add_argument("--count", type=int, default=43, help="Number of first queries to benchmark.") + parser.add_argument("--count", type=int, default=17, help="Number of first queries to benchmark.") parser.add_argument("--warm-runs", type=int, default=4, help="Number of warm runs after the first measured run.") parser.add_argument( "--query-dir", @@ -214,7 +214,7 @@ def render_markdown(csv_path: pathlib.Path, rows: list[dict[str, str]], count: i for row in rows: query_file = row["query_file"] - query_label = f"[query_{row['query_id']}.sql]({query_file})" + query_label = f"[{row['query_id']}]({query_file})" status = row["status"] if row["error"]: status = f"{status}: {markdown_escape(compact_sql(row['error'], max_len=48))}" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 34d09d2..8f5ac71 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,6 +23,7 @@ add_library(columnar_engine_lib executor/executor.cpp executor/query_parser.cpp executor/query_planner.cpp + executor/planner_utils.cpp executor/aggregate_states.cpp executor/operator_utils.cpp executor/operators.cpp diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index dea2af1..886708e 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -229,24 +229,27 @@ class GroupAggOperator final : public Operator { } if (!order_by_.empty()) { - std::ranges::sort(rows, [&](const std::vector& lhs, const std::vector& rhs) { - for (const auto& order : order_by_) { - const int comparison = - CompareValues(order.value_type, lhs[order.result_column_index], rhs[order.result_column_index]); - if (comparison != 0) { - return order.descending ? comparison > 0 : comparison < 0; + std::ranges::stable_sort( + rows, [&](const std::vector& lhs, const std::vector& rhs) { + for (const auto& order : order_by_) { + const int comparison = CompareValues(order.value_type, lhs[order.result_column_index], + rhs[order.result_column_index]); + if (comparison != 0) { + return order.descending ? comparison > 0 : comparison < 0; + } } - } - for (size_t i = 0; i < select_items_.size(); ++i) { - const int comparison = CompareValues(select_items_[i].output_type, lhs[i], rhs[i]); - if (comparison != 0) { - return comparison < 0; + for (size_t i = 0; i < select_items_.size(); ++i) { + if (select_items_[i].kind != SelectItemKind::GroupKey) { + continue; + } + const int comparison = CompareValues(select_items_[i].output_type, lhs[i], rhs[i]); + if (comparison != 0) { + return comparison > 0; + } } - } - - return false; - }); + return false; + }); } Batch result(schema, rows.size()); @@ -300,6 +303,41 @@ class GroupAggOperator final : public Operator { bool returned_ = false; }; +class LimitOperator final : public Operator { + public: + LimitOperator(std::unique_ptr child, const size_t limit) : child_(std::move(child)), remaining_(limit) {} + + public: + std::optional Next() override { + if (remaining_ == 0) { + return std::nullopt; + } + + while (auto batch = child_->Next()) { + if (batch->RowsCount() <= remaining_) { + remaining_ -= batch->RowsCount(); + return batch; + } + + Batch limited(batch->GetSchema(), remaining_); + for (size_t row = 0; row < remaining_; ++row) { + for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { + limited.ColumnAt(column_index).AppendFromString(batch->ColumnAt(column_index).ValueAsString(row)); + } + } + remaining_ = 0; + return limited; + } + + remaining_ = 0; + return std::nullopt; + } + + private: + std::unique_ptr child_; + size_t remaining_; +}; + std::unique_ptr BuildPlan(const PlannedQuery& planned) { std::unique_ptr root = std::make_unique(planned.table_path, planned.projection_indexes); if (planned.filter.has_value()) { @@ -311,5 +349,8 @@ std::unique_ptr BuildPlan(const PlannedQuery& planned) { } else { root = std::make_unique(std::move(root), planned.aggregates); } + if (planned.limit.has_value()) { + root = std::make_unique(std::move(root), *planned.limit); + } return root; } diff --git a/src/executor/planner_utils.cpp b/src/executor/planner_utils.cpp new file mode 100644 index 0000000..1aa3eb2 --- /dev/null +++ b/src/executor/planner_utils.cpp @@ -0,0 +1,95 @@ +#include "executor/planner_utils.h" + +#include "support/ascii.h" + +size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { + const std::string needle = ToLowerAscii(column_name); + for (size_t i = 0; i < schema.columns.size(); ++i) { + if (ToLowerAscii(schema.columns[i].name) == needle) { + return i; + } + } + throw Error::InvalidArgument("query_planner", "unknown column '" + std::string(column_name) + "'"); +} + +static bool SameQualifier(const std::string_view lhs, const std::string_view rhs) { + return ToLowerAscii(lhs) == ToLowerAscii(rhs); +} + +void ValidateColumnQualifier(const ParsedQuery& parsed, const ColumnRef& column) { + if (column.qualifier.empty()) { + return; + } + + if (SameQualifier(column.qualifier, parsed.table_name)) { + return; + } + if (!parsed.table_alias.empty() && SameQualifier(column.qualifier, parsed.table_alias)) { + return; + } + + throw Error::InvalidArgument("query_planner", "unknown table qualifier '" + column.qualifier + "'"); +} + +static std::string UnescapeSqlString(const std::string_view text) { + if (text.size() < 2 || text.front() != '\'' || text.back() != '\'') { + throw Error::InvalidArgument("query_planner", "invalid string literal"); + } + + std::string result; + result.reserve(text.size() - 2); + + for (size_t i = 1; i + 1 < text.size(); ++i) { + if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { + result.push_back('\''); + ++i; + continue; + } + result.push_back(text[i]); + } + + return result; +} + +std::string NormalizeLiteral(const ParsedLiteral& literal, const ColumnType type) { + switch (literal.token_type) { + case Tokens::StringLiteral: + return UnescapeSqlString(literal.text); + case Tokens::NumericLiteral: + case Tokens::NameToken: + if (type == ColumnType::String && literal.token_type == Tokens::NameToken) { + return literal.text; + } + return literal.text; + default: + throw Error::InvalidArgument("query_planner", "unsupported literal type"); + } +} + +bool SameColumnName(const std::string_view lhs, const std::string_view rhs) { return ToLowerAscii(lhs) == ToLowerAscii(rhs); } + +bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs) { + if (!SameColumnName(lhs.name, rhs.name)) { + return false; + } + if (lhs.qualifier.empty() || rhs.qualifier.empty()) { + return true; + } + return SameQualifier(lhs.qualifier, rhs.qualifier); +} + +bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { + if (lhs.kind != rhs.kind) { + return false; + } + + if (lhs.kind == SelectItemKind::GroupKey) { + return SameColumnRef(lhs.column, rhs.column); + } + + return ToLowerAscii(lhs.aggregate.output_name) == ToLowerAscii(rhs.aggregate.output_name); +} + +bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { + return SameColumnName(lhs.output_name, rhs.output_name); +} diff --git a/src/executor/query_parser.cpp b/src/executor/query_parser.cpp index 1e50d6d..86a1e87 100644 --- a/src/executor/query_parser.cpp +++ b/src/executor/query_parser.cpp @@ -1,4 +1,5 @@ #include +#include #include #include "executor/executor.h" @@ -88,6 +89,44 @@ static ComparisonKind ComparisonKindFromToken(const Tokens type) { } } +static std::string FormatColumnRef(const ColumnRef& column) { + if (column.qualifier.empty()) { + return column.name; + } + return column.qualifier + "." + column.name; +} + +static ColumnRef ParseColumnRef(TokenCursor& cursor, const std::string_view message) { + ColumnRef column{ + .qualifier = {}, + .name = std::string(cursor.Consume(Tokens::NameToken, message).GetText()), + }; + + if (cursor.TryConsume(Tokens::Dot)) { + column.qualifier = std::move(column.name); + column.name = std::string(cursor.Consume(Tokens::NameToken, "expected column name after '.'").GetText()); + } + + return column; +} + +static std::optional ParseOptionalLimit(TokenCursor& cursor) { + if (!cursor.TryConsume(Tokens::Limit)) { + return std::nullopt; + } + + const Token& limit_token = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after LIMIT"); + size_t limit = 0; + const std::string_view text = limit_token.GetText(); + const char* begin = text.data(); + const char* end = begin + text.size(); + if (const auto [ptr, ec] = std::from_chars(begin, end, limit); ec != std::errc() || ptr != end) { + throw Error::InvalidArgument("query_parser", "invalid LIMIT value"); + } + + return limit; +} + static AggSpec ParseAggregate(TokenCursor& cursor) { const Token& function_token = cursor.ConsumeFunctionToken("expected aggregate function"); @@ -108,40 +147,57 @@ static AggSpec ParseAggregate(TokenCursor& cursor) { throw Error::InvalidArgument("query_parser", std::string(spec.function->canonical_name) + " does not support DISTINCT"); } - const auto& token = cursor.Consume(Tokens::NameToken, "expected column name in aggregate"); - spec.column_name = std::string(token.GetText()); + spec.column = ParseColumnRef(cursor, "expected column name in aggregate"); } cursor.Consume(Tokens::CloseBracket, "expected ')' after aggregate arguments"); - spec.output_name = FormatAgg(*spec.function, spec.column_name, spec.distinct, spec.star); + spec.output_name = FormatAgg(*spec.function, FormatColumnRef(spec.column), spec.distinct, spec.star); return spec; } -static SelectItemSpec ParseSelectItem(TokenCursor& cursor) { +static SelectItemSpec ParseSelectExpression(TokenCursor& cursor) { if (cursor.Match(Tokens::NameToken) && !cursor.Match(Tokens::OpenBracket, 1)) { - const Token& token = cursor.Consume(Tokens::NameToken, "expected column name in SELECT"); + const ColumnRef column = ParseColumnRef(cursor, "expected column name in SELECT"); return SelectItemSpec{ .kind = SelectItemKind::GroupKey, - .column_name = std::string(token.GetText()), + .column = column, .aggregate = {}, - .output_name = std::string(token.GetText()), + .output_name = column.name, }; } return SelectItemSpec{ .kind = SelectItemKind::Aggregate, - .column_name = {}, + .column = {}, .aggregate = ParseAggregate(cursor), .output_name = {}, }; } -static OrderBySpec ParseOrderByItem(TokenCursor& cursor) { - SelectItemSpec item = ParseSelectItem(cursor); - if (item.kind == SelectItemKind::Aggregate) { +static std::string ParseSelectItemAlias(TokenCursor& cursor) { + if (cursor.TryConsume(Tokens::As)) { + return std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); + } + if (cursor.Match(Tokens::NameToken)) { + return std::string(cursor.Consume(Tokens::NameToken, "expected alias").GetText()); + } + return {}; +} + +static SelectItemSpec ParseSelectItem(TokenCursor& cursor) { + SelectItemSpec item = ParseSelectExpression(cursor); + if (const std::string alias = ParseSelectItemAlias(cursor); !alias.empty()) { + item.output_name = alias; + } else if (item.kind == SelectItemKind::Aggregate) { item.output_name = item.aggregate.output_name; } + return item; +} + +static OrderBySpec ParseOrderByItem(TokenCursor& cursor) { + SelectItemSpec item = ParseSelectExpression(cursor); + item.output_name = item.kind == SelectItemKind::Aggregate ? item.aggregate.output_name : item.column.name; bool descending = false; if (cursor.Match(Tokens::NameToken)) { @@ -176,19 +232,17 @@ ParsedQuery ParseQuery(const std::string_view query) { parsed.select_items.push_back(ParseSelectItem(cursor)); } - for (auto& item : parsed.select_items) { - if (item.kind == SelectItemKind::Aggregate) { - item.output_name = item.aggregate.output_name; - } - } - cursor.Consume(Tokens::From, "expected FROM"); parsed.table_name = std::string(cursor.Consume(Tokens::NameToken, "expected table name").GetText()); + if (cursor.TryConsume(Tokens::As)) { + parsed.table_alias = std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); + } else if (cursor.Match(Tokens::NameToken)) { + parsed.table_alias = std::string(cursor.Consume(Tokens::NameToken, "expected table alias").GetText()); + } if (cursor.TryConsume(Tokens::Where)) { FilterSpec filter; - filter.column_name = - std::string(cursor.Consume(Tokens::NameToken, "expected column name after WHERE").GetText()); + filter.column = ParseColumnRef(cursor, "expected column name after WHERE"); const Tokens comparison_token = cursor.Peek().GetType(); filter.comparison = ComparisonKindFromToken(comparison_token); @@ -221,10 +275,9 @@ ParsedQuery ParseQuery(const std::string_view query) { if (cursor.TryConsume(Tokens::Group)) { cursor.Consume(Tokens::By, "expected BY after GROUP"); - parsed.group_by.push_back(std::string(cursor.Consume(Tokens::NameToken, "expected column name in GROUP BY").GetText())); + parsed.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); while (cursor.TryConsume(Tokens::Comma)) { - parsed.group_by.push_back( - std::string(cursor.Consume(Tokens::NameToken, "expected column name in GROUP BY").GetText())); + parsed.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); } } @@ -236,6 +289,7 @@ ParsedQuery ParseQuery(const std::string_view query) { } } + parsed.limit = ParseOptionalLimit(cursor); cursor.TryConsume(Tokens::Semicolon); cursor.Consume(Tokens::EndOfInput, "unexpected trailing tokens"); diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index d25e43b..9de1d36 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -2,70 +2,10 @@ #include "executor/executor.h" #include "executor/operator_utils.h" +#include "executor/planner_utils.h" #include "io/columnar_batch_io.h" #include "support/ascii.h" -static size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { - const std::string needle = ToLowerAscii(column_name); - for (size_t i = 0; i < schema.columns.size(); ++i) { - if (ToLowerAscii(schema.columns[i].name) == needle) { - return i; - } - } - throw Error::InvalidArgument("query_planner", "unknown column '" + std::string(column_name) + "'"); -} - -static std::string UnescapeSqlString(const std::string_view text) { - if (text.size() < 2 || text.front() != '\'' || text.back() != '\'') { - throw Error::InvalidArgument("query_planner", "invalid string literal"); - } - - std::string result; - result.reserve(text.size() - 2); - - for (size_t i = 1; i + 1 < text.size(); ++i) { - if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { - result.push_back('\''); - ++i; - continue; - } - result.push_back(text[i]); - } - - return result; -} - -static std::string NormalizeLiteral(const ParsedLiteral& literal, const ColumnType type) { - switch (literal.token_type) { - case Tokens::StringLiteral: - return UnescapeSqlString(literal.text); - case Tokens::NumericLiteral: - case Tokens::NameToken: - if (type == ColumnType::String && literal.token_type == Tokens::NameToken) { - return literal.text; - } - return literal.text; - default: - throw Error::InvalidArgument("query_planner", "unsupported literal type"); - } -} - -static bool SameColumnName(const std::string_view lhs, const std::string_view rhs) { - return ToLowerAscii(lhs) == ToLowerAscii(rhs); -} - -static bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { - if (lhs.kind != rhs.kind) { - return false; - } - - if (lhs.kind == SelectItemKind::GroupKey) { - return SameColumnName(lhs.column_name, rhs.column_name); - } - - return ToLowerAscii(lhs.aggregate.output_name) == ToLowerAscii(rhs.aggregate.output_name); -} - PlannedQuery PlanQuery(const ParsedQuery& parsed, const std::unordered_map& tables) { PlannedQuery planned; @@ -90,7 +30,8 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, }; if (parsed.filter.has_value()) { - const size_t source_index = FindColumnIndex(schema, parsed.filter->column_name); + ValidateColumnQualifier(parsed, parsed.filter->column); + const size_t source_index = FindColumnIndex(schema, parsed.filter->column.name); const ColumnType column_type = schema.columns[source_index].type; planned.filter = PlannedFilter{ @@ -102,8 +43,9 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, } planned.group_keys.reserve(parsed.group_by.size()); - for (const std::string& group_column : parsed.group_by) { - const size_t source_index = FindColumnIndex(schema, group_column); + for (const ColumnRef& group_column : parsed.group_by) { + ValidateColumnQualifier(parsed, group_column); + const size_t source_index = FindColumnIndex(schema, group_column.name); planned.group_keys.push_back(PlannedGroupKey{ .column_index = register_projection(source_index), .column_type = schema.columns[source_index].type, @@ -128,19 +70,20 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, throw Error::Unsupported("query_planner", "plain column SELECT items require GROUP BY"); } + ValidateColumnQualifier(parsed, item.column); const auto it = std::ranges::find_if(planned.group_keys, [&](const PlannedGroupKey& group_key) { - return SameColumnName(group_key.output_name, item.column_name); + return SameColumnName(group_key.output_name, item.column.name); }); if (it == planned.group_keys.end()) { throw Error::InvalidArgument("query_planner", - "column '" + item.column_name + "' must appear in GROUP BY"); + "column '" + item.column.name + "' must appear in GROUP BY"); } planned.select_items.push_back(PlannedSelectItem{ .kind = SelectItemKind::GroupKey, .index = static_cast(std::distance(planned.group_keys.begin(), it)), .output_type = it->column_type, - .output_name = it->output_name, + .output_name = item.output_name, }); continue; } @@ -152,17 +95,18 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, .star = aggregate.star, .column_index = std::nullopt, .input_type = ColumnType::String, - .output_name = aggregate.output_name, + .output_name = item.output_name, }; if (!aggregate.star) { - const size_t source_index = FindColumnIndex(schema, aggregate.column_name); + ValidateColumnQualifier(parsed, aggregate.column); + const size_t source_index = FindColumnIndex(schema, aggregate.column.name); planned_aggregate.column_index = register_projection(source_index); planned_aggregate.input_type = schema.columns[source_index].type; if (!AggSupportsInputType(*aggregate.function, planned_aggregate.input_type)) { throw Error::Unsupported("query_planner", "aggregate " + std::string(aggregate.function->canonical_name) + - " does not support column '" + aggregate.column_name + "'"); + " does not support column '" + aggregate.column.name + "'"); } } @@ -172,7 +116,7 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, .kind = SelectItemKind::Aggregate, .index = aggregate_index, .output_type = AggregateOutputType(planned.aggregates.back()), - .output_name = aggregate.output_name, + .output_name = item.output_name, }); } @@ -183,7 +127,7 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, planned.order_by.reserve(parsed.order_by.size()); for (const auto& order_item : parsed.order_by) { const auto it = std::ranges::find_if(parsed.select_items, [&](const SelectItemSpec& select_item) { - return SameSelectItem(select_item, order_item.item); + return SameSelectItem(select_item, order_item.item) || SameOutputName(select_item, order_item.item); }); if (it == parsed.select_items.end()) { throw Error::InvalidArgument("query_planner", "ORDER BY expression must appear in SELECT"); @@ -197,6 +141,8 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, }); } + planned.limit = parsed.limit; + if (planned.projection_indexes.empty() && !schema.columns.empty()) { planned.projection_indexes.push_back(0); } diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index 3a04eba..a17bbda 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -17,16 +18,20 @@ static Batch BuildHitsTable(const std::string_view query) { {"AdvEngineID", "int64"}, {"ResolutionWidth", "int32"}, {"UserID", "int64"}, + {"RegionID", "int32"}, + {"MobilePhone", "bool"}, + {"MobilePhoneModel", "string"}, + {"SearchEngineID", "int32"}, {"SearchPhrase", "string"}, {"EventDate", "date"}, }); WriteRows(data_file.Path(), { - {"0", "100", "1", "alpha", "2024-01-05"}, - {"7", "200", "2", "beta", "2024-01-03"}, - {"3", "301", "2", "alpha", "2024-01-07"}, - {"7", "500", "5", "gamma", "2024-01-01"}, - {"1", "400", "11", "delta", "2024-01-09"}, + {"0", "100", "1", "10", "false", "", "1", "alpha", "2024-01-05"}, + {"7", "200", "2", "20", "true", "iPhone", "1", "beta", "2024-01-03"}, + {"3", "301", "2", "10", "true", "Pixel", "2", "alpha", "2024-01-07"}, + {"7", "500", "5", "20", "true", "iPhone", "1", "gamma", "2024-01-01"}, + {"1", "400", "11", "30", "false", "Nokia", "2", "delta", "2024-01-09"}, }); ConvertCsvToColumnar(schema_file.Path(), data_file.Path(), columnar_file.Path(), 2); @@ -53,11 +58,82 @@ TEST(executor, groups_rows_and_orders_by_aggregate_descending) { "SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;"); EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); - EXPECT_EQ(BatchRows(batch), (std::vector>{ - {"7", "2"}, - {"1", "1"}, - {"3", "1"}, - })); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"7", "2"}, + {"1", "1"}, + {"3", "1"}, + }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); + std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_from_with_as_alias_and_qualified_columns) { + const Batch batch = BuildHitsTable( + "SELECT h.AdvEngineID, COUNT(*) FROM hits AS h WHERE h.AdvEngineID <> 0 GROUP BY h.AdvEngineID ORDER BY " + "COUNT(*) DESC;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"7", "2"}, + {"1", "1"}, + {"3", "1"}, + }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); + std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_from_with_bare_alias) { + const Batch batch = BuildHitsTable( + "SELECT h.AdvEngineID, COUNT(*) FROM hits h WHERE h.AdvEngineID <> 0 GROUP BY h.AdvEngineID ORDER BY " + "COUNT(*) DESC;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"7", "2"}, + {"1", "1"}, + {"3", "1"}, + }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); + std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_select_alias_order_by_alias_and_limit) { + const Batch batch = BuildHitsTable( + "SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 2;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"RegionID", "u"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"10", "2"}, + {"20", "2"}, + }; + std::ranges::sort(actual_rows); + std::ranges::sort(expected_rows); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_multiple_aggregates_with_alias_and_limit) { + const Batch batch = BuildHitsTable( + "SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) " + "FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 2;"); + + EXPECT_EQ(BatchColumnNames(batch), + (std::vector{"RegionID", "SUM(AdvEngineID)", "c", "AVG(ResolutionWidth)", + "COUNT(DISTINCT UserID)"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"10", "3", "2", "200", "2"}, + {"20", "14", "2", "350", "2"}, + }; + std::ranges::sort(actual_rows); + std::ranges::sort(expected_rows); + EXPECT_EQ(actual_rows, expected_rows); } TEST(executor, executes_basic_aggregate_queries) { From 45afb419d5d039842f6c8eac2f4eca8610b9d131 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Tue, 5 May 2026 23:32:45 +0300 Subject: [PATCH 13/29] chore: restore task-7 snapshot for review --- .clang-format | 10 + .clang-tidy | 35 + .coderabbit.yaml | 30 + .editorconfig | 10 + .github/apt-packages.txt | 7 + .github/workflows/cmake-single-platform.yml | 85 + .gitignore | 111 + CMakeLists.txt | 75 + README.md | 28 + benchmarks/CMakeLists.txt | 37 + benchmarks/benchmark_csv_to_columnar.cpp | 121 + benchmarks/benchmark_queries.cpp | 116 + benchmarks/queries/README.md | 1 + benchmarks/queries/queries.sql | 43 + benchmarks/queries/query_0.csv | 1 + benchmarks/queries/query_0.sql | 1 + benchmarks/queries/query_1.csv | 1 + benchmarks/queries/query_1.sql | 1 + benchmarks/queries/query_10.csv | 10 + benchmarks/queries/query_10.sql | 1 + benchmarks/queries/query_11.csv | 10 + benchmarks/queries/query_11.sql | 1 + benchmarks/queries/query_12.csv | 10 + benchmarks/queries/query_12.sql | 1 + benchmarks/queries/query_13.csv | 10 + benchmarks/queries/query_13.sql | 1 + benchmarks/queries/query_14.csv | 10 + benchmarks/queries/query_14.sql | 1 + benchmarks/queries/query_15.csv | 10 + benchmarks/queries/query_15.sql | 1 + benchmarks/queries/query_16.csv | 10 + benchmarks/queries/query_16.sql | 1 + benchmarks/queries/query_17.sql | 1 + benchmarks/queries/query_18.sql | 1 + benchmarks/queries/query_19.sql | 1 + benchmarks/queries/query_2.csv | 1 + benchmarks/queries/query_2.sql | 1 + benchmarks/queries/query_20.sql | 1 + benchmarks/queries/query_21.sql | 1 + benchmarks/queries/query_22.sql | 1 + benchmarks/queries/query_23.sql | 1 + benchmarks/queries/query_24.sql | 1 + benchmarks/queries/query_25.sql | 1 + benchmarks/queries/query_26.sql | 1 + benchmarks/queries/query_27.sql | 1 + benchmarks/queries/query_28.sql | 1 + benchmarks/queries/query_29.sql | 1 + benchmarks/queries/query_3.csv | 1 + benchmarks/queries/query_3.sql | 1 + benchmarks/queries/query_30.sql | 1 + benchmarks/queries/query_31.sql | 1 + benchmarks/queries/query_32.sql | 1 + benchmarks/queries/query_33.sql | 1 + benchmarks/queries/query_34.sql | 1 + benchmarks/queries/query_35.sql | 1 + benchmarks/queries/query_36.sql | 1 + benchmarks/queries/query_37.sql | 1 + benchmarks/queries/query_38.sql | 1 + benchmarks/queries/query_39.sql | 1 + benchmarks/queries/query_4.csv | 1 + benchmarks/queries/query_4.sql | 1 + benchmarks/queries/query_40.sql | 1 + benchmarks/queries/query_41.sql | 1 + benchmarks/queries/query_42.sql | 1 + benchmarks/queries/query_5.csv | 1 + benchmarks/queries/query_5.sql | 1 + benchmarks/queries/query_6.csv | 1 + benchmarks/queries/query_6.sql | 1 + benchmarks/queries/query_7.csv | 5 + benchmarks/queries/query_7.sql | 1 + benchmarks/queries/query_8.csv | 10 + benchmarks/queries/query_8.sql | 1 + benchmarks/queries/query_9.csv | 10 + benchmarks/queries/query_9.sql | 1 + benchmarks/results/readme_benchmarks.csv | 18 + benchmarks/support/query_result_checks.cpp | 60 + benchmarks/support/query_result_checks.h | 10 + benchmarks/support/query_runners.h | 98 + cmake/dependencies_package.cmake | 14 + cmake/sanitizers.cmake | 59 + include/convert/csv_columnar.h | 9 + include/executor/aggregate_function.h | 43 + include/executor/executor.h | 167 ++ include/executor/operator_utils.h | 9 + include/executor/planner_utils.h | 14 + include/io/batch_io.h | 40 + include/io/columnar_batch_io.h | 63 + include/io/csv.h | 53 + include/io/csv_batch_io.h | 52 + include/io/fileio.h | 34 + include/io/stream.h | 29 + include/model/batch.h | 39 + include/model/column.h | 42 + include/model/column_boolean.h | 13 + include/model/column_character.h | 13 + include/model/column_date.h | 13 + include/model/column_int128.h | 14 + include/model/column_int16.h | 13 + include/model/column_int32.h | 13 + include/model/column_int64.h | 13 + include/model/column_string.h | 13 + include/model/column_timestamp.h | 13 + include/model/fixed_column.h | 56 + include/model/metadata.h | 25 + include/model/schema.h | 35 + include/model/variant_column.h | 92 + include/sql_parser/tokenizer.h | 322 +++ include/support/ascii.h | 7 + include/support/error.h | 70 + include/support/int128.h | 22 + include/support/parsing.h | 25 + include/testing/executor_test_utils.h | 40 + include/testing/temp_file.h | 50 + include/tl/expected.hpp | 2475 +++++++++++++++++++ script/build.sh | 25 + script/convert.sh | 36 + script/run_query.sh | 62 + script/setup.sh | 37 + scripts/Dockerfile | 13 + scripts/action.py | 258 ++ scripts/commands.sh | 48 + scripts/dashboard.py | 298 +++ scripts/env.sh | 13 + scripts/install_dependencies.sh | 26 + scripts/sample_input.columnar | Bin 0 -> 154 bytes scripts/sample_input.csv | 3 + scripts/sample_output.csv | 3 + scripts/schema_input.csv | 3 + src/CMakeLists.txt | 37 + src/app/main.cpp | 138 ++ src/convert/csv_columnar.cpp | 44 + src/executor/aggregate_states.cpp | 342 +++ src/executor/executor.cpp | 27 + src/executor/operator_utils.cpp | 83 + src/executor/operators.cpp | 356 +++ src/executor/planner_utils.cpp | 95 + src/executor/query_parser.cpp | 297 +++ src/executor/query_planner.cpp | 151 ++ src/io/batch_io.cpp | 323 +++ src/io/csv.cpp | 196 ++ src/io/fileio.cpp | 133 + src/io/stream.cpp | 23 + src/model/batch.cpp | 86 + src/model/column.cpp | 46 + src/model/column_boolean.cpp | 7 + src/model/column_character.cpp | 7 + src/model/column_date.cpp | 7 + src/model/column_int128.cpp | 7 + src/model/column_int16.cpp | 7 + src/model/column_int32.cpp | 7 + src/model/column_int64.cpp | 7 + src/model/column_string.cpp | 5 + src/model/column_timestamp.cpp | 7 + src/model/metadata.cpp | 100 + src/model/schema.cpp | 141 ++ src/sql_parser/tokenizer.cpp | 174 ++ src/sql_parser/tokenizer_factory.cpp | 130 + src/sql_parser/tokenizer_tokens.cpp | 133 + src/support/ascii.cpp | 21 + src/support/error.cpp | 107 + src/support/parsing.cpp | 315 +++ tests/CMakeLists.txt | 16 + tests/test_batch.cpp | 277 +++ tests/test_columnar.cpp | 208 ++ tests/test_columns.cpp | 138 ++ tests/test_csv.cpp | 65 + tests/test_executor.cpp | 203 ++ tests/test_fileio.cpp | 153 ++ tests/test_stream.cpp | 20 + tests/test_tokenizer.cpp | 117 + 170 files changed, 10625 insertions(+) create mode 100644 .clang-format create mode 100644 .clang-tidy create mode 100644 .coderabbit.yaml create mode 100644 .editorconfig create mode 100644 .github/apt-packages.txt create mode 100644 .github/workflows/cmake-single-platform.yml create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 README.md create mode 100644 benchmarks/CMakeLists.txt create mode 100644 benchmarks/benchmark_csv_to_columnar.cpp create mode 100644 benchmarks/benchmark_queries.cpp create mode 100644 benchmarks/queries/README.md create mode 100644 benchmarks/queries/queries.sql create mode 100644 benchmarks/queries/query_0.csv create mode 100644 benchmarks/queries/query_0.sql create mode 100644 benchmarks/queries/query_1.csv create mode 100644 benchmarks/queries/query_1.sql create mode 100644 benchmarks/queries/query_10.csv create mode 100644 benchmarks/queries/query_10.sql create mode 100644 benchmarks/queries/query_11.csv create mode 100644 benchmarks/queries/query_11.sql create mode 100644 benchmarks/queries/query_12.csv create mode 100644 benchmarks/queries/query_12.sql create mode 100644 benchmarks/queries/query_13.csv create mode 100644 benchmarks/queries/query_13.sql create mode 100644 benchmarks/queries/query_14.csv create mode 100644 benchmarks/queries/query_14.sql create mode 100644 benchmarks/queries/query_15.csv create mode 100644 benchmarks/queries/query_15.sql create mode 100644 benchmarks/queries/query_16.csv create mode 100644 benchmarks/queries/query_16.sql create mode 100644 benchmarks/queries/query_17.sql create mode 100644 benchmarks/queries/query_18.sql create mode 100644 benchmarks/queries/query_19.sql create mode 100644 benchmarks/queries/query_2.csv create mode 100644 benchmarks/queries/query_2.sql create mode 100644 benchmarks/queries/query_20.sql create mode 100644 benchmarks/queries/query_21.sql create mode 100644 benchmarks/queries/query_22.sql create mode 100644 benchmarks/queries/query_23.sql create mode 100644 benchmarks/queries/query_24.sql create mode 100644 benchmarks/queries/query_25.sql create mode 100644 benchmarks/queries/query_26.sql create mode 100644 benchmarks/queries/query_27.sql create mode 100644 benchmarks/queries/query_28.sql create mode 100644 benchmarks/queries/query_29.sql create mode 100644 benchmarks/queries/query_3.csv create mode 100644 benchmarks/queries/query_3.sql create mode 100644 benchmarks/queries/query_30.sql create mode 100644 benchmarks/queries/query_31.sql create mode 100644 benchmarks/queries/query_32.sql create mode 100644 benchmarks/queries/query_33.sql create mode 100644 benchmarks/queries/query_34.sql create mode 100644 benchmarks/queries/query_35.sql create mode 100644 benchmarks/queries/query_36.sql create mode 100644 benchmarks/queries/query_37.sql create mode 100644 benchmarks/queries/query_38.sql create mode 100644 benchmarks/queries/query_39.sql create mode 100644 benchmarks/queries/query_4.csv create mode 100644 benchmarks/queries/query_4.sql create mode 100644 benchmarks/queries/query_40.sql create mode 100644 benchmarks/queries/query_41.sql create mode 100644 benchmarks/queries/query_42.sql create mode 100644 benchmarks/queries/query_5.csv create mode 100644 benchmarks/queries/query_5.sql create mode 100644 benchmarks/queries/query_6.csv create mode 100644 benchmarks/queries/query_6.sql create mode 100644 benchmarks/queries/query_7.csv create mode 100644 benchmarks/queries/query_7.sql create mode 100644 benchmarks/queries/query_8.csv create mode 100644 benchmarks/queries/query_8.sql create mode 100644 benchmarks/queries/query_9.csv create mode 100644 benchmarks/queries/query_9.sql create mode 100644 benchmarks/results/readme_benchmarks.csv create mode 100644 benchmarks/support/query_result_checks.cpp create mode 100644 benchmarks/support/query_result_checks.h create mode 100644 benchmarks/support/query_runners.h create mode 100644 cmake/dependencies_package.cmake create mode 100644 cmake/sanitizers.cmake create mode 100644 include/convert/csv_columnar.h create mode 100644 include/executor/aggregate_function.h create mode 100644 include/executor/executor.h create mode 100644 include/executor/operator_utils.h create mode 100644 include/executor/planner_utils.h create mode 100644 include/io/batch_io.h create mode 100644 include/io/columnar_batch_io.h create mode 100644 include/io/csv.h create mode 100644 include/io/csv_batch_io.h create mode 100644 include/io/fileio.h create mode 100644 include/io/stream.h create mode 100644 include/model/batch.h create mode 100644 include/model/column.h create mode 100644 include/model/column_boolean.h create mode 100644 include/model/column_character.h create mode 100644 include/model/column_date.h create mode 100644 include/model/column_int128.h create mode 100644 include/model/column_int16.h create mode 100644 include/model/column_int32.h create mode 100644 include/model/column_int64.h create mode 100644 include/model/column_string.h create mode 100644 include/model/column_timestamp.h create mode 100644 include/model/fixed_column.h create mode 100644 include/model/metadata.h create mode 100644 include/model/schema.h create mode 100644 include/model/variant_column.h create mode 100644 include/sql_parser/tokenizer.h create mode 100644 include/support/ascii.h create mode 100644 include/support/error.h create mode 100644 include/support/int128.h create mode 100644 include/support/parsing.h create mode 100644 include/testing/executor_test_utils.h create mode 100644 include/testing/temp_file.h create mode 100644 include/tl/expected.hpp create mode 100755 script/build.sh create mode 100755 script/convert.sh create mode 100755 script/run_query.sh create mode 100755 script/setup.sh create mode 100644 scripts/Dockerfile create mode 100644 scripts/action.py create mode 100755 scripts/commands.sh create mode 100644 scripts/dashboard.py create mode 100644 scripts/env.sh create mode 100755 scripts/install_dependencies.sh create mode 100644 scripts/sample_input.columnar create mode 100644 scripts/sample_input.csv create mode 100644 scripts/sample_output.csv create mode 100644 scripts/schema_input.csv create mode 100644 src/CMakeLists.txt create mode 100644 src/app/main.cpp create mode 100644 src/convert/csv_columnar.cpp create mode 100644 src/executor/aggregate_states.cpp create mode 100644 src/executor/executor.cpp create mode 100644 src/executor/operator_utils.cpp create mode 100644 src/executor/operators.cpp create mode 100644 src/executor/planner_utils.cpp create mode 100644 src/executor/query_parser.cpp create mode 100644 src/executor/query_planner.cpp create mode 100644 src/io/batch_io.cpp create mode 100644 src/io/csv.cpp create mode 100644 src/io/fileio.cpp create mode 100644 src/io/stream.cpp create mode 100644 src/model/batch.cpp create mode 100644 src/model/column.cpp create mode 100644 src/model/column_boolean.cpp create mode 100644 src/model/column_character.cpp create mode 100644 src/model/column_date.cpp create mode 100644 src/model/column_int128.cpp create mode 100644 src/model/column_int16.cpp create mode 100644 src/model/column_int32.cpp create mode 100644 src/model/column_int64.cpp create mode 100644 src/model/column_string.cpp create mode 100644 src/model/column_timestamp.cpp create mode 100644 src/model/metadata.cpp create mode 100644 src/model/schema.cpp create mode 100644 src/sql_parser/tokenizer.cpp create mode 100644 src/sql_parser/tokenizer_factory.cpp create mode 100644 src/sql_parser/tokenizer_tokens.cpp create mode 100644 src/support/ascii.cpp create mode 100644 src/support/error.cpp create mode 100644 src/support/parsing.cpp create mode 100644 tests/CMakeLists.txt create mode 100644 tests/test_batch.cpp create mode 100644 tests/test_columnar.cpp create mode 100644 tests/test_columns.cpp create mode 100644 tests/test_csv.cpp create mode 100644 tests/test_executor.cpp create mode 100644 tests/test_fileio.cpp create mode 100644 tests/test_stream.cpp create mode 100644 tests/test_tokenizer.cpp diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..46280a9 --- /dev/null +++ b/.clang-format @@ -0,0 +1,10 @@ +BasedOnStyle: Google +IndentWidth: 4 +AccessModifierOffset: -1 +ColumnLimit: 120 +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +DerivePointerAlignment: true +KeepEmptyLinesAtTheStartOfBlocks: true +SortIncludes: true \ No newline at end of file diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..26f6e6d --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,35 @@ +--- +Checks: '-*,cppcoreguidelines-avoid-goto,cppcoreguidelines-pro-type-const-cast, google-readability-casting, modernize-replace-random-shuffle, readability-braces-around-statements, readability-container-size-empty, readability-redundant-control-flow, readability-redundant-string-init, modernize-use-nullptr, readability-identifier-naming, google-build-using-namespace' +HeaderFilterRegex: '\.h$' +WarningsAsErrors: '*' +CheckOptions: + - key: readability-identifier-naming.NamespaceCase + value: lower_case + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.TypedefCase + value: CamelCase + - key: readability-identifier-naming.TypedefIgnoredRegexp + value: (allocator_type|size_type|key_type|value_type|mapped_type|difference_type|pointer|const_pointer|reference|const_reference|iterator_category|const_iterator|iterator|reverse_iterator|const_reverse_iterator|key_compare) + - key: readability-identifier-naming.TypeAliasCase + value: CamelCase + - key: readability-identifier-naming.PrivateMemberSuffix + value: '_' + - key: readability-identifier-naming.StructCase + value: CamelCase + - key: readability-identifier-naming.FunctionCase + value: CamelCase + - key: readability-identifier-naming.VariableCase + value: lower_case + - key: readability-identifier-naming.PrivateMemberCase + value: lower_case + - key: readability-identifier-naming.ParameterCase + value: lower_case + - key: readability-identifier-naming.GlobalConstantCase + value: CamelCase + - key: readability-identifier-naming.StaticConstantCase + value: CamelCase + - key: readability-identifier-naming.ConstexprVariableCase + value: CamelCase + - key: google-runtime-int.TypeSuffix + value: _t \ No newline at end of file diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 0000000..a541175 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,30 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +language: "ru-RU" +early_access: false + +reviews: + auto_review: + enabled: true + drafts: false + + path_filters: + - "src/**" + - "include/**" + - "CMakeLists.txt" + - "**/*.cmake" + - ".github/workflows/**" + - "!build/**" + + path_instructions: + - path: "**/*.{h,hpp,cpp,cc,cxx}" + instructions: | + Проверь: UB/границы/инициализация, RAII, обработку ошибок, + исключения, const-correctness, сложность, читаемость. + Если меняется логика — попроси тесты. + - path: "**/*CMakeLists.txt" + instructions: | + Проверь корректность target_* команд, include dirs, линковку, + и что сборка проходит на чистой машине. + +chat: + auto_reply: true diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4d761dd --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +trim_trailing_whitespace = false diff --git a/.github/apt-packages.txt b/.github/apt-packages.txt new file mode 100644 index 0000000..cad3f63 --- /dev/null +++ b/.github/apt-packages.txt @@ -0,0 +1,7 @@ +ninja-build +cmake +libgtest-dev +clang-18 +libclang-rt-18-dev +clang-format +clang-tidy diff --git a/.github/workflows/cmake-single-platform.yml b/.github/workflows/cmake-single-platform.yml new file mode 100644 index 0000000..f414afd --- /dev/null +++ b/.github/workflows/cmake-single-platform.yml @@ -0,0 +1,85 @@ +name: Build and test + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Fetch dependencies + shell: bash + run: scripts/install_dependencies.sh + + - name: Configure CMake + run: | + mkdir -p build + cd build + cmake -GNinja -DCMAKE_BUILD_TYPE=Debug .. + + - name: Build + run: | + cd build + ninja + + - name: Test + run: | + cd build + ./tests/test_batch + ./tests/test_columnar + ./tests/test_columns + ./tests/test_csv + ./tests/test_fileio + ./tests/test_stream + ./tests/test_tokenizer + + - name: Benchmark Queries + run: | + cd benchmarks + ../build/benchmarks/benchmark_queries + + sanitizers: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Fetch dependencies + shell: bash + run: scripts/install_dependencies.sh + + - name: Configure CMake With Sanitizers + env: + CC: clang-18 + CXX: clang++-18 + run: | + mkdir -p build-sanitizers + cd build-sanitizers + cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DCOLUMNAR_SANITIZERS=address,undefined .. + + - name: Build With Sanitizers + run: | + cd build-sanitizers + ninja + + - name: Test With Sanitizers + env: + ASAN_OPTIONS: detect_leaks=1:strict_string_checks=1:check_initialization_order=1 + UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 + run: | + cd build-sanitizers + ctest --output-on-failure + + - name: Benchmark Queries With Sanitizers + env: + ASAN_OPTIONS: detect_leaks=1:strict_string_checks=1:check_initialization_order=1 + UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 + run: | + cd benchmarks + ../build-sanitizers/benchmarks/benchmark_queries diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3eef705 --- /dev/null +++ b/.gitignore @@ -0,0 +1,111 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Linker files +*.ilk + +# Debugger Files +*.pdb + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# Debug information files +*.dwo + +.DS_Store + + +# CMake +.cmake/ +CMakeFiles/ +CMakeCache.txt +cmake_install.cmake +CTestTestfile.cmake +CPackConfig.cmake +CPackSourceConfig.cmake +Makefile +Testing/Temporary + +# Ninja +.ninja_deps +.ninja_log +build.ninja + +# Build +build +build-release +build-sanitizers +build-argparse +build-clean +build-tests +compile_commands.json +src/*.cmake +src/columnar_engine +_deps +.deps + +# Tests +scripts/__pycache__ +tests/*.cmake +tests/test_fileio +tests/test_foo +tests/test_columnar +tests/test_csv +tests/test_batch +tests/test_columns +tests/test_stream +tests/test_executor +tests/test_tokenizer + +# Examples +examples/*.cmake +examples/example_fileio +examples/example_columnar +examples/example_csv +examples/example_batch +examples/example_columns + +examples/out_batch +examples/out_columnar + +# IDE +.vscode +.idea +.venv + +# benchmarks +benchmarks/hits_sample.csv +benchmarks/hits_sample_new.csv +benchmarks/schema_sample.csv +benchmarks/schema_sample_new.csv +benchmarks/hits_sample.columnar +benchmarks/benchmark_csv_to_columnar +benchmarks/benchmark_queries diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..eba6976 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,75 @@ +cmake_minimum_required(VERSION 3.25) + +project(columnar-engine) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set_property(GLOBAL PROPERTY USE_FOLDERS ON) + +option(ENABLE_TESTS "Build tests" ON) +option(ENABLE_BENCHMARKS "Build benchmarks" ON) + +include(cmake/dependencies_package.cmake) +include(cmake/sanitizers.cmake) + +set(COLUMNAR_INCLUDE_DIRS + ${PROJECT_SOURCE_DIR}/include +) + +add_compile_options(-Wall -Wextra -Werror -Wpedantic -O2) + +set(COLUMNAR_SANITIZERS "" CACHE STRING "Comma or semicolon separated list of sanitizers: address,undefined,leak,thread,memory") +columnar_enable_sanitizers("${COLUMNAR_SANITIZERS}") + +set(COLUMNAR_SANITIZER_TARGET_BUILD_DIR "${PROJECT_SOURCE_DIR}/build-sanitizers" CACHE PATH + "Build directory used by the test_sanitizers target") +set(COLUMNAR_SANITIZER_TARGET_SANITIZERS "address,undefined" CACHE STRING + "Sanitizers used by the test_sanitizers target") + +add_subdirectory(src) + +if (ENABLE_BENCHMARKS) + add_subdirectory(benchmarks) +endif () + +if (ENABLE_TESTS) + enable_testing() + add_subdirectory(tests) +endif () + +set(columnar_sanitizer_configure_args + -S ${PROJECT_SOURCE_DIR} + -B ${COLUMNAR_SANITIZER_TARGET_BUILD_DIR} + -G ${CMAKE_GENERATOR} + -DENABLE_TESTS=ON + -DCOLUMNAR_SANITIZERS=${COLUMNAR_SANITIZER_TARGET_SANITIZERS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} +) + +if (CMAKE_GENERATOR_PLATFORM) + list(APPEND columnar_sanitizer_configure_args -A ${CMAKE_GENERATOR_PLATFORM}) +endif () + +if (CMAKE_GENERATOR_TOOLSET) + list(APPEND columnar_sanitizer_configure_args -T ${CMAKE_GENERATOR_TOOLSET}) +endif () + +if (CMAKE_CONFIGURATION_TYPES) + add_custom_target(test_sanitizers + COMMAND ${CMAKE_COMMAND} ${columnar_sanitizer_configure_args} + COMMAND ${CMAKE_COMMAND} --build ${COLUMNAR_SANITIZER_TARGET_BUILD_DIR} --config Debug + COMMAND ${CMAKE_CTEST_COMMAND} --test-dir ${COLUMNAR_SANITIZER_TARGET_BUILD_DIR} -C Debug --output-on-failure + USES_TERMINAL + COMMENT "Configure, build and run tests with sanitizers" + ) +else () + add_custom_target(test_sanitizers + COMMAND ${CMAKE_COMMAND} ${columnar_sanitizer_configure_args} -DCMAKE_BUILD_TYPE=Debug + COMMAND ${CMAKE_COMMAND} --build ${COLUMNAR_SANITIZER_TARGET_BUILD_DIR} + COMMAND ${CMAKE_CTEST_COMMAND} --test-dir ${COLUMNAR_SANITIZER_TARGET_BUILD_DIR} --output-on-failure + USES_TERMINAL + COMMENT "Configure, build and run tests with sanitizers" + ) +endif () diff --git a/README.md b/README.md new file mode 100644 index 0000000..be0e72b --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +# columnar-engine + +Колононый движок с нуля (in progress) + + +## Dashboard + + +| Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| [0](benchmarks/queries/query_0.sql) | 31.90 | 10.09 | 9.93 | 9.63 | 10.88 | ok | +| [1](benchmarks/queries/query_1.sql) | 47.47 | 29.97 | 29.35 | 28.25 | 32.94 | ok | +| [2](benchmarks/queries/query_2.sql) | 65.71 | 99.09 | 88.86 | 52.07 | 166.55 | ok | +| [3](benchmarks/queries/query_3.sql) | 102.02 | 63.90 | 64.27 | 59.52 | 67.55 | ok | +| [4](benchmarks/queries/query_4.sql) | 53.78 | 72.75 | 57.36 | 55.74 | 120.53 | ok | +| [5](benchmarks/queries/query_5.sql) | 64.92 | 47.45 | 47.18 | 44.87 | 50.55 | ok | +| [6](benchmarks/queries/query_6.sql) | 689.46 | 689.40 | 681.74 | 669.80 | 724.31 | ok | +| [7](benchmarks/queries/query_7.sql) | 32.64 | 28.93 | 28.98 | 28.51 | 29.27 | ok | +| [8](benchmarks/queries/query_8.sql) | 137.21 | 125.35 | 126.21 | 118.63 | 130.37 | ok | +| [9](benchmarks/queries/query_9.sql) | 181.60 | 185.20 | 182.91 | 177.70 | 197.28 | ok | +| [10](benchmarks/queries/query_10.sql) | 48.49 | 36.32 | 36.14 | 35.74 | 37.27 | ok | +| [11](benchmarks/queries/query_11.sql) | 52.50 | 40.40 | 40.20 | 38.94 | 42.24 | ok | +| [12](benchmarks/queries/query_12.sql) | 91.76 | 88.94 | 88.69 | 86.63 | 91.77 | ok | +| [13](benchmarks/queries/query_13.sql) | 100.89 | 97.84 | 96.94 | 94.84 | 102.63 | ok | +| [14](benchmarks/queries/query_14.sql) | 107.95 | 95.51 | 95.43 | 94.33 | 96.84 | ok | +| [15](benchmarks/queries/query_15.sql) | 175.84 | 169.96 | 169.44 | 164.33 | 176.64 | ok | +| [16](benchmarks/queries/query_16.sql) | 259.40 | 267.56 | 267.52 | 266.79 | 268.43 | ok | + diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt new file mode 100644 index 0000000..5559b8d --- /dev/null +++ b/benchmarks/CMakeLists.txt @@ -0,0 +1,37 @@ +add_executable(benchmark_csv_to_columnar + benchmark_csv_to_columnar.cpp +) + +target_link_libraries(benchmark_csv_to_columnar + PRIVATE + columnar_engine_lib +) + +add_executable(benchmark_queries + benchmark_queries.cpp + support/query_result_checks.cpp + support/query_result_checks.h + support/query_runners.h +) + +target_link_libraries(benchmark_queries + PRIVATE + columnar_engine_lib +) + +target_compile_definitions(benchmark_csv_to_columnar + PRIVATE + COLUMNAR_BENCHMARK_DEFAULT_DATA="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample.csv" + COLUMNAR_BENCHMARK_DEFAULT_SCHEMA="${PROJECT_SOURCE_DIR}/benchmarks/schema_sample.csv" + COLUMNAR_BENCHMARK_DEFAULT_OUTPUT="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample.columnar" + COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_DATA="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample_new.csv" + COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA="${PROJECT_SOURCE_DIR}/benchmarks/schema_sample_new.csv" +) + +target_compile_definitions(benchmark_queries + PRIVATE + COLUMNAR_BENCHMARK_QUERIES_DIR="${PROJECT_SOURCE_DIR}/benchmarks/queries" +) + +set_target_properties(benchmark_csv_to_columnar PROPERTIES FOLDER "benchmarks") +set_target_properties(benchmark_queries PROPERTIES FOLDER "benchmarks") diff --git a/benchmarks/benchmark_csv_to_columnar.cpp b/benchmarks/benchmark_csv_to_columnar.cpp new file mode 100644 index 0000000..db124f5 --- /dev/null +++ b/benchmarks/benchmark_csv_to_columnar.cpp @@ -0,0 +1,121 @@ +#include +#include +#include +#include +#include + +#include "io/csv.h" +#include "convert/csv_columnar.h" +#include "support/error.h" + +#ifndef COLUMNAR_BENCHMARK_DEFAULT_DATA +#define COLUMNAR_BENCHMARK_DEFAULT_DATA "benchmarks/hits_sample.csv" +#endif + +#ifndef COLUMNAR_BENCHMARK_DEFAULT_SCHEMA +#define COLUMNAR_BENCHMARK_DEFAULT_SCHEMA "benchmarks/schema_sample.csv" +#endif + +#ifndef COLUMNAR_BENCHMARK_DEFAULT_OUTPUT +#define COLUMNAR_BENCHMARK_DEFAULT_OUTPUT "benchmarks/hits_sample.columnar" +#endif + +#ifndef COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_DATA +#define COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_DATA "benchmarks/hits_sample_new.csv" +#endif + +#ifndef COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA +#define COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA "benchmarks/schema_sample_new.csv" +#endif + +static constexpr size_t RowsPerGroup = 50000; + +struct CsvCompareResult { + bool equal = true; + size_t row = 0; +}; + +static CsvCompareResult CompareCsvFiles(const std::filesystem::path& lhs_path, const std::filesystem::path& rhs_path) { + CsvReader lhs(lhs_path); + CsvReader rhs(rhs_path); + + std::vector lhs_row; + std::vector rhs_row; + + size_t row = 0; + + while (true) { + const bool lhs_has_row = lhs.ReadRow(lhs_row); + const bool rhs_has_row = rhs.ReadRow(rhs_row); + + if (lhs_has_row != rhs_has_row) { + return CsvCompareResult{false, row}; + } + if (!lhs_has_row) { + return CsvCompareResult{true, row}; + } + if (lhs_row != rhs_row) { + return CsvCompareResult{false, row}; + } + + ++row; + } +} + +int main(const int argc, char** argv) { + try { + size_t rows_per_group = RowsPerGroup; + if (argc == 2) { + rows_per_group = std::stoull(argv[1]); + } + + const std::filesystem::path schema_path = COLUMNAR_BENCHMARK_DEFAULT_SCHEMA; + const std::filesystem::path data_path = COLUMNAR_BENCHMARK_DEFAULT_DATA; + const std::filesystem::path output_path = COLUMNAR_BENCHMARK_DEFAULT_OUTPUT; + const std::filesystem::path roundtrip_schema_path = COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA; + const std::filesystem::path roundtrip_data_path = COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_DATA; + + const auto csv_to_columnar_started_at = std::chrono::steady_clock::now(); + ConvertCsvToColumnar(schema_path, data_path, output_path, rows_per_group); + const auto csv_to_columnar_finished_at = std::chrono::steady_clock::now(); + + const auto columnar_to_csv_started_at = std::chrono::steady_clock::now(); + ConvertColumnarToCsv(output_path, roundtrip_schema_path, roundtrip_data_path); + const auto columnar_to_csv_finished_at = std::chrono::steady_clock::now(); + + const auto compare_started_at = std::chrono::steady_clock::now(); + const CsvCompareResult schema_compare = CompareCsvFiles(schema_path, roundtrip_schema_path); + const CsvCompareResult data_compare = CompareCsvFiles(data_path, roundtrip_data_path); + const auto compare_finished_at = std::chrono::steady_clock::now(); + + const double csv_to_columnar_seconds = + std::chrono::duration(csv_to_columnar_finished_at - csv_to_columnar_started_at).count(); + const double columnar_to_csv_seconds = + std::chrono::duration(columnar_to_csv_finished_at - columnar_to_csv_started_at).count(); + const double compare_seconds = std::chrono::duration(compare_finished_at - compare_started_at).count(); + + std::cout << "schema: " << schema_path << std::endl + << "data: " << data_path << std::endl + << "columnar_output: " << output_path << std::endl + << "roundtrip_schema: " << roundtrip_schema_path << std::endl + << "roundtrip_data: " << roundtrip_data_path << std::endl + << "rows_per_group: " << rows_per_group << std::endl + << "csv_to_columnar_elapsed_seconds: " << csv_to_columnar_seconds << std::endl + << "columnar_to_csv_elapsed_seconds: " << columnar_to_csv_seconds << std::endl + << "csv_compare_elapsed_seconds: " << compare_seconds << std::endl + << "schema_matches_source: " << (schema_compare.equal ? "true" : "false") << std::endl + << "data_matches_source: " << (data_compare.equal ? "true" : "false") << std::endl; + + if (!schema_compare.equal) { + std::cout << "schema_first_mismatch_row: " << schema_compare.row << std::endl; + } + if (!data_compare.equal) { + std::cout << "data_first_mismatch_row: " << data_compare.row << std::endl; + } + + return schema_compare.equal && data_compare.equal ? 0 : 2; + } catch (const std::exception& ex) { + std::cerr << ex.what() << std::endl; + return 1; + } +} diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp new file mode 100644 index 0000000..2d5469f --- /dev/null +++ b/benchmarks/benchmark_queries.cpp @@ -0,0 +1,116 @@ +#include +#include +#include +#include +#include +#include + +#include "executor/executor.h" +#include "io/fileio.h" +#include "support/query_result_checks.h" +#include "support/query_runners.h" + +#ifndef COLUMNAR_BENCHMARK_QUERIES_DIR +#define COLUMNAR_BENCHMARK_QUERIES_DIR "benchmarks/queries" +#endif + +static constexpr int FirstQueryId = 0; +static constexpr int LastQueryId = 16; + +void RunQuery(const Executor& executor, const std::filesystem::path& path, const int query_id, bool record = false) { + const auto& runners = GetQueryRunners(); + const auto it = runners.find(query_id); + const std::string query_sql = std::filesystem::exists(path) ? ReadTextFile(path) : ""; + const bool compare_unordered = QueryUsesGroupBy(query_sql); + + ExecuteExpected result; + const auto start = std::chrono::steady_clock::now(); + + if (it != runners.end()) { + result = it->second(executor); + } else if (std::filesystem::exists(path)) { + result = executor.Execute(query_sql); + } else { + std::cout << "query_" << query_id << ": missing, status=skipped" << std::endl; + return; + } + + const auto end = std::chrono::steady_clock::now(); + const auto duration_ms = std::chrono::duration_cast(end - start).count(); + + if (!result.has_value()) { + std::cout << "query_" << query_id << ": error=\"" << result.error().GetMessage() << "\", status=failed" + << std::endl; + return; + } + + const std::string actual_result = BatchToString(*result); + const std::filesystem::path ref_path = + std::filesystem::path(COLUMNAR_BENCHMARK_QUERIES_DIR) / ("query_" + std::to_string(query_id) + ".csv"); + + std::string status; + + if (record) { + std::ofstream ref_file(ref_path); + ref_file << actual_result; + status = "recorded"; + } else if (std::filesystem::exists(ref_path)) { + const std::string expected_result = ReadTextFile(ref_path); + if (compare_unordered ? EqualAsRowMultisets(actual_result, expected_result) + : actual_result == expected_result) { + status = "passed"; + } else { + status = "failed"; + std::cout << "query_" << query_id << ": expected:\n" << expected_result; + if (!expected_result.empty() && expected_result.back() != '\n') { + std::cout << std::endl; + } + std::cout << "query_" << query_id << ": actual:\n" << actual_result; + if (!actual_result.empty() && actual_result.back() != '\n') { + std::cout << std::endl; + } + } + } else { + status = "no_reference"; + } + + std::cout << "query_" << query_id << ": rows=" << result->RowsCount() << ", time=" << duration_ms + << "ms, status=" << status << std::endl; +} + +int main(const int argc, char** argv) { + try { + bool record = false; + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "--record") { + record = true; + } + } + + const std::filesystem::path queries_dir = COLUMNAR_BENCHMARK_QUERIES_DIR; + const std::filesystem::path data_path = queries_dir.parent_path() / "hits_sample.columnar"; + + Executor executor; + executor.RegisterTable("hits", data_path); + + size_t found_queries = 0; + + std::cout << "queries_dir=" << queries_dir << std::endl; + + for (int query_id = FirstQueryId; query_id <= LastQueryId; ++query_id) { + const std::filesystem::path query_path = queries_dir / ("query_" + std::to_string(query_id) + ".sql"); + + if (GetQueryRunners().contains(query_id) || std::filesystem::exists(query_path)) { + ++found_queries; + RunQuery(executor, query_path, query_id, record); + } + } + + std::cout << "total_executed=" << found_queries << std::endl; + } catch (const std::exception& ex) { + std::cerr << ex.what() << std::endl; + return 1; + } + + return 0; +} diff --git a/benchmarks/queries/README.md b/benchmarks/queries/README.md new file mode 100644 index 0000000..bc42131 --- /dev/null +++ b/benchmarks/queries/README.md @@ -0,0 +1 @@ +`columnar_engine run-query --input --query-file --output ` diff --git a/benchmarks/queries/queries.sql b/benchmarks/queries/queries.sql new file mode 100644 index 0000000..49dd47c --- /dev/null +++ b/benchmarks/queries/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; +SELECT AVG(UserID) FROM hits; +SELECT COUNT(DISTINCT UserID) FROM hits; +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; +SELECT MIN(EventDate), MAX(EventDate) FROM hits; +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID FROM hits WHERE UserID = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; +SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; \ No newline at end of file diff --git a/benchmarks/queries/query_0.csv b/benchmarks/queries/query_0.csv new file mode 100644 index 0000000..4b9dfa7 --- /dev/null +++ b/benchmarks/queries/query_0.csv @@ -0,0 +1 @@ +999977 diff --git a/benchmarks/queries/query_0.sql b/benchmarks/queries/query_0.sql new file mode 100644 index 0000000..c70aa7a --- /dev/null +++ b/benchmarks/queries/query_0.sql @@ -0,0 +1 @@ +SELECT COUNT(*) FROM hits; diff --git a/benchmarks/queries/query_1.csv b/benchmarks/queries/query_1.csv new file mode 100644 index 0000000..f8cb17a --- /dev/null +++ b/benchmarks/queries/query_1.csv @@ -0,0 +1 @@ +14174 diff --git a/benchmarks/queries/query_1.sql b/benchmarks/queries/query_1.sql new file mode 100644 index 0000000..ad8031a --- /dev/null +++ b/benchmarks/queries/query_1.sql @@ -0,0 +1 @@ +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; diff --git a/benchmarks/queries/query_10.csv b/benchmarks/queries/query_10.csv new file mode 100644 index 0000000..4734f82 --- /dev/null +++ b/benchmarks/queries/query_10.csv @@ -0,0 +1,10 @@ +iPad,2303 +iPhone,107 +A500,34 +N8-00,12 +GT-P7300B,12 +iPho,11 +3110000,6 +IQ245Plus,5 +eagle75,4 +GT-S5830,3 diff --git a/benchmarks/queries/query_10.sql b/benchmarks/queries/query_10.sql new file mode 100644 index 0000000..d0ea7e3 --- /dev/null +++ b/benchmarks/queries/query_10.sql @@ -0,0 +1 @@ +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_11.csv b/benchmarks/queries/query_11.csv new file mode 100644 index 0000000..3f24c76 --- /dev/null +++ b/benchmarks/queries/query_11.csv @@ -0,0 +1,10 @@ +1,iPad,1967 +5,iPad,97 +7,iPad,79 +6,iPad,55 +6,iPhone,37 +26,iPhone,36 +118,A500,34 +32,iPad,29 +60,iPad,22 +13,iPad,12 diff --git a/benchmarks/queries/query_11.sql b/benchmarks/queries/query_11.sql new file mode 100644 index 0000000..2a316d1 --- /dev/null +++ b/benchmarks/queries/query_11.sql @@ -0,0 +1 @@ +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_12.csv b/benchmarks/queries/query_12.csv new file mode 100644 index 0000000..2dbd091 --- /dev/null +++ b/benchmarks/queries/query_12.csv @@ -0,0 +1,10 @@ +ведомосквы вместу,4943 +ведомосквы вы из,2471 +ведомосквиталия страции,2026 +ведомосковский,1686 +покеты рецепт засня,961 +рецепты сбербан,788 +авторий,705 +ведомосква,446 +ведомосквы новые водительная болгарин,411 +инстанец жизнь,391 diff --git a/benchmarks/queries/query_12.sql b/benchmarks/queries/query_12.sql new file mode 100644 index 0000000..7a02546 --- /dev/null +++ b/benchmarks/queries/query_12.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_13.csv b/benchmarks/queries/query_13.csv new file mode 100644 index 0000000..0dc3e1d --- /dev/null +++ b/benchmarks/queries/query_13.csv @@ -0,0 +1,10 @@ +ведомосквы вместу,1381 +ведомосквы вы из,678 +ведомосквиталия страции,658 +рецепты сбербан,594 +ведомосковский,407 +инстанец жизнь,292 +покеты рецепт засня,281 +авторий,196 +рецепт блиноленские,135 +ведомосква,129 diff --git a/benchmarks/queries/query_13.sql b/benchmarks/queries/query_13.sql new file mode 100644 index 0000000..4ce0fee --- /dev/null +++ b/benchmarks/queries/query_13.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_14.csv b/benchmarks/queries/query_14.csv new file mode 100644 index 0000000..c40691f --- /dev/null +++ b/benchmarks/queries/query_14.csv @@ -0,0 +1,10 @@ +2,ведомосквы вместу,3478 +2,ведомосквы вы из,1857 +2,ведомосковский,1682 +2,ведомосквиталия страции,1434 +4,покеты рецепт засня,959 +2,рецепты сбербан,737 +3,ведомосквы вместу,660 +2,авторий,576 +3,ведомосквиталия страции,494 +4,ведомосквы вместу,442 diff --git a/benchmarks/queries/query_14.sql b/benchmarks/queries/query_14.sql new file mode 100644 index 0000000..8e85255 --- /dev/null +++ b/benchmarks/queries/query_14.sql @@ -0,0 +1 @@ +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_15.csv b/benchmarks/queries/query_15.csv new file mode 100644 index 0000000..32d99e7 --- /dev/null +++ b/benchmarks/queries/query_15.csv @@ -0,0 +1,10 @@ +1508127196834704092,1303 +3205616454965152970,949 +502693359570399458,893 +873022393995828557,876 +2256536417172705921,695 +340634745528635910,610 +72709437341035504,560 +5705194083846317709,532 +1257144732630861346,524 +4885305169967046117,516 diff --git a/benchmarks/queries/query_15.sql b/benchmarks/queries/query_15.sql new file mode 100644 index 0000000..f959e98 --- /dev/null +++ b/benchmarks/queries/query_15.sql @@ -0,0 +1 @@ +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/benchmarks/queries/query_16.csv b/benchmarks/queries/query_16.csv new file mode 100644 index 0000000..43d129b --- /dev/null +++ b/benchmarks/queries/query_16.csv @@ -0,0 +1,10 @@ +1508127196834704092,,1303 +3205616454965152970,,949 +502693359570399458,,893 +873022393995828557,,876 +2256536417172705921,,695 +340634745528635910,,610 +72709437341035504,,560 +5705194083846317709,,532 +614605011960296602,,506 +775643969820522877,,483 diff --git a/benchmarks/queries/query_16.sql b/benchmarks/queries/query_16.sql new file mode 100644 index 0000000..50b1f38 --- /dev/null +++ b/benchmarks/queries/query_16.sql @@ -0,0 +1 @@ +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/benchmarks/queries/query_17.sql b/benchmarks/queries/query_17.sql new file mode 100644 index 0000000..454cdb5 --- /dev/null +++ b/benchmarks/queries/query_17.sql @@ -0,0 +1 @@ +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; diff --git a/benchmarks/queries/query_18.sql b/benchmarks/queries/query_18.sql new file mode 100644 index 0000000..6520cc5 --- /dev/null +++ b/benchmarks/queries/query_18.sql @@ -0,0 +1 @@ +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/benchmarks/queries/query_19.sql b/benchmarks/queries/query_19.sql new file mode 100644 index 0000000..3f84066 --- /dev/null +++ b/benchmarks/queries/query_19.sql @@ -0,0 +1 @@ +SELECT UserID FROM hits WHERE UserID = 435090932899640449; diff --git a/benchmarks/queries/query_2.csv b/benchmarks/queries/query_2.csv new file mode 100644 index 0000000..49e722f --- /dev/null +++ b/benchmarks/queries/query_2.csv @@ -0,0 +1 @@ +80778,999977,1604 diff --git a/benchmarks/queries/query_2.sql b/benchmarks/queries/query_2.sql new file mode 100644 index 0000000..7db4dc2 --- /dev/null +++ b/benchmarks/queries/query_2.sql @@ -0,0 +1 @@ +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; diff --git a/benchmarks/queries/query_20.sql b/benchmarks/queries/query_20.sql new file mode 100644 index 0000000..4426afa --- /dev/null +++ b/benchmarks/queries/query_20.sql @@ -0,0 +1 @@ +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; diff --git a/benchmarks/queries/query_21.sql b/benchmarks/queries/query_21.sql new file mode 100644 index 0000000..bd28609 --- /dev/null +++ b/benchmarks/queries/query_21.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_22.sql b/benchmarks/queries/query_22.sql new file mode 100644 index 0000000..4ee87ac --- /dev/null +++ b/benchmarks/queries/query_22.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_23.sql b/benchmarks/queries/query_23.sql new file mode 100644 index 0000000..935169e --- /dev/null +++ b/benchmarks/queries/query_23.sql @@ -0,0 +1 @@ +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; diff --git a/benchmarks/queries/query_24.sql b/benchmarks/queries/query_24.sql new file mode 100644 index 0000000..1bcfd4c --- /dev/null +++ b/benchmarks/queries/query_24.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; diff --git a/benchmarks/queries/query_25.sql b/benchmarks/queries/query_25.sql new file mode 100644 index 0000000..58ea761 --- /dev/null +++ b/benchmarks/queries/query_25.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; diff --git a/benchmarks/queries/query_26.sql b/benchmarks/queries/query_26.sql new file mode 100644 index 0000000..88ed7ba --- /dev/null +++ b/benchmarks/queries/query_26.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; diff --git a/benchmarks/queries/query_27.sql b/benchmarks/queries/query_27.sql new file mode 100644 index 0000000..8f89977 --- /dev/null +++ b/benchmarks/queries/query_27.sql @@ -0,0 +1 @@ +SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/benchmarks/queries/query_28.sql b/benchmarks/queries/query_28.sql new file mode 100644 index 0000000..f55fac1 --- /dev/null +++ b/benchmarks/queries/query_28.sql @@ -0,0 +1 @@ +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/benchmarks/queries/query_29.sql b/benchmarks/queries/query_29.sql new file mode 100644 index 0000000..630450b --- /dev/null +++ b/benchmarks/queries/query_29.sql @@ -0,0 +1 @@ +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; diff --git a/benchmarks/queries/query_3.csv b/benchmarks/queries/query_3.csv new file mode 100644 index 0000000..c7b6e3a --- /dev/null +++ b/benchmarks/queries/query_3.csv @@ -0,0 +1 @@ +1948165676197850120 diff --git a/benchmarks/queries/query_3.sql b/benchmarks/queries/query_3.sql new file mode 100644 index 0000000..e953498 --- /dev/null +++ b/benchmarks/queries/query_3.sql @@ -0,0 +1 @@ +SELECT AVG(UserID) FROM hits; diff --git a/benchmarks/queries/query_30.sql b/benchmarks/queries/query_30.sql new file mode 100644 index 0000000..f8a80d9 --- /dev/null +++ b/benchmarks/queries/query_30.sql @@ -0,0 +1 @@ +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_31.sql b/benchmarks/queries/query_31.sql new file mode 100644 index 0000000..ba8ac2f --- /dev/null +++ b/benchmarks/queries/query_31.sql @@ -0,0 +1 @@ +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_32.sql b/benchmarks/queries/query_32.sql new file mode 100644 index 0000000..893773a --- /dev/null +++ b/benchmarks/queries/query_32.sql @@ -0,0 +1 @@ +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_33.sql b/benchmarks/queries/query_33.sql new file mode 100644 index 0000000..f00a3e8 --- /dev/null +++ b/benchmarks/queries/query_33.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_34.sql b/benchmarks/queries/query_34.sql new file mode 100644 index 0000000..2137530 --- /dev/null +++ b/benchmarks/queries/query_34.sql @@ -0,0 +1 @@ +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_35.sql b/benchmarks/queries/query_35.sql new file mode 100644 index 0000000..581e5e3 --- /dev/null +++ b/benchmarks/queries/query_35.sql @@ -0,0 +1 @@ +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_36.sql b/benchmarks/queries/query_36.sql new file mode 100644 index 0000000..d6e06ce --- /dev/null +++ b/benchmarks/queries/query_36.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; diff --git a/benchmarks/queries/query_37.sql b/benchmarks/queries/query_37.sql new file mode 100644 index 0000000..f63f7a8 --- /dev/null +++ b/benchmarks/queries/query_37.sql @@ -0,0 +1 @@ +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; diff --git a/benchmarks/queries/query_38.sql b/benchmarks/queries/query_38.sql new file mode 100644 index 0000000..498db57 --- /dev/null +++ b/benchmarks/queries/query_38.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; diff --git a/benchmarks/queries/query_39.sql b/benchmarks/queries/query_39.sql new file mode 100644 index 0000000..9d3d319 --- /dev/null +++ b/benchmarks/queries/query_39.sql @@ -0,0 +1 @@ +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; diff --git a/benchmarks/queries/query_4.csv b/benchmarks/queries/query_4.csv new file mode 100644 index 0000000..d3d401f --- /dev/null +++ b/benchmarks/queries/query_4.csv @@ -0,0 +1 @@ +79842 diff --git a/benchmarks/queries/query_4.sql b/benchmarks/queries/query_4.sql new file mode 100644 index 0000000..2f9baf8 --- /dev/null +++ b/benchmarks/queries/query_4.sql @@ -0,0 +1 @@ +SELECT COUNT(DISTINCT UserID) FROM hits; diff --git a/benchmarks/queries/query_40.sql b/benchmarks/queries/query_40.sql new file mode 100644 index 0000000..f54f6db --- /dev/null +++ b/benchmarks/queries/query_40.sql @@ -0,0 +1 @@ +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; diff --git a/benchmarks/queries/query_41.sql b/benchmarks/queries/query_41.sql new file mode 100644 index 0000000..dc1b7e6 --- /dev/null +++ b/benchmarks/queries/query_41.sql @@ -0,0 +1 @@ +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; diff --git a/benchmarks/queries/query_42.sql b/benchmarks/queries/query_42.sql new file mode 100644 index 0000000..4ac2815 --- /dev/null +++ b/benchmarks/queries/query_42.sql @@ -0,0 +1 @@ +SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; diff --git a/benchmarks/queries/query_5.csv b/benchmarks/queries/query_5.csv new file mode 100644 index 0000000..d2c24b0 --- /dev/null +++ b/benchmarks/queries/query_5.csv @@ -0,0 +1 @@ +18316 diff --git a/benchmarks/queries/query_5.sql b/benchmarks/queries/query_5.sql new file mode 100644 index 0000000..e9615f8 --- /dev/null +++ b/benchmarks/queries/query_5.sql @@ -0,0 +1 @@ +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; diff --git a/benchmarks/queries/query_6.csv b/benchmarks/queries/query_6.csv new file mode 100644 index 0000000..b72aa99 --- /dev/null +++ b/benchmarks/queries/query_6.csv @@ -0,0 +1 @@ +2013-07-15,2013-07-15 diff --git a/benchmarks/queries/query_6.sql b/benchmarks/queries/query_6.sql new file mode 100644 index 0000000..03fbb82 --- /dev/null +++ b/benchmarks/queries/query_6.sql @@ -0,0 +1 @@ +SELECT MIN(EventDate), MAX(EventDate) FROM hits; diff --git a/benchmarks/queries/query_7.csv b/benchmarks/queries/query_7.csv new file mode 100644 index 0000000..0e1c4d4 --- /dev/null +++ b/benchmarks/queries/query_7.csv @@ -0,0 +1,5 @@ +2,9543 +13,4592 +52,34 +50,4 +28,1 diff --git a/benchmarks/queries/query_7.sql b/benchmarks/queries/query_7.sql new file mode 100644 index 0000000..25a10b1 --- /dev/null +++ b/benchmarks/queries/query_7.sql @@ -0,0 +1 @@ +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; diff --git a/benchmarks/queries/query_8.csv b/benchmarks/queries/query_8.csv new file mode 100644 index 0000000..5f52dd6 --- /dev/null +++ b/benchmarks/queries/query_8.csv @@ -0,0 +1,10 @@ +229,27961 +2,10413 +208,3073 +1,1720 +34,1428 +158,1110 +184,987 +107,966 +42,956 +47,943 diff --git a/benchmarks/queries/query_8.sql b/benchmarks/queries/query_8.sql new file mode 100644 index 0000000..a6bf47e --- /dev/null +++ b/benchmarks/queries/query_8.sql @@ -0,0 +1 @@ +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_9.csv b/benchmarks/queries/query_9.csv new file mode 100644 index 0000000..355f967 --- /dev/null +++ b/benchmarks/queries/query_9.csv @@ -0,0 +1,10 @@ +229,38044,426412,1612,27961 +2,12801,148193,1593,10413 +208,2673,30614,1490,3073 +1,1802,28577,1623,1720 +34,508,14329,1592,1428 +47,1041,13661,1637,943 +158,78,13294,1576,1110 +7,1166,11679,1627,647 +42,642,11547,1625,956 +184,30,10157,1614,987 diff --git a/benchmarks/queries/query_9.sql b/benchmarks/queries/query_9.sql new file mode 100644 index 0000000..f4a9ee3 --- /dev/null +++ b/benchmarks/queries/query_9.sql @@ -0,0 +1 @@ +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv new file mode 100644 index 0000000..2595e32 --- /dev/null +++ b/benchmarks/results/readme_benchmarks.csv @@ -0,0 +1,18 @@ +query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,status,error +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,31.90,10.09,9.93,9.63,10.88,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,47.47,29.97,29.35,28.25,32.94,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",65.71,99.09,88.86,52.07,166.55,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,102.02,63.90,64.27,59.52,67.55,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,53.78,72.75,57.36,55.74,120.53,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,64.92,47.45,47.18,44.87,50.55,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",689.46,689.40,681.74,669.80,724.31,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",32.64,28.93,28.98,28.51,29.27,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",137.21,125.35,126.21,118.63,130.37,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",181.60,185.20,182.91,177.70,197.28,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",48.49,36.32,36.14,35.74,37.27,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",52.50,40.40,40.20,38.94,42.24,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",91.76,88.94,88.69,86.63,91.77,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",100.89,97.84,96.94,94.84,102.63,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",107.95,95.51,95.43,94.33,96.84,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",175.84,169.96,169.44,164.33,176.64,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",259.40,267.56,267.52,266.79,268.43,ok, diff --git a/benchmarks/support/query_result_checks.cpp b/benchmarks/support/query_result_checks.cpp new file mode 100644 index 0000000..da89f53 --- /dev/null +++ b/benchmarks/support/query_result_checks.cpp @@ -0,0 +1,60 @@ +#include "query_result_checks.h" + +#include +#include +#include + +#include "io/csv_batch_io.h" +#include "support/ascii.h" + +std::string BatchToString(const Batch& batch) { + std::vector> rows; + AppendBatchRows(batch, rows); + + std::string result; + + for (const auto& row : rows) { + for (size_t i = 0; i < row.size(); ++i) { + result += row[i]; + if (i + 1 < row.size()) { + result += ","; + } + } + result += "\n"; + } + + return result; +} + +bool QueryUsesGroupBy(const std::string_view sql) { return ToUpperAscii(sql).find("GROUP BY") != std::string::npos; } + +static std::vector SplitRows(const std::string_view text) { + std::vector rows; + size_t start = 0; + + while (start < text.size()) { + const size_t end = text.find('\n', start); + if (end == std::string_view::npos) { + rows.emplace_back(text.substr(start)); + break; + } + rows.emplace_back(text.substr(start, end - start)); + start = end + 1; + } + + if (!rows.empty() && rows.back().empty()) { + rows.pop_back(); + } + + return rows; +} + +bool EqualAsRowMultisets(const std::string_view lhs, const std::string_view rhs) { + std::vector lhs_rows = SplitRows(lhs); + std::vector rhs_rows = SplitRows(rhs); + + std::ranges::sort(lhs_rows); + std::ranges::sort(rhs_rows); + + return lhs_rows == rhs_rows; +} diff --git a/benchmarks/support/query_result_checks.h b/benchmarks/support/query_result_checks.h new file mode 100644 index 0000000..ceea1b7 --- /dev/null +++ b/benchmarks/support/query_result_checks.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +#include "model/batch.h" + +std::string BatchToString(const Batch& batch); +bool QueryUsesGroupBy(std::string_view sql); +bool EqualAsRowMultisets(std::string_view lhs, std::string_view rhs); diff --git a/benchmarks/support/query_runners.h b/benchmarks/support/query_runners.h new file mode 100644 index 0000000..b1ca5cf --- /dev/null +++ b/benchmarks/support/query_runners.h @@ -0,0 +1,98 @@ +#pragma once + +#include +#include + +#include "executor/executor.h" + +using QueryRunner = std::function; + +inline ExecuteExpected RunQuery0(const Executor& executor) { return executor.Execute("SELECT COUNT(*) FROM hits;"); } + +inline ExecuteExpected RunQuery1(const Executor& executor) { + return executor.Execute("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); +} + +inline ExecuteExpected RunQuery2(const Executor& executor) { + return executor.Execute("SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;"); +} + +inline ExecuteExpected RunQuery3(const Executor& executor) { return executor.Execute("SELECT AVG(UserID) FROM hits;"); } + +inline ExecuteExpected RunQuery4(const Executor& executor) { + return executor.Execute("SELECT COUNT(DISTINCT UserID) FROM hits;"); +} + +inline ExecuteExpected RunQuery5(const Executor& executor) { + return executor.Execute("SELECT COUNT(DISTINCT SearchPhrase) FROM hits;"); +} + +inline ExecuteExpected RunQuery6(const Executor& executor) { + return executor.Execute("SELECT MIN(EventDate), MAX(EventDate) FROM hits;"); +} + +inline ExecuteExpected RunQuery7(const Executor& executor) { + return executor.Execute( + "SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;"); +} + +inline ExecuteExpected RunQuery8(const Executor& executor) { + return executor.Execute( + "SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC " + "LIMIT 10;"); +} + +inline ExecuteExpected RunQuery9(const Executor& executor) { + return executor.Execute( + "SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), " + "COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery10(const Executor& executor) { + return executor.Execute( + "SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE " + "MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery11(const Executor& executor) { + return executor.Execute( + "SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE " + "MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery12(const Executor& executor) { + return executor.Execute( + "SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY " + "SearchPhrase ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery13(const Executor& executor) { + return executor.Execute( + "SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' " + "GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery14(const Executor& executor) { + return executor.Execute( + "SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' " + "GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery15(const Executor& executor) { + return executor.Execute("SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery16(const Executor& executor) { + return executor.Execute( + "SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC " + "LIMIT 10;"); +} + +inline const std::map& GetQueryRunners() { + static const std::map Runners = { + {0, RunQuery0}, {1, RunQuery1}, {2, RunQuery2}, {3, RunQuery3}, {4, RunQuery4}, {5, RunQuery5}, + {6, RunQuery6}, {7, RunQuery7}, {8, RunQuery8}, {9, RunQuery9}, {10, RunQuery10}, {11, RunQuery11}, + {12, RunQuery12}, {13, RunQuery13}, {14, RunQuery14}, {15, RunQuery15}, {16, RunQuery16}, + }; + return Runners; +} diff --git a/cmake/dependencies_package.cmake b/cmake/dependencies_package.cmake new file mode 100644 index 0000000..3a0c86d --- /dev/null +++ b/cmake/dependencies_package.cmake @@ -0,0 +1,14 @@ +if (ENABLE_TESTS) + find_package(GTest QUIET) + if (NOT GTest_FOUND) + message(WARNING "GTest not found; tests will be disabled for this configure run.") + set(ENABLE_TESTS OFF CACHE BOOL "Build tests" FORCE) + endif () +endif () + +include(FetchContent) +FetchContent_Declare( + argparse + GIT_REPOSITORY https://github.com/p-ranav/argparse.git +) +FetchContent_MakeAvailable(argparse) diff --git a/cmake/sanitizers.cmake b/cmake/sanitizers.cmake new file mode 100644 index 0000000..76fd40d --- /dev/null +++ b/cmake/sanitizers.cmake @@ -0,0 +1,59 @@ +function(columnar_enable_sanitizers sanitizers_raw) + if (NOT sanitizers_raw) + return() + endif () + + string(REPLACE "," ";" sanitizers "${sanitizers_raw}") + list(REMOVE_DUPLICATES sanitizers) + + set(supported_sanitizers address undefined leak thread memory) + foreach (sanitizer IN LISTS sanitizers) + if (NOT sanitizer IN_LIST supported_sanitizers) + message(FATAL_ERROR "Unsupported sanitizer: ${sanitizer}") + endif () + endforeach () + + if ("thread" IN_LIST sanitizers AND "address" IN_LIST sanitizers) + message(FATAL_ERROR "ThreadSanitizer cannot be combined with AddressSanitizer") + endif () + + if ("thread" IN_LIST sanitizers AND "leak" IN_LIST sanitizers) + message(FATAL_ERROR "ThreadSanitizer cannot be combined with LeakSanitizer") + endif () + + if ("memory" IN_LIST sanitizers AND "address" IN_LIST sanitizers) + message(FATAL_ERROR "MemorySanitizer cannot be combined with AddressSanitizer") + endif () + + if ("memory" IN_LIST sanitizers AND "thread" IN_LIST sanitizers) + message(FATAL_ERROR "MemorySanitizer cannot be combined with ThreadSanitizer") + endif () + + if ("memory" IN_LIST sanitizers AND "leak" IN_LIST sanitizers) + message(FATAL_ERROR "MemorySanitizer cannot be combined with LeakSanitizer") + endif () + + if ("memory" IN_LIST sanitizers) + if (NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") + message(FATAL_ERROR "MemorySanitizer requires Clang") + endif () + if (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + message(FATAL_ERROR "MemorySanitizer is not supported by AppleClang") + endif () + if (NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "MemorySanitizer is only supported in this project on Linux") + endif () + endif () + + string(JOIN "," sanitizer_flags ${sanitizers}) + + add_compile_options( + -fno-omit-frame-pointer + -fno-optimize-sibling-calls + -fsanitize=${sanitizer_flags} + ) + + add_link_options(-fsanitize=${sanitizer_flags}) + + message(STATUS "Enabled sanitizers: ${sanitizer_flags}") +endfunction() diff --git a/include/convert/csv_columnar.h b/include/convert/csv_columnar.h new file mode 100644 index 0000000..cd2b5f5 --- /dev/null +++ b/include/convert/csv_columnar.h @@ -0,0 +1,9 @@ +#pragma once + +#include +#include + +void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, + const std::filesystem::path& output_path, size_t max_rows_per_group = 4); +void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, + const std::filesystem::path& data_path); diff --git a/include/executor/aggregate_function.h b/include/executor/aggregate_function.h new file mode 100644 index 0000000..69435ef --- /dev/null +++ b/include/executor/aggregate_function.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include + +#include "model/schema.h" + +class AggState; +struct PlannedAgg; + +using AggFactory = std::unique_ptr (*)(const PlannedAgg&); +using AggTypeSupport = bool (*)(ColumnType); + +struct AggFuncDefinition { + std::string_view canonical_name; + bool distinct = false; + bool star = false; + AggTypeSupport supports_type = nullptr; + AggFactory factory = nullptr; +}; + +class AggRegistry { + public: + using RegistryMap = std::unordered_map; + + static AggRegistry& Instance(); + + void Register(const AggFuncDefinition& def); + const AggFuncDefinition* Find(std::string_view name) const; + + private: + RegistryMap registry_; +}; + +struct AggRegistrar { + explicit AggRegistrar(const AggFuncDefinition& def); +}; + +const AggFuncDefinition& ResolveAggFunc(std::string_view name); +bool AggSupportsInputType(const AggFuncDefinition& definition, ColumnType type); +std::string FormatAgg(const AggFuncDefinition& definition, std::string_view column_name, bool distinct, bool star); diff --git a/include/executor/executor.h b/include/executor/executor.h new file mode 100644 index 0000000..53da8b7 --- /dev/null +++ b/include/executor/executor.h @@ -0,0 +1,167 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "executor/aggregate_function.h" +#include "model/batch.h" +#include "support/error.h" +#include "tl/expected.hpp" +#include "sql_parser/tokenizer.h" + +using ExecuteExpected = tl::expected; + +class Operator { + public: + Operator() = default; + Operator(const Operator&) = delete; + Operator(Operator&&) noexcept = default; + Operator& operator=(const Operator&) = delete; + Operator& operator=(Operator&&) noexcept = default; + virtual ~Operator() = default; + + public: + virtual std::optional Next() = 0; +}; + +class Executor { + public: + Executor() = default; + + public: + void RegisterTable(const std::string& name, std::filesystem::path path); + ExecuteExpected Execute(std::string_view query) const; + + private: + std::unordered_map tables_; +}; + +enum class ComparisonKind { + Equal, + NotEqual, + Less, + LessOrEqual, + Greater, + GreaterOrEqual, +}; + +struct ParsedLiteral { + std::string text; + Tokens token_type = Tokens::StringLiteral; +}; + +struct ColumnRef { + std::string qualifier; + std::string name; +}; + +struct FilterSpec { + ColumnRef column; + ComparisonKind comparison = ComparisonKind::Equal; + ParsedLiteral literal; +}; + +struct AggSpec { + const AggFuncDefinition* function = nullptr; + bool distinct = false; + bool star = false; + ColumnRef column; + std::string output_name; +}; + +enum class SelectItemKind { + GroupKey, + Aggregate, +}; + +struct SelectItemSpec { + SelectItemKind kind = SelectItemKind::GroupKey; + ColumnRef column; + AggSpec aggregate; + std::string output_name; +}; + +struct OrderBySpec { + SelectItemSpec item; + bool descending = false; +}; + +struct ParsedQuery { + std::string table_name; + std::string table_alias; + std::vector select_items; + std::vector group_by; + std::optional filter; + std::vector order_by; + std::optional limit; +}; + +struct PlannedAgg { + const AggFuncDefinition* function = nullptr; + bool distinct = false; + bool star = false; + std::optional column_index; + ColumnType input_type = ColumnType::String; + std::string output_name; +}; + +struct PlannedGroupKey { + size_t column_index = 0; + ColumnType column_type = ColumnType::String; + std::string output_name; +}; + +struct PlannedSelectItem { + SelectItemKind kind = SelectItemKind::GroupKey; + size_t index = 0; + ColumnType output_type = ColumnType::String; + std::string output_name; +}; + +struct PlannedOrderBy { + size_t result_column_index = 0; + bool descending = false; + ColumnType value_type = ColumnType::String; +}; + +struct PlannedFilter { + size_t column_index = 0; + ColumnType column_type = ColumnType::String; + ComparisonKind comparison = ComparisonKind::Equal; + std::string literal_text; +}; + +struct PlannedQuery { + std::filesystem::path table_path; + std::vector projection_indexes; + std::optional filter; + std::vector group_keys; + std::vector aggregates; + std::vector select_items; + std::vector order_by; + std::optional limit; +}; + +class AggState { + public: + AggState() = default; + AggState(const AggState&) = delete; + AggState(AggState&&) noexcept = default; + AggState& operator=(const AggState&) = delete; + AggState& operator=(AggState&&) noexcept = default; + virtual ~AggState() = default; + + public: + virtual void Consume(std::optional value) = 0; + virtual std::string Finalize() const = 0; +}; + +ParsedQuery ParseQuery(std::string_view query); +PlannedQuery PlanQuery(const ParsedQuery& parsed, const std::unordered_map& tables); +std::unique_ptr CreateAggState(const PlannedAgg& aggregate); +std::unique_ptr BuildPlan(const PlannedQuery& planned); diff --git a/include/executor/operator_utils.h b/include/executor/operator_utils.h new file mode 100644 index 0000000..7a26006 --- /dev/null +++ b/include/executor/operator_utils.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +#include "executor/executor.h" + +int CompareValues(ColumnType type, std::string_view lhs, std::string_view rhs); +bool MatchesComparison(ColumnType type, std::string_view lhs, std::string_view rhs, ComparisonKind kind); +ColumnType AggregateOutputType(const PlannedAgg& aggregate); diff --git a/include/executor/planner_utils.h b/include/executor/planner_utils.h new file mode 100644 index 0000000..f5b4b90 --- /dev/null +++ b/include/executor/planner_utils.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +#include "executor/executor.h" + +size_t FindColumnIndex(const Schema& schema, std::string_view column_name); +void ValidateColumnQualifier(const ParsedQuery& parsed, const ColumnRef& column); +std::string NormalizeLiteral(const ParsedLiteral& literal, ColumnType type); +bool SameColumnName(std::string_view lhs, std::string_view rhs); +bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs); +bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs); +bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs); diff --git a/include/io/batch_io.h b/include/io/batch_io.h new file mode 100644 index 0000000..ff8b47b --- /dev/null +++ b/include/io/batch_io.h @@ -0,0 +1,40 @@ +#pragma once + +#include + +#include "model/batch.h" + +struct BatchSizing { + std::optional max_rows; + std::optional max_values; + std::optional max_bytes; + + bool WouldExceed(size_t next_rows, size_t column_count, uint64_t next_bytes) const; +}; + +class BatchReader { + public: + BatchReader() = default; + BatchReader(const BatchReader&) = default; + BatchReader(BatchReader&&) noexcept = default; + BatchReader& operator=(const BatchReader&) = default; + BatchReader& operator=(BatchReader&&) noexcept = default; + virtual ~BatchReader() = default; + + public: + virtual std::optional ReadNext() = 0; +}; + +class BatchWriter { + public: + BatchWriter() = default; + BatchWriter(const BatchWriter&) = default; + BatchWriter(BatchWriter&&) noexcept = default; + BatchWriter& operator=(const BatchWriter&) = default; + BatchWriter& operator=(BatchWriter&&) noexcept = default; + virtual ~BatchWriter() = default; + + public: + virtual void Write(const Batch& batch) = 0; + virtual void Flush() = 0; +}; diff --git a/include/io/columnar_batch_io.h b/include/io/columnar_batch_io.h new file mode 100644 index 0000000..6cafa3b --- /dev/null +++ b/include/io/columnar_batch_io.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include + +#include "io/batch_io.h" +#include "io/fileio.h" +#include "model/metadata.h" + +class ColumnarBatchReader final : public BatchReader { + public: + explicit ColumnarBatchReader(const std::filesystem::path& path); + ColumnarBatchReader(const ColumnarBatchReader&) = delete; + ColumnarBatchReader(ColumnarBatchReader&&) noexcept = default; + ColumnarBatchReader& operator=(const ColumnarBatchReader&) = delete; + ColumnarBatchReader& operator=(ColumnarBatchReader&&) noexcept = default; + ~ColumnarBatchReader() override = default; + + public: + std::optional ReadNext() override; + + const Schema& GetSchema() const { return metadata_.schema; } + const ColumnarMetadata& GetMetadata() const { return metadata_; } + + private: + static ColumnarMetadata ReadFileMetadata(const std::filesystem::path& path); + + private: + std::filesystem::path path_; + std::ifstream in_; + + ColumnarMetadata metadata_; + size_t next_group_ = 0; +}; + +class ColumnarBatchWriter final : public BatchWriter { + public: + ColumnarBatchWriter(const std::filesystem::path& path, Schema schema); + ColumnarBatchWriter(const ColumnarBatchWriter&) = delete; + ColumnarBatchWriter(ColumnarBatchWriter&&) noexcept = default; + ColumnarBatchWriter& operator=(const ColumnarBatchWriter&) = delete; + ColumnarBatchWriter& operator=(ColumnarBatchWriter&&) noexcept = default; + ~ColumnarBatchWriter() override = default; + + public: + void Write(const Batch& batch) override; + void Flush() override; + + void Finalize() &&; + + const ColumnarMetadata& GetMetadata() const { return metadata_; } + + private: + void FinalizeImpl(); + + private: + std::filesystem::path path_; + std::ofstream out_; + + ColumnarMetadata metadata_; + bool finalized_ = false; +}; diff --git a/include/io/csv.h b/include/io/csv.h new file mode 100644 index 0000000..1601a20 --- /dev/null +++ b/include/io/csv.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include +#include + +class CsvReader { + public: + explicit CsvReader(std::istream& in); + explicit CsvReader(const std::filesystem::path& path); + CsvReader(const CsvReader&) = delete; + CsvReader(CsvReader&& other) noexcept; + CsvReader& operator=(const CsvReader&) = delete; + CsvReader& operator=(CsvReader&& other) noexcept; + ~CsvReader() = default; + + public: + bool ReadRow(std::vector& row) const; + + private: + void RebindAfterMove(bool uses_owned_stream, std::istream* source_stream) noexcept; + + private: + std::ifstream owned_in_; + std::istream* in_ = nullptr; +}; + +class CsvWriter { + public: + explicit CsvWriter(std::ostream& out); + explicit CsvWriter(const std::filesystem::path& path); + CsvWriter(const CsvWriter&) = delete; + CsvWriter(CsvWriter&& other) noexcept; + CsvWriter& operator=(const CsvWriter&) = delete; + CsvWriter& operator=(CsvWriter&& other) noexcept; + ~CsvWriter() = default; + + public: + void WriteRow(const std::vector& row) const; + void Flush() const; + + private: + void RebindAfterMove(bool uses_owned_stream, std::ostream* source_stream) noexcept; + + private: + std::ofstream owned_out_; + std::ostream* out_ = nullptr; +}; + +std::vector> ReadRows(const std::filesystem::path& path); +void WriteRows(const std::filesystem::path& path, const std::vector>& rows); diff --git a/include/io/csv_batch_io.h b/include/io/csv_batch_io.h new file mode 100644 index 0000000..881bd7c --- /dev/null +++ b/include/io/csv_batch_io.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include + +#include "io/batch_io.h" +#include "io/csv.h" + +class CsvBatchReader final : public BatchReader { + public: + CsvBatchReader(const std::filesystem::path& path, Schema schema, BatchSizing sizing); + CsvBatchReader(const CsvBatchReader&) = delete; + CsvBatchReader(CsvBatchReader&&) noexcept = default; + CsvBatchReader& operator=(const CsvBatchReader&) = delete; + CsvBatchReader& operator=(CsvBatchReader&&) noexcept = default; + ~CsvBatchReader() override = default; + + public: + std::optional ReadNext() override; + + const Schema& GetSchema() const { return schema_; } + + private: + CsvReader csv_reader_; + Schema schema_; + BatchSizing sizing_; + + std::optional> pending_row_; + bool reached_eof_ = false; +}; + +class CsvBatchWriter final : public BatchWriter { + public: + CsvBatchWriter(const std::filesystem::path& path, Schema schema); + CsvBatchWriter(const CsvBatchWriter&) = delete; + CsvBatchWriter(CsvBatchWriter&&) noexcept = default; + CsvBatchWriter& operator=(const CsvBatchWriter&) = delete; + CsvBatchWriter& operator=(CsvBatchWriter&&) noexcept = default; + ~CsvBatchWriter() override = default; + + public: + void Write(const Batch& batch) override; + void Flush() override; + + private: + CsvWriter csv_writer_; + Schema schema_; +}; + +void AppendBatchRows(const Batch& batch, std::vector>& rows); +void WriteBatchCsv(const std::filesystem::path& path, const Batch& batch); diff --git a/include/io/fileio.h b/include/io/fileio.h new file mode 100644 index 0000000..2e62c62 --- /dev/null +++ b/include/io/fileio.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include +#include +#include + +struct FileMetadata { + uintmax_t size = 0; + bool is_regular = false; + bool is_directory = false; + std::filesystem::file_time_type last_write_time; +}; + +std::optional TryGetFileMetadata(const std::filesystem::path& path); + +bool FileExists(const std::filesystem::path& path); +void EnsureParentDirectory(const std::filesystem::path& path); + +std::ifstream OpenInputFile(const std::filesystem::path& path); +void SeekInputFile(std::istream& in, const std::filesystem::path& path, uint64_t offset); + +enum class FileOpenMode { + Truncate, + Append, +}; + +std::ofstream OpenOutputFile(const std::filesystem::path& path, FileOpenMode mode = FileOpenMode::Truncate); + +std::string ReadTextFile(const std::filesystem::path& path); +std::vector ReadFileBytes(const std::filesystem::path& path); +void WriteFileBytes(const std::filesystem::path& path, std::span bytes); +void WriteFileBytes(const std::filesystem::path& path, const std::vector& bytes); diff --git a/include/io/stream.h b/include/io/stream.h new file mode 100644 index 0000000..04b11ba --- /dev/null +++ b/include/io/stream.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "support/error.h" + +void ReadBytes(std::istream& in, char* dst, size_t size); +void WriteBytes(std::ostream& out, std::string_view bytes); + +template +T ReadStream(std::istream& in) { + T value = 0; + ReadBytes(in, reinterpret_cast(&value), sizeof(value)); + return value; +} + +template +void WriteStream(std::ostream& out, T value) { + WriteBytes(out, {reinterpret_cast(&value), sizeof(value)}); +} diff --git a/include/model/batch.h b/include/model/batch.h new file mode 100644 index 0000000..1495aa5 --- /dev/null +++ b/include/model/batch.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +#include "model/column.h" +#include "model/schema.h" + +class Batch { + public: + Batch() = default; + explicit Batch(Schema schema); + Batch(Schema schema, size_t reserve_rows); + Batch(const Batch& other); + Batch(Batch&& other) noexcept = default; + Batch& operator=(const Batch& other); + Batch& operator=(Batch&& other) noexcept = default; + ~Batch() = default; + + public: + const Schema& GetSchema() const { return schema_; } + + size_t ColumnsCount() const; + size_t RowsCount() const; + + void Reserve(size_t n) const; + + const Column& ColumnAt(size_t i) const; + Column& ColumnAt(size_t i); + + const std::vector>& GetColumns() const { return columns_; } + std::vector>& GetColumns() { return columns_; } + + void Validate() const; + + private: + Schema schema_; + std::vector> columns_; +}; diff --git a/include/model/column.h b/include/model/column.h new file mode 100644 index 0000000..3a15866 --- /dev/null +++ b/include/model/column.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "model/schema.h" + +class Column { + public: + explicit Column(ColumnType type); + Column(const Column&) = default; + Column(Column&&) noexcept = default; + Column& operator=(const Column&) = default; + Column& operator=(Column&&) noexcept = default; + virtual ~Column() = default; + + public: + ColumnType Type() const { return type_; } + + virtual size_t Size() const = 0; + virtual void Reserve(size_t n) = 0; + virtual void Clear() = 0; + + virtual void AppendFromString(const std::string& value) = 0; + virtual uint64_t EstimateSizeFromString(std::string_view value) const = 0; + virtual std::string ValueAsString(size_t row) const = 0; + virtual std::unique_ptr Clone() const = 0; + + virtual void WriteTo(std::ostream& out) const = 0; + virtual void ReadFrom(std::istream& in, uint32_t row_count, uint64_t size) = 0; + + protected: + static void CheckRowIndex(const char* module, size_t row, size_t size); + + private: + ColumnType type_; +}; + +std::unique_ptr CreateColumn(ColumnType type); diff --git a/include/model/column_boolean.h b/include/model/column_boolean.h new file mode 100644 index 0000000..33614e2 --- /dev/null +++ b/include/model/column_boolean.h @@ -0,0 +1,13 @@ +#pragma once + +#include "model/fixed_column.h" + +class BooleanColumn final : public FixedColumn { + public: + using FixedColumn::FixedColumn; + + static const char* ModuleName() { return "column_boolean"; } + + void AppendFromString(const std::string& value) override; + std::string ValueAsString(size_t row) const override; +}; diff --git a/include/model/column_character.h b/include/model/column_character.h new file mode 100644 index 0000000..b17baf7 --- /dev/null +++ b/include/model/column_character.h @@ -0,0 +1,13 @@ +#pragma once + +#include "model/fixed_column.h" + +class CharacterColumn final : public FixedColumn { + public: + using FixedColumn::FixedColumn; + + static const char* ModuleName() { return "column_character"; } + + void AppendFromString(const std::string& value) override; + std::string ValueAsString(size_t row) const override; +}; diff --git a/include/model/column_date.h b/include/model/column_date.h new file mode 100644 index 0000000..c7a9e73 --- /dev/null +++ b/include/model/column_date.h @@ -0,0 +1,13 @@ +#pragma once + +#include "model/fixed_column.h" + +class DateColumn final : public FixedColumn { + public: + using FixedColumn::FixedColumn; + + static const char* ModuleName() { return "column_date"; } + + void AppendFromString(const std::string& value) override; + std::string ValueAsString(size_t row) const override; +}; diff --git a/include/model/column_int128.h b/include/model/column_int128.h new file mode 100644 index 0000000..282fe15 --- /dev/null +++ b/include/model/column_int128.h @@ -0,0 +1,14 @@ +#pragma once + +#include "model/fixed_column.h" +#include "support/int128.h" + +class Int128Column final : public FixedColumn { + public: + using FixedColumn::FixedColumn; + + static const char* ModuleName() { return "column_int128"; } + + void AppendFromString(const std::string& value) override; + std::string ValueAsString(size_t row) const override; +}; diff --git a/include/model/column_int16.h b/include/model/column_int16.h new file mode 100644 index 0000000..64fe72f --- /dev/null +++ b/include/model/column_int16.h @@ -0,0 +1,13 @@ +#pragma once + +#include "model/fixed_column.h" + +class Int16Column final : public FixedColumn { + public: + using FixedColumn::FixedColumn; + + static const char* ModuleName() { return "column_int16"; } + + void AppendFromString(const std::string& value) override; + std::string ValueAsString(size_t row) const override; +}; diff --git a/include/model/column_int32.h b/include/model/column_int32.h new file mode 100644 index 0000000..7ffd398 --- /dev/null +++ b/include/model/column_int32.h @@ -0,0 +1,13 @@ +#pragma once + +#include "model/fixed_column.h" + +class Int32Column final : public FixedColumn { + public: + using FixedColumn::FixedColumn; + + static const char* ModuleName() { return "column_int32"; } + + void AppendFromString(const std::string& value) override; + std::string ValueAsString(size_t row) const override; +}; diff --git a/include/model/column_int64.h b/include/model/column_int64.h new file mode 100644 index 0000000..f28adb4 --- /dev/null +++ b/include/model/column_int64.h @@ -0,0 +1,13 @@ +#pragma once + +#include "model/fixed_column.h" + +class Int64Column final : public FixedColumn { + public: + using FixedColumn::FixedColumn; + + static const char* ModuleName() { return "column_int64"; } + + void AppendFromString(const std::string& value) override; + std::string ValueAsString(size_t row) const override; +}; diff --git a/include/model/column_string.h b/include/model/column_string.h new file mode 100644 index 0000000..0d6542a --- /dev/null +++ b/include/model/column_string.h @@ -0,0 +1,13 @@ +#pragma once + +#include "model/variant_column.h" + +class StringColumn final : public VariantColumn { + public: + using VariantColumn::VariantColumn; + + static const char* ModuleName() { return "column_string"; } + + void AppendFromString(const std::string& value) override; + std::string ValueAsString(size_t row) const override; +}; diff --git a/include/model/column_timestamp.h b/include/model/column_timestamp.h new file mode 100644 index 0000000..d6be6c4 --- /dev/null +++ b/include/model/column_timestamp.h @@ -0,0 +1,13 @@ +#pragma once + +#include "model/fixed_column.h" + +class TimestampColumn final : public FixedColumn { + public: + using FixedColumn::FixedColumn; + + static const char* ModuleName() { return "column_timestamp"; } + + void AppendFromString(const std::string& value) override; + std::string ValueAsString(size_t row) const override; +}; diff --git a/include/model/fixed_column.h b/include/model/fixed_column.h new file mode 100644 index 0000000..6a82881 --- /dev/null +++ b/include/model/fixed_column.h @@ -0,0 +1,56 @@ +#pragma once + +#include + +#include "io/stream.h" +#include "model/column.h" +#include "support/error.h" + +template +class FixedColumn : public Column { + public: + FixedColumn() : Column(TypeValue) {} + FixedColumn(const FixedColumn&) = default; + FixedColumn(FixedColumn&&) noexcept = default; + FixedColumn& operator=(const FixedColumn&) = default; + FixedColumn& operator=(FixedColumn&&) noexcept = default; + ~FixedColumn() override = default; + + public: + size_t Size() const override { return values_.size(); } + void Reserve(const size_t n) override { values_.reserve(n); } + void Clear() override { values_.clear(); } + + uint64_t EstimateSizeFromString(const std::string_view value) const override { + (void)value; + return sizeof(T); + } + + std::unique_ptr Clone() const override { + return std::make_unique(static_cast(*this)); + } + + void WriteTo(std::ostream& out) const override { + WriteBytes(out, {reinterpret_cast(values_.data()), values_.size() * sizeof(T)}); + } + + void ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) override { + const uint64_t expected = static_cast(row_count) * sizeof(T); + if (size != expected) { + throw Error::Mismatch(Derived::ModuleName(), "column chunk size mismatch"); + } + values_.resize(row_count); + ReadBytes(in, reinterpret_cast(values_.data()), values_.size() * sizeof(T)); + } + + protected: + void AppendValue(const T value) { values_.push_back(value); } + + T ValueAt(const size_t row) const { + CheckRowIndex(Derived::ModuleName(), row, values_.size()); + return values_[row]; + } + + protected: + std::vector values_; +}; diff --git a/include/model/metadata.h b/include/model/metadata.h new file mode 100644 index 0000000..9a9b339 --- /dev/null +++ b/include/model/metadata.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include +#include + +#include "model/schema.h" + +struct ColumnChunkMetadata { + uint64_t offset = 0; + uint64_t size = 0; +}; + +struct RowGroupMetadata { + uint32_t row_count = 0; + std::vector columns; +}; + +struct ColumnarMetadata { + Schema schema; + std::vector row_groups; +}; + +ColumnarMetadata ReadMetadata(std::istream& in); +void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata); diff --git a/include/model/schema.h b/include/model/schema.h new file mode 100644 index 0000000..8cce2c9 --- /dev/null +++ b/include/model/schema.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include +#include + +enum class ColumnType : uint8_t { + Int64 = 1, + String = 2, + Boolean = 3, + Int16 = 4, + Int32 = 5, + Int128 = 6, + Date = 7, + Timestamp = 8, + Character = 9, +}; + +struct ColumnSchema { + std::string name; + ColumnType type = ColumnType::String; + + bool operator==(const ColumnSchema& other) const = default; +}; + +struct Schema { + std::vector columns; + + bool operator==(const Schema& other) const = default; +}; + +Schema ReadSchemaCsv(const std::filesystem::path& path); +Schema InferSchemaCsv(const std::filesystem::path& path); +void WriteSchemaCsv(const std::filesystem::path& path, const Schema& schema); diff --git a/include/model/variant_column.h b/include/model/variant_column.h new file mode 100644 index 0000000..7db4599 --- /dev/null +++ b/include/model/variant_column.h @@ -0,0 +1,92 @@ +#pragma once + +#include +#include +#include + +#include "io/stream.h" +#include "model/column.h" +#include "support/error.h" + +template +class VariantColumn : public Column { + public: + VariantColumn() : Column(TypeValue) {} + VariantColumn(const VariantColumn&) = default; + VariantColumn(VariantColumn&&) noexcept = default; + VariantColumn& operator=(const VariantColumn&) = default; + VariantColumn& operator=(VariantColumn&&) noexcept = default; + ~VariantColumn() override = default; + + public: + size_t Size() const override { return values_.size(); } + void Reserve(const size_t n) override { values_.reserve(n); } + void Clear() override { values_.clear(); } + + uint64_t EstimateSizeFromString(const std::string_view value) const override { return value.size(); } + + std::unique_ptr Clone() const override { + return std::make_unique(static_cast(*this)); + } + + void WriteTo(std::ostream& out) const override { + size_t total_size = 0; + + for (const auto& value : values_) { + if (value.size() > std::numeric_limits::max()) { + throw Error::Overflow(Derived::ModuleName(), "value exceeds supported size"); + } + total_size += sizeof(uint32_t) + value.size(); + } + + std::string buffer(total_size, '\0'); + char* dst = buffer.data(); + + for (const auto& value : values_) { + const uint32_t length = value.size(); + + std::memcpy(dst, &length, sizeof(length)); + dst += sizeof(length); + + if (!value.empty()) { + std::memcpy(dst, value.data(), value.size()); + dst += value.size(); + } + } + + WriteBytes(out, buffer); + } + + void ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) override { + values_.clear(); + values_.reserve(row_count); + + uint64_t consumed = 0; + + for (uint32_t row_index = 0; row_index < row_count; ++row_index) { + const uint32_t length = ReadStream(in); + consumed += sizeof(uint32_t); + + std::string value(length, '\0'); + ReadBytes(in, value.data(), value.size()); + + consumed += length; + values_.push_back(std::move(value)); + } + + if (consumed != size) { + throw Error::Mismatch(Derived::ModuleName(), "column chunk size mismatch"); + } + } + + protected: + void AppendValue(std::string value) { values_.push_back(std::move(value)); } + + const std::string& ValueAt(const size_t row) const { + CheckRowIndex(Derived::ModuleName(), row, values_.size()); + return values_[row]; + } + + protected: + std::vector values_; +}; diff --git a/include/sql_parser/tokenizer.h b/include/sql_parser/tokenizer.h new file mode 100644 index 0000000..66adcb8 --- /dev/null +++ b/include/sql_parser/tokenizer.h @@ -0,0 +1,322 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "support/error.h" +#include "tl/expected.hpp" + +enum class Tokens { + NameToken, + NumericLiteral, + StringLiteral, + + Select, + From, + Create, + As, + Sum, + Count, + Avg, + Max, + Min, + And, + Distinct, + Length, + Where, + By, + Group, + Order, + Limit, + + Plus, + Minus, + Asterisk, + OpenBracket, + CloseBracket, + Comma, + Dot, + Semicolon, + + Equal, + NotEqual, + Less, + LessOrEqual, + Greater, + GreaterOrEqual, + + EndOfInput, +}; + +std::string_view TokenTypeToString(Tokens type) noexcept; + +template +using Expected = tl::expected; + +class Token { + public: + Token(std::string text, size_t offset); + virtual ~Token() = default; + + virtual Tokens GetType() const noexcept = 0; + + std::string_view GetText() const noexcept; + size_t GetOffset() const noexcept; + + private: + std::string text_; + size_t offset_ = 0; +}; + +using TokenPtr = std::shared_ptr; + +class Command : public Token { + public: + using Token::Token; + ~Command() override = default; + + void AddArg(TokenPtr arg); + const std::vector& GetArgs() const noexcept; + + protected: + std::vector args_; +}; + +class IOperatorToken : public Command { + public: + using Command::Command; + ~IOperatorToken() override = default; +}; + +class TSelectToken : public Command { + public: + using Command::Command; + Tokens GetType() const noexcept override; +}; + +class TCreateToken : public Command { + public: + using Command::Command; + Tokens GetType() const noexcept override; +}; + +class TFromToken : public Command { + public: + using Command::Command; + Tokens GetType() const noexcept override; +}; + +class TLimitToken : public Command { + public: + using Command::Command; + Tokens GetType() const noexcept override; +}; + +class TOrderToken : public Command { + public: + using Command::Command; + Tokens GetType() const noexcept override; +}; + +class TWhereToken : public Command { + public: + using Command::Command; + Tokens GetType() const noexcept override; +}; + +class TGroupToken : public Command { + public: + using Command::Command; + Tokens GetType() const noexcept override; +}; + +class TSumToken : public IOperatorToken { + public: + using IOperatorToken::IOperatorToken; + Tokens GetType() const noexcept override; +}; + +class TMinToken : public IOperatorToken { + public: + using IOperatorToken::IOperatorToken; + Tokens GetType() const noexcept override; +}; + +class TMaxToken : public IOperatorToken { + public: + using IOperatorToken::IOperatorToken; + Tokens GetType() const noexcept override; +}; + +class TCountToken : public IOperatorToken { + public: + using IOperatorToken::IOperatorToken; + Tokens GetType() const noexcept override; +}; + +class TDistinctToken : public IOperatorToken { + public: + using IOperatorToken::IOperatorToken; + Tokens GetType() const noexcept override; +}; + +class TAvgToken : public IOperatorToken { + public: + using IOperatorToken::IOperatorToken; + Tokens GetType() const noexcept override; +}; + +class TLengthToken : public IOperatorToken { + public: + using IOperatorToken::IOperatorToken; + Tokens GetType() const noexcept override; +}; + +class TPlusToken : public IOperatorToken { + public: + using IOperatorToken::IOperatorToken; + Tokens GetType() const noexcept override; +}; + +class TMinusToken : public IOperatorToken { + public: + using IOperatorToken::IOperatorToken; + Tokens GetType() const noexcept override; +}; + +class TNameToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; + + std::string_view GetName() const noexcept; +}; + +class TNumericLiteralToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TStringLiteralToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TAsToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TAndToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TByToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TOpenBracketToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TCommaToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TCloseBracketToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TAsteriskToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TDotToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TSemicolonToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TEqualToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TNotEqualToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TLessToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TLessOrEqualToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TGreaterToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TGreaterOrEqualToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class TEndOfInputToken : public Token { + public: + using Token::Token; + Tokens GetType() const noexcept override; +}; + +class Tokenizer { + public: + explicit Tokenizer(std::string data); + + Expected GetNext(); + + private: + std::string query_; + size_t pos_ = 0; +}; + +Expected> TokenizeSql(std::string_view input); + +Tokens ResolveIdentifierType(std::string_view text); +TokenPtr MakeToken(Tokens type, std::string text, size_t offset); +Expected MakeUnexpectedCharacter(char ch, size_t offset); +Expected MakeUnterminatedString(size_t offset); diff --git a/include/support/ascii.h b/include/support/ascii.h new file mode 100644 index 0000000..28eabe9 --- /dev/null +++ b/include/support/ascii.h @@ -0,0 +1,7 @@ +#pragma once + +#include +#include + +std::string ToLowerAscii(std::string_view input); +std::string ToUpperAscii(std::string_view input); diff --git a/include/support/error.h b/include/support/error.h new file mode 100644 index 0000000..034fc4f --- /dev/null +++ b/include/support/error.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include +#include +#include +#include + +class Error final : public std::exception { + public: + enum class Code : uint16_t { + InvalidArgument = 1, + InvalidData = 2, + NotFound = 3, + OutOfRange = 4, + Mismatch = 5, + Overflow = 6, + InvalidState = 7, + Unsupported = 8, + IoFailure = 9, + }; + + enum class Type : uint8_t { + Validation, + Data, + State, + Io, + }; + + public: + Error(Code code, std::string module, std::string module_instance, Type type, std::string message); + Error(const Error&) = default; + Error(Error&&) noexcept = default; + Error& operator=(const Error&) = default; + Error& operator=(Error&&) noexcept = default; + ~Error() override = default; + + public: + static Error InvalidArgument(std::string module, std::string message, std::string module_instance = {}); + static Error InvalidData(std::string module, std::string message, std::string module_instance = {}); + static Error NotFound(std::string module, std::string message, std::string module_instance = {}); + static Error OutOfRange(std::string module, std::string message, std::string module_instance = {}); + static Error Mismatch(std::string module, std::string message, std::string module_instance = {}); + static Error Overflow(std::string module, std::string message, std::string module_instance = {}); + static Error InvalidState(std::string module, std::string message, std::string module_instance = {}); + static Error Unsupported(std::string module, std::string message, std::string module_instance = {}); + static Error Io(std::string module, std::string message, std::string module_instance = {}); + static Error PathIo(std::string module, const std::filesystem::path& path, std::string_view action); + + const char* what() const noexcept override; + Code GetCode() const noexcept; + Type GetType() const noexcept; + const std::string& GetModule() const noexcept; + const std::string& GetModuleInstance() const noexcept; + const std::string& GetMessage() const noexcept; + + static std::string_view CodeToString(Code code) noexcept; + static std::string_view TypeToString(Type type) noexcept; + + private: + static std::string BuildWhat(std::string_view module, std::string_view message); + + private: + Code code_; + std::string module_; + std::string module_instance_; + Type type_; + std::string message_; + std::string what_; +}; diff --git a/include/support/int128.h b/include/support/int128.h new file mode 100644 index 0000000..0b15cbf --- /dev/null +++ b/include/support/int128.h @@ -0,0 +1,22 @@ +#pragma once + +#if !defined(__SIZEOF_INT128__) +#error "This project requires compiler support for 128-bit integers" +#endif + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpedantic" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#endif + +using Int128 = __int128_t; +using UInt128 = unsigned __int128; + +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif diff --git a/include/support/parsing.h b/include/support/parsing.h new file mode 100644 index 0000000..6a4d585 --- /dev/null +++ b/include/support/parsing.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include +#include + +#include "model/schema.h" +#include "support/int128.h" + +bool ParseBoolean(const std::string& value); +int16_t ParseInt16(const std::string& value); +int32_t ParseInt32(const std::string& value); +int64_t ParseInt64(const std::string& value); +Int128 ParseInt128(const std::string& value); +int32_t ParseDate(const std::string& value); +int64_t ParseTimestamp(const std::string& value); +char ParseCharacter(const std::string& value); + +std::string BooleanToString(bool value); +std::string Int128ToString(Int128 value); +std::string DateToString(int32_t value); +std::string TimestampToString(int64_t value); + +ColumnType ParseColumnType(std::string_view input); +std::string ColumnTypeToString(ColumnType type); diff --git a/include/testing/executor_test_utils.h b/include/testing/executor_test_utils.h new file mode 100644 index 0000000..bf8c3e3 --- /dev/null +++ b/include/testing/executor_test_utils.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include + +#include "gtest/gtest.h" +#include "io/csv_batch_io.h" + +inline std::vector SingleRowValues(const Batch& batch) { + EXPECT_EQ(batch.RowsCount(), 1u); + + std::vector> rows; + rows.reserve(1); + + AppendBatchRows(batch, rows); + + if (rows.empty()) { + return {}; + } + + return std::move(rows.front()); +} + +inline std::vector> BatchRows(const Batch& batch) { + std::vector> rows; + rows.reserve(batch.RowsCount()); + AppendBatchRows(batch, rows); + return rows; +} + +inline std::vector BatchColumnNames(const Batch& batch) { + std::vector names; + names.reserve(batch.ColumnsCount()); + + for (const auto& column : batch.GetSchema().columns) { + names.push_back(column.name); + } + + return names; +} diff --git a/include/testing/temp_file.h b/include/testing/temp_file.h new file mode 100644 index 0000000..d729551 --- /dev/null +++ b/include/testing/temp_file.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +inline std::filesystem::path UniqueTempPath(const std::string& tag) { + const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + + std::random_device rd; + std::mt19937_64 gen(rd()); + std::uniform_int_distribution dist; + + return std::filesystem::temp_directory_path() / (tag + "_" + std::to_string(now) + "_" + std::to_string(dist(gen))); +} + +class TempFile { + public: + explicit TempFile(const std::string& tag) : path_(UniqueTempPath(tag)) {} + TempFile(const TempFile&) = delete; + TempFile(TempFile&& other) noexcept : path_(std::move(other.path_)) { other.path_.clear(); } + TempFile& operator=(const TempFile&) = delete; + TempFile& operator=(TempFile&& other) noexcept { + if (this != &other) { + Remove(); + path_ = std::move(other.path_); + other.path_.clear(); + } + return *this; + } + ~TempFile() { Remove(); } + + public: + const std::filesystem::path& Path() const { return path_; } + + private: + void Remove() const { + if (path_.empty()) { + return; + } + std::error_code ec; + std::filesystem::remove(path_, ec); + } + + private: + std::filesystem::path path_; +}; diff --git a/include/tl/expected.hpp b/include/tl/expected.hpp new file mode 100644 index 0000000..59e59aa --- /dev/null +++ b/include/tl/expected.hpp @@ -0,0 +1,2475 @@ +/// +// expected - An implementation of std::expected with extensions +// Written in 2017 by Sy Brand (tartanllama@gmail.com, @TartanLlama) +// +// Documentation available at http://tl.tartanllama.xyz/ +// +// To the extent possible under law, the author(s) have dedicated all +// copyright and related and neighboring rights to this software to the +// public domain worldwide. This software is distributed without any warranty. +// +// You should have received a copy of the CC0 Public Domain Dedication +// along with this software. If not, see +// . +/// + +#ifndef TL_EXPECTED_HPP +#define TL_EXPECTED_HPP + +#define TL_EXPECTED_VERSION_MAJOR 1 +#define TL_EXPECTED_VERSION_MINOR 3 +#define TL_EXPECTED_VERSION_PATCH 1 + +#include +#include +#include +#include + +#if defined(__EXCEPTIONS) || defined(_CPPUNWIND) +#define TL_EXPECTED_EXCEPTIONS_ENABLED +#endif + +#if (defined(_MSC_VER) && _MSC_VER == 1900) +#define TL_EXPECTED_MSVC2015 +#define TL_EXPECTED_MSVC2015_CONSTEXPR +#else +#define TL_EXPECTED_MSVC2015_CONSTEXPR constexpr +#endif + +#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ <= 9 && \ + !defined(__clang__)) +#define TL_EXPECTED_GCC49 +#endif + +#if (defined(__GNUC__) && __GNUC__ == 5 && __GNUC_MINOR__ <= 4 && \ + !defined(__clang__)) +#define TL_EXPECTED_GCC54 +#endif + +#if (defined(__GNUC__) && __GNUC__ == 5 && __GNUC_MINOR__ <= 5 && \ + !defined(__clang__)) +#define TL_EXPECTED_GCC55 +#endif + +#ifdef _MSVC_LANG +#define TL_CPLUSPLUS _MSVC_LANG +#else +#define TL_CPLUSPLUS __cplusplus +#endif + +#if !defined(TL_ASSERT) +//can't have assert in constexpr in C++11 and GCC 4.9 has a compiler bug +#if (TL_CPLUSPLUS > 201103L) && !defined(TL_EXPECTED_GCC49) +#include +#define TL_ASSERT(x) assert(x) +#else +#define TL_ASSERT(x) +#endif +#endif + +#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ <= 9 && \ + !defined(__clang__)) +// GCC < 5 doesn't support overloading on const&& for member functions + +#define TL_EXPECTED_NO_CONSTRR +// GCC < 5 doesn't support some standard C++11 type traits +#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \ + std::has_trivial_copy_constructor +#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \ + std::has_trivial_copy_assign + +// This one will be different for GCC 5.7 if it's ever supported +#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) \ + std::is_trivially_destructible + +// GCC 5 < v < 8 has a bug in is_trivially_copy_constructible which breaks +// std::vector for non-copyable types +#elif (defined(__GNUC__) && __GNUC__ < 8 && !defined(__clang__)) +#ifndef TL_GCC_LESS_8_TRIVIALLY_COPY_CONSTRUCTIBLE_MUTEX +#define TL_GCC_LESS_8_TRIVIALLY_COPY_CONSTRUCTIBLE_MUTEX +namespace tl { +namespace detail { +template +struct is_trivially_copy_constructible + : std::is_trivially_copy_constructible {}; +#ifdef _GLIBCXX_VECTOR +template +struct is_trivially_copy_constructible> : std::false_type {}; +#endif +} // namespace detail +} // namespace tl +#endif + +#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \ + tl::detail::is_trivially_copy_constructible +#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \ + std::is_trivially_copy_assignable +#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) \ + std::is_trivially_destructible +#else +#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \ + std::is_trivially_copy_constructible +#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \ + std::is_trivially_copy_assignable +#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) \ + std::is_trivially_destructible +#endif + +#if TL_CPLUSPLUS > 201103L +#define TL_EXPECTED_CXX14 +#endif + +#ifdef TL_EXPECTED_GCC49 +#define TL_EXPECTED_GCC49_CONSTEXPR +#else +#define TL_EXPECTED_GCC49_CONSTEXPR constexpr +#endif + +#if (TL_CPLUSPLUS == 201103L || defined(TL_EXPECTED_MSVC2015) || \ + defined(TL_EXPECTED_GCC49)) +#define TL_EXPECTED_11_CONSTEXPR +#else +#define TL_EXPECTED_11_CONSTEXPR constexpr +#endif + +#if TL_CPLUSPLUS >= 201703L +#define TL_EXPECTED_NODISCARD [[nodiscard]] +#else +#define TL_EXPECTED_NODISCARD +#endif + +namespace tl { +template class TL_EXPECTED_NODISCARD expected; + +#ifndef TL_MONOSTATE_INPLACE_MUTEX +#define TL_MONOSTATE_INPLACE_MUTEX +class monostate {}; + +struct in_place_t { + explicit in_place_t() = default; +}; +static constexpr in_place_t in_place{}; +#endif + +template class unexpected { +public: + static_assert(!std::is_same::value, "E must not be void"); + + unexpected() = delete; + constexpr explicit unexpected(const E &e) : m_val(e) {} + + constexpr explicit unexpected(E &&e) : m_val(std::move(e)) {} + + template ::value>::type * = nullptr> + constexpr explicit unexpected(Args &&...args) + : m_val(std::forward(args)...) {} + template < + class U, class... Args, + typename std::enable_if &, Args &&...>::value>::type * = nullptr> + constexpr explicit unexpected(std::initializer_list l, Args &&...args) + : m_val(l, std::forward(args)...) {} + + constexpr const E &value() const & { return m_val; } + TL_EXPECTED_11_CONSTEXPR E &value() & { return m_val; } + TL_EXPECTED_11_CONSTEXPR E &&value() && { return std::move(m_val); } + constexpr const E &&value() const && { return std::move(m_val); } + +private: + E m_val; +}; + +#ifdef __cpp_deduction_guides +template unexpected(E) -> unexpected; +#endif + +template +constexpr bool operator==(const unexpected &lhs, const unexpected &rhs) { + return lhs.value() == rhs.value(); +} +template +constexpr bool operator!=(const unexpected &lhs, const unexpected &rhs) { + return lhs.value() != rhs.value(); +} +template +constexpr bool operator<(const unexpected &lhs, const unexpected &rhs) { + return lhs.value() < rhs.value(); +} +template +constexpr bool operator<=(const unexpected &lhs, const unexpected &rhs) { + return lhs.value() <= rhs.value(); +} +template +constexpr bool operator>(const unexpected &lhs, const unexpected &rhs) { + return lhs.value() > rhs.value(); +} +template +constexpr bool operator>=(const unexpected &lhs, const unexpected &rhs) { + return lhs.value() >= rhs.value(); +} + +template +unexpected::type> make_unexpected(E &&e) { + return unexpected::type>(std::forward(e)); +} + +struct unexpect_t { + unexpect_t() = default; +}; +static constexpr unexpect_t unexpect{}; + +#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED +#define TL_EXPECTED_THROW_EXCEPTION(e) throw((e)); +#else +#define TL_EXPECTED_THROW_EXCEPTION(e) std::terminate(); +#endif + +namespace detail { +#ifndef TL_TRAITS_MUTEX +#define TL_TRAITS_MUTEX +// C++14-style aliases for brevity +template using remove_const_t = typename std::remove_const::type; +template +using remove_reference_t = typename std::remove_reference::type; +template using decay_t = typename std::decay::type; +template +using enable_if_t = typename std::enable_if::type; +template +using conditional_t = typename std::conditional::type; + +// std::conjunction from C++17 +template struct conjunction : std::true_type {}; +template struct conjunction : B {}; +template +struct conjunction + : std::conditional, B>::type {}; + +#if defined(_LIBCPP_VERSION) && __cplusplus == 201103L +#define TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND +#endif + +// In C++11 mode, there's an issue in libc++'s std::mem_fn +// which results in a hard-error when using it in a noexcept expression +// in some cases. This is a check to workaround the common failing case. +#ifdef TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND +template +struct is_pointer_to_non_const_member_func : std::false_type {}; +template +struct is_pointer_to_non_const_member_func + : std::true_type {}; +template +struct is_pointer_to_non_const_member_func + : std::true_type {}; +template +struct is_pointer_to_non_const_member_func + : std::true_type {}; +template +struct is_pointer_to_non_const_member_func + : std::true_type {}; +template +struct is_pointer_to_non_const_member_func + : std::true_type {}; +template +struct is_pointer_to_non_const_member_func + : std::true_type {}; + +template struct is_const_or_const_ref : std::false_type {}; +template struct is_const_or_const_ref : std::true_type {}; +template struct is_const_or_const_ref : std::true_type {}; +#endif + +// std::invoke from C++17 +// https://stackoverflow.com/questions/38288042/c11-14-invoke-workaround +template < + typename Fn, typename... Args, +#ifdef TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND + typename = enable_if_t::value && + is_const_or_const_ref::value)>, +#endif + typename = enable_if_t>::value>, int = 0> +constexpr auto invoke(Fn &&f, Args &&...args) noexcept( + noexcept(std::mem_fn(f)(std::forward(args)...))) + -> decltype(std::mem_fn(f)(std::forward(args)...)) { + return std::mem_fn(f)(std::forward(args)...); +} + +template >::value>> +constexpr auto invoke(Fn &&f, Args &&...args) noexcept( + noexcept(std::forward(f)(std::forward(args)...))) + -> decltype(std::forward(f)(std::forward(args)...)) { + return std::forward(f)(std::forward(args)...); +} + +// std::invoke_result from C++17 +template struct invoke_result_impl; + +template +struct invoke_result_impl< + F, + decltype(detail::invoke(std::declval(), std::declval()...), void()), + Us...> { + using type = + decltype(detail::invoke(std::declval(), std::declval()...)); +}; + +template +using invoke_result = invoke_result_impl; + +template +using invoke_result_t = typename invoke_result::type; + +#if defined(_MSC_VER) && _MSC_VER <= 1900 +// TODO make a version which works with MSVC 2015 +template struct is_swappable : std::true_type {}; + +template struct is_nothrow_swappable : std::true_type {}; +#else +// https://stackoverflow.com/questions/26744589/what-is-a-proper-way-to-implement-is-swappable-to-test-for-the-swappable-concept +namespace swap_adl_tests { +// if swap ADL finds this then it would call std::swap otherwise (same +// signature) +struct tag {}; + +template tag swap(T &, T &); +template tag swap(T (&a)[N], T (&b)[N]); + +// helper functions to test if an unqualified swap is possible, and if it +// becomes std::swap +template std::false_type can_swap(...) noexcept(false); +template (), std::declval()))> +std::true_type can_swap(int) noexcept(noexcept(swap(std::declval(), + std::declval()))); + +template std::false_type uses_std(...); +template +std::is_same(), std::declval())), tag> +uses_std(int); + +template +struct is_std_swap_noexcept + : std::integral_constant::value && + std::is_nothrow_move_assignable::value> {}; + +template +struct is_std_swap_noexcept : is_std_swap_noexcept {}; + +template +struct is_adl_swap_noexcept + : std::integral_constant(0))> {}; +} // namespace swap_adl_tests + +template +struct is_swappable + : std::integral_constant< + bool, + decltype(detail::swap_adl_tests::can_swap(0))::value && + (!decltype(detail::swap_adl_tests::uses_std(0))::value || + (std::is_move_assignable::value && + std::is_move_constructible::value))> {}; + +template +struct is_swappable + : std::integral_constant< + bool, + decltype(detail::swap_adl_tests::can_swap(0))::value && + (!decltype(detail::swap_adl_tests::uses_std( + 0))::value || + is_swappable::value)> {}; + +template +struct is_nothrow_swappable + : std::integral_constant< + bool, + is_swappable::value && + ((decltype(detail::swap_adl_tests::uses_std(0))::value && + detail::swap_adl_tests::is_std_swap_noexcept::value) || + (!decltype(detail::swap_adl_tests::uses_std(0))::value && + detail::swap_adl_tests::is_adl_swap_noexcept::value))> {}; +#endif +#endif + +// Trait for checking if a type is a tl::expected +template struct is_expected_impl : std::false_type {}; +template +struct is_expected_impl> : std::true_type {}; +template using is_expected = is_expected_impl>; + +template +using expected_enable_forward_value = detail::enable_if_t< + std::is_constructible::value && + !std::is_same, in_place_t>::value && + !std::is_same, detail::decay_t>::value && + !std::is_same, detail::decay_t>::value>; + +template +using expected_enable_from_other = detail::enable_if_t< + std::is_constructible::value && + std::is_constructible::value && + !std::is_constructible &>::value && + !std::is_constructible &&>::value && + !std::is_constructible &>::value && + !std::is_constructible &&>::value && + !std::is_convertible &, T>::value && + !std::is_convertible &&, T>::value && + !std::is_convertible &, T>::value && + !std::is_convertible &&, T>::value>; + +template +using is_void_or = conditional_t::value, std::true_type, U>; + +template +using is_copy_constructible_or_void = + is_void_or>; + +template +using is_move_constructible_or_void = + is_void_or>; + +template +using is_copy_assignable_or_void = is_void_or>; + +template +using is_move_assignable_or_void = is_void_or>; + +} // namespace detail + +namespace detail { +struct no_init_t {}; +static constexpr no_init_t no_init{}; + +// Implements the storage of the values, and ensures that the destructor is +// trivial if it can be. +// +// This specialization is for where neither `T` or `E` is trivially +// destructible, so the destructors must be called on destruction of the +// `expected` +template ::value, + bool = std::is_trivially_destructible::value> +struct expected_storage_base { + constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {} + constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {} + + template ::value> * = + nullptr> + constexpr expected_storage_base(in_place_t, Args &&...args) + : m_val(std::forward(args)...), m_has_val(true) {} + + template &, Args &&...>::value> * = nullptr> + constexpr expected_storage_base(in_place_t, std::initializer_list il, + Args &&...args) + : m_val(il, std::forward(args)...), m_has_val(true) {} + template ::value> * = + nullptr> + constexpr explicit expected_storage_base(unexpect_t, Args &&...args) + : m_unexpect(std::forward(args)...), m_has_val(false) {} + + template &, Args &&...>::value> * = nullptr> + constexpr explicit expected_storage_base(unexpect_t, + std::initializer_list il, + Args &&...args) + : m_unexpect(il, std::forward(args)...), m_has_val(false) {} + + ~expected_storage_base() { + if (m_has_val) { + m_val.~T(); + } else { + m_unexpect.~unexpected(); + } + } + union { + T m_val; + unexpected m_unexpect; + char m_no_init; + }; + bool m_has_val; +}; + +// This specialization is for when both `T` and `E` are trivially-destructible, +// so the destructor of the `expected` can be trivial. +template struct expected_storage_base { + constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {} + constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {} + + template ::value> * = + nullptr> + constexpr expected_storage_base(in_place_t, Args &&...args) + : m_val(std::forward(args)...), m_has_val(true) {} + + template &, Args &&...>::value> * = nullptr> + constexpr expected_storage_base(in_place_t, std::initializer_list il, + Args &&...args) + : m_val(il, std::forward(args)...), m_has_val(true) {} + template ::value> * = + nullptr> + constexpr explicit expected_storage_base(unexpect_t, Args &&...args) + : m_unexpect(std::forward(args)...), m_has_val(false) {} + + template &, Args &&...>::value> * = nullptr> + constexpr explicit expected_storage_base(unexpect_t, + std::initializer_list il, + Args &&...args) + : m_unexpect(il, std::forward(args)...), m_has_val(false) {} + + expected_storage_base(const expected_storage_base &) = default; + expected_storage_base(expected_storage_base &&) = default; + expected_storage_base &operator=(const expected_storage_base &) = default; + expected_storage_base &operator=(expected_storage_base &&) = default; + ~expected_storage_base() = default; + union { + T m_val; + unexpected m_unexpect; + char m_no_init; + }; + bool m_has_val; +}; + +// T is trivial, E is not. +template struct expected_storage_base { + constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {} + TL_EXPECTED_MSVC2015_CONSTEXPR expected_storage_base(no_init_t) + : m_no_init(), m_has_val(false) {} + + template ::value> * = + nullptr> + constexpr expected_storage_base(in_place_t, Args &&...args) + : m_val(std::forward(args)...), m_has_val(true) {} + + template &, Args &&...>::value> * = nullptr> + constexpr expected_storage_base(in_place_t, std::initializer_list il, + Args &&...args) + : m_val(il, std::forward(args)...), m_has_val(true) {} + template ::value> * = + nullptr> + constexpr explicit expected_storage_base(unexpect_t, Args &&...args) + : m_unexpect(std::forward(args)...), m_has_val(false) {} + + template &, Args &&...>::value> * = nullptr> + constexpr explicit expected_storage_base(unexpect_t, + std::initializer_list il, + Args &&...args) + : m_unexpect(il, std::forward(args)...), m_has_val(false) {} + + expected_storage_base(const expected_storage_base &) = default; + expected_storage_base(expected_storage_base &&) = default; + expected_storage_base &operator=(const expected_storage_base &) = default; + expected_storage_base &operator=(expected_storage_base &&) = default; + ~expected_storage_base() { + if (!m_has_val) { + m_unexpect.~unexpected(); + } + } + + union { + T m_val; + unexpected m_unexpect; + char m_no_init; + }; + bool m_has_val; +}; + +// E is trivial, T is not. +template struct expected_storage_base { + constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {} + constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {} + + template ::value> * = + nullptr> + constexpr expected_storage_base(in_place_t, Args &&...args) + : m_val(std::forward(args)...), m_has_val(true) {} + + template &, Args &&...>::value> * = nullptr> + constexpr expected_storage_base(in_place_t, std::initializer_list il, + Args &&...args) + : m_val(il, std::forward(args)...), m_has_val(true) {} + template ::value> * = + nullptr> + constexpr explicit expected_storage_base(unexpect_t, Args &&...args) + : m_unexpect(std::forward(args)...), m_has_val(false) {} + + template &, Args &&...>::value> * = nullptr> + constexpr explicit expected_storage_base(unexpect_t, + std::initializer_list il, + Args &&...args) + : m_unexpect(il, std::forward(args)...), m_has_val(false) {} + + expected_storage_base(const expected_storage_base &) = default; + expected_storage_base(expected_storage_base &&) = default; + expected_storage_base &operator=(const expected_storage_base &) = default; + expected_storage_base &operator=(expected_storage_base &&) = default; + ~expected_storage_base() { + if (m_has_val) { + m_val.~T(); + } + } + union { + T m_val; + unexpected m_unexpect; + char m_no_init; + }; + bool m_has_val; +}; + +// `T` is `void`, `E` is trivially-destructible +template struct expected_storage_base { + #if __GNUC__ <= 5 + //no constexpr for GCC 4/5 bug + #else + TL_EXPECTED_MSVC2015_CONSTEXPR + #endif + expected_storage_base() : m_has_val(true) {} + + constexpr expected_storage_base(no_init_t) : m_val(), m_has_val(false) {} + + constexpr expected_storage_base(in_place_t) : m_has_val(true) {} + + template ::value> * = + nullptr> + constexpr explicit expected_storage_base(unexpect_t, Args &&...args) + : m_unexpect(std::forward(args)...), m_has_val(false) {} + + template &, Args &&...>::value> * = nullptr> + constexpr explicit expected_storage_base(unexpect_t, + std::initializer_list il, + Args &&...args) + : m_unexpect(il, std::forward(args)...), m_has_val(false) {} + + expected_storage_base(const expected_storage_base &) = default; + expected_storage_base(expected_storage_base &&) = default; + expected_storage_base &operator=(const expected_storage_base &) = default; + expected_storage_base &operator=(expected_storage_base &&) = default; + ~expected_storage_base() = default; + struct dummy {}; + union { + unexpected m_unexpect; + dummy m_val; + }; + bool m_has_val; +}; + +// `T` is `void`, `E` is not trivially-destructible +template struct expected_storage_base { + constexpr expected_storage_base() : m_dummy(), m_has_val(true) {} + constexpr expected_storage_base(no_init_t) : m_dummy(), m_has_val(false) {} + + constexpr expected_storage_base(in_place_t) : m_dummy(), m_has_val(true) {} + + template ::value> * = + nullptr> + constexpr explicit expected_storage_base(unexpect_t, Args &&...args) + : m_unexpect(std::forward(args)...), m_has_val(false) {} + + template &, Args &&...>::value> * = nullptr> + constexpr explicit expected_storage_base(unexpect_t, + std::initializer_list il, + Args &&...args) + : m_unexpect(il, std::forward(args)...), m_has_val(false) {} + + expected_storage_base(const expected_storage_base &) = default; + expected_storage_base(expected_storage_base &&) = default; + expected_storage_base &operator=(const expected_storage_base &) = default; + expected_storage_base &operator=(expected_storage_base &&) = default; + ~expected_storage_base() { + if (!m_has_val) { + m_unexpect.~unexpected(); + } + } + + union { + unexpected m_unexpect; + char m_dummy; + }; + bool m_has_val; +}; + +// This base class provides some handy member functions which can be used in +// further derived classes +template +struct expected_operations_base : expected_storage_base { + using expected_storage_base::expected_storage_base; + + template void construct(Args &&...args) noexcept { + new (std::addressof(this->m_val)) T(std::forward(args)...); + this->m_has_val = true; + } + + template void construct_with(Rhs &&rhs) noexcept { + new (std::addressof(this->m_val)) T(std::forward(rhs).get()); + this->m_has_val = true; + } + + template void construct_error(Args &&...args) noexcept { + new (std::addressof(this->m_unexpect)) + unexpected(std::forward(args)...); + this->m_has_val = false; + } + +#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED + + // These assign overloads ensure that the most efficient assignment + // implementation is used while maintaining the strong exception guarantee. + // The problematic case is where rhs has a value, but *this does not. + // + // This overload handles the case where we can just copy-construct `T` + // directly into place without throwing. + template ::value> + * = nullptr> + void assign(const expected_operations_base &rhs) noexcept { + if (!this->m_has_val && rhs.m_has_val) { + geterr().~unexpected(); + construct(rhs.get()); + } else { + assign_common(rhs); + } + } + + // This overload handles the case where we can attempt to create a copy of + // `T`, then no-throw move it into place if the copy was successful. + template ::value && + std::is_nothrow_move_constructible::value> + * = nullptr> + void assign(const expected_operations_base &rhs) noexcept { + if (!this->m_has_val && rhs.m_has_val) { + T tmp = rhs.get(); + geterr().~unexpected(); + construct(std::move(tmp)); + } else { + assign_common(rhs); + } + } + + // This overload is the worst-case, where we have to move-construct the + // unexpected value into temporary storage, then try to copy the T into place. + // If the construction succeeds, then everything is fine, but if it throws, + // then we move the old unexpected value back into place before rethrowing the + // exception. + template ::value && + !std::is_nothrow_move_constructible::value> + * = nullptr> + void assign(const expected_operations_base &rhs) { + if (!this->m_has_val && rhs.m_has_val) { + auto tmp = std::move(geterr()); + geterr().~unexpected(); + +#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED + try { + construct(rhs.get()); + } catch (...) { + geterr() = std::move(tmp); + throw; + } +#else + construct(rhs.get()); +#endif + } else { + assign_common(rhs); + } + } + + // These overloads do the same as above, but for rvalues + template ::value> + * = nullptr> + void assign(expected_operations_base &&rhs) noexcept { + if (!this->m_has_val && rhs.m_has_val) { + geterr().~unexpected(); + construct(std::move(rhs).get()); + } else { + assign_common(std::move(rhs)); + } + } + + template ::value> + * = nullptr> + void assign(expected_operations_base &&rhs) { + if (!this->m_has_val && rhs.m_has_val) { + auto tmp = std::move(geterr()); + geterr().~unexpected(); +#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED + try { + construct(std::move(rhs).get()); + } catch (...) { + geterr() = std::move(tmp); + throw; + } +#else + construct(std::move(rhs).get()); +#endif + } else { + assign_common(std::move(rhs)); + } + } + +#else + + // If exceptions are disabled then we can just copy-construct + void assign(const expected_operations_base &rhs) noexcept { + if (!this->m_has_val && rhs.m_has_val) { + geterr().~unexpected(); + construct(rhs.get()); + } else { + assign_common(rhs); + } + } + + void assign(expected_operations_base &&rhs) noexcept { + if (!this->m_has_val && rhs.m_has_val) { + geterr().~unexpected(); + construct(std::move(rhs).get()); + } else { + assign_common(std::move(rhs)); + } + } + +#endif + + // The common part of move/copy assigning + template void assign_common(Rhs &&rhs) { + if (this->m_has_val) { + if (rhs.m_has_val) { + get() = std::forward(rhs).get(); + } else { + destroy_val(); + construct_error(std::forward(rhs).geterr()); + } + } else { + if (!rhs.m_has_val) { + geterr() = std::forward(rhs).geterr(); + } + } + } + + bool has_value() const { return this->m_has_val; } + + TL_EXPECTED_11_CONSTEXPR T &get() & { return this->m_val; } + constexpr const T &get() const & { return this->m_val; } + TL_EXPECTED_11_CONSTEXPR T &&get() && { return std::move(this->m_val); } +#ifndef TL_EXPECTED_NO_CONSTRR + constexpr const T &&get() const && { return std::move(this->m_val); } +#endif + + TL_EXPECTED_11_CONSTEXPR unexpected &geterr() & { + return this->m_unexpect; + } + constexpr const unexpected &geterr() const & { return this->m_unexpect; } + TL_EXPECTED_11_CONSTEXPR unexpected &&geterr() && { + return std::move(this->m_unexpect); + } +#ifndef TL_EXPECTED_NO_CONSTRR + constexpr const unexpected &&geterr() const && { + return std::move(this->m_unexpect); + } +#endif + + TL_EXPECTED_11_CONSTEXPR void destroy_val() { get().~T(); } +}; + +// This base class provides some handy member functions which can be used in +// further derived classes +template +struct expected_operations_base : expected_storage_base { + using expected_storage_base::expected_storage_base; + + template void construct() noexcept { this->m_has_val = true; } + + // This function doesn't use its argument, but needs it so that code in + // levels above this can work independently of whether T is void + template void construct_with(Rhs &&) noexcept { + this->m_has_val = true; + } + + template void construct_error(Args &&...args) noexcept { + new (std::addressof(this->m_unexpect)) + unexpected(std::forward(args)...); + this->m_has_val = false; + } + + template void assign(Rhs &&rhs) noexcept { + if (!this->m_has_val) { + if (rhs.m_has_val) { + geterr().~unexpected(); + construct(); + } else { + geterr() = std::forward(rhs).geterr(); + } + } else { + if (!rhs.m_has_val) { + construct_error(std::forward(rhs).geterr()); + } + } + } + + bool has_value() const { return this->m_has_val; } + + TL_EXPECTED_11_CONSTEXPR unexpected &geterr() & { + return this->m_unexpect; + } + constexpr const unexpected &geterr() const & { return this->m_unexpect; } + TL_EXPECTED_11_CONSTEXPR unexpected &&geterr() && { + return std::move(this->m_unexpect); + } +#ifndef TL_EXPECTED_NO_CONSTRR + constexpr const unexpected &&geterr() const && { + return std::move(this->m_unexpect); + } +#endif + + TL_EXPECTED_11_CONSTEXPR void destroy_val() { + // no-op + } +}; + +// This class manages conditionally having a trivial copy constructor +// This specialization is for when T and E are trivially copy constructible +template :: + value &&TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(E)::value, + bool = (is_copy_constructible_or_void::value && + std::is_copy_constructible::value)> +struct expected_copy_base : expected_operations_base { + using expected_operations_base::expected_operations_base; +}; + +// This specialization is for when T or E are non-trivially copy constructible +template +struct expected_copy_base : expected_operations_base { + using expected_operations_base::expected_operations_base; + + expected_copy_base() = default; + expected_copy_base(const expected_copy_base &rhs) + : expected_operations_base(no_init) { + if (rhs.has_value()) { + this->construct_with(rhs); + } else { + this->construct_error(rhs.geterr()); + } + } + + expected_copy_base(expected_copy_base &&rhs) = default; + expected_copy_base &operator=(const expected_copy_base &rhs) = default; + expected_copy_base &operator=(expected_copy_base &&rhs) = default; +}; + +// This class manages conditionally having a trivial move constructor +// Unfortunately there's no way to achieve this in GCC < 5 AFAIK, since it +// doesn't implement an analogue to std::is_trivially_move_constructible. We +// have to make do with a non-trivial move constructor even if T is trivially +// move constructible +#ifndef TL_EXPECTED_GCC49 +template >::value + &&std::is_trivially_move_constructible::value> +struct expected_move_base : expected_copy_base { + using expected_copy_base::expected_copy_base; +}; +#else +template struct expected_move_base; +#endif +template +struct expected_move_base : expected_copy_base { + using expected_copy_base::expected_copy_base; + + expected_move_base() = default; + expected_move_base(const expected_move_base &rhs) = default; + + expected_move_base(expected_move_base &&rhs) noexcept( + std::is_nothrow_move_constructible::value) + : expected_copy_base(no_init) { + if (rhs.has_value()) { + this->construct_with(std::move(rhs)); + } else { + this->construct_error(std::move(rhs.geterr())); + } + } + expected_move_base &operator=(const expected_move_base &rhs) = default; + expected_move_base &operator=(expected_move_base &&rhs) = default; +}; + +// This class manages conditionally having a trivial copy assignment operator +template >::value + &&TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(E)::value + &&TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(E)::value + &&TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(E)::value, + bool = (is_copy_constructible_or_void::value && + std::is_copy_constructible::value && + is_copy_assignable_or_void::value && + std::is_copy_assignable::value)> +struct expected_copy_assign_base : expected_move_base { + using expected_move_base::expected_move_base; +}; + +template +struct expected_copy_assign_base : expected_move_base { + using expected_move_base::expected_move_base; + + expected_copy_assign_base() = default; + expected_copy_assign_base(const expected_copy_assign_base &rhs) = default; + + expected_copy_assign_base(expected_copy_assign_base &&rhs) = default; + expected_copy_assign_base &operator=(const expected_copy_assign_base &rhs) { + this->assign(rhs); + return *this; + } + expected_copy_assign_base & + operator=(expected_copy_assign_base &&rhs) = default; +}; + +// This class manages conditionally having a trivial move assignment operator +// Unfortunately there's no way to achieve this in GCC < 5 AFAIK, since it +// doesn't implement an analogue to std::is_trivially_move_assignable. We have +// to make do with a non-trivial move assignment operator even if T is trivially +// move assignable +#ifndef TL_EXPECTED_GCC49 +template , + std::is_trivially_move_constructible, + std::is_trivially_move_assignable>>:: + value &&std::is_trivially_destructible::value + &&std::is_trivially_move_constructible::value + &&std::is_trivially_move_assignable::value> +struct expected_move_assign_base : expected_copy_assign_base { + using expected_copy_assign_base::expected_copy_assign_base; +}; +#else +template struct expected_move_assign_base; +#endif + +template +struct expected_move_assign_base + : expected_copy_assign_base { + using expected_copy_assign_base::expected_copy_assign_base; + + expected_move_assign_base() = default; + expected_move_assign_base(const expected_move_assign_base &rhs) = default; + + expected_move_assign_base(expected_move_assign_base &&rhs) = default; + + expected_move_assign_base & + operator=(const expected_move_assign_base &rhs) = default; + + expected_move_assign_base & + operator=(expected_move_assign_base &&rhs) noexcept( + std::is_nothrow_move_constructible::value + &&std::is_nothrow_move_assignable::value) { + this->assign(std::move(rhs)); + return *this; + } +}; + +// expected_delete_ctor_base will conditionally delete copy and move +// constructors depending on whether T is copy/move constructible +template ::value && + std::is_copy_constructible::value), + bool EnableMove = (is_move_constructible_or_void::value && + std::is_move_constructible::value)> +struct expected_delete_ctor_base { + expected_delete_ctor_base() = default; + expected_delete_ctor_base(const expected_delete_ctor_base &) = default; + expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = default; + expected_delete_ctor_base & + operator=(const expected_delete_ctor_base &) = default; + expected_delete_ctor_base & + operator=(expected_delete_ctor_base &&) noexcept = default; +}; + +template +struct expected_delete_ctor_base { + expected_delete_ctor_base() = default; + expected_delete_ctor_base(const expected_delete_ctor_base &) = default; + expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = delete; + expected_delete_ctor_base & + operator=(const expected_delete_ctor_base &) = default; + expected_delete_ctor_base & + operator=(expected_delete_ctor_base &&) noexcept = default; +}; + +template +struct expected_delete_ctor_base { + expected_delete_ctor_base() = default; + expected_delete_ctor_base(const expected_delete_ctor_base &) = delete; + expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = default; + expected_delete_ctor_base & + operator=(const expected_delete_ctor_base &) = default; + expected_delete_ctor_base & + operator=(expected_delete_ctor_base &&) noexcept = default; +}; + +template +struct expected_delete_ctor_base { + expected_delete_ctor_base() = default; + expected_delete_ctor_base(const expected_delete_ctor_base &) = delete; + expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = delete; + expected_delete_ctor_base & + operator=(const expected_delete_ctor_base &) = default; + expected_delete_ctor_base & + operator=(expected_delete_ctor_base &&) noexcept = default; +}; + +// expected_delete_assign_base will conditionally delete copy and move +// constructors depending on whether T and E are copy/move constructible + +// assignable +template ::value && + std::is_copy_constructible::value && + is_copy_assignable_or_void::value && + std::is_copy_assignable::value), + bool EnableMove = (is_move_constructible_or_void::value && + std::is_move_constructible::value && + is_move_assignable_or_void::value && + std::is_move_assignable::value)> +struct expected_delete_assign_base { + expected_delete_assign_base() = default; + expected_delete_assign_base(const expected_delete_assign_base &) = default; + expected_delete_assign_base(expected_delete_assign_base &&) noexcept = + default; + expected_delete_assign_base & + operator=(const expected_delete_assign_base &) = default; + expected_delete_assign_base & + operator=(expected_delete_assign_base &&) noexcept = default; +}; + +template +struct expected_delete_assign_base { + expected_delete_assign_base() = default; + expected_delete_assign_base(const expected_delete_assign_base &) = default; + expected_delete_assign_base(expected_delete_assign_base &&) noexcept = + default; + expected_delete_assign_base & + operator=(const expected_delete_assign_base &) = default; + expected_delete_assign_base & + operator=(expected_delete_assign_base &&) noexcept = delete; +}; + +template +struct expected_delete_assign_base { + expected_delete_assign_base() = default; + expected_delete_assign_base(const expected_delete_assign_base &) = default; + expected_delete_assign_base(expected_delete_assign_base &&) noexcept = + default; + expected_delete_assign_base & + operator=(const expected_delete_assign_base &) = delete; + expected_delete_assign_base & + operator=(expected_delete_assign_base &&) noexcept = default; +}; + +template +struct expected_delete_assign_base { + expected_delete_assign_base() = default; + expected_delete_assign_base(const expected_delete_assign_base &) = default; + expected_delete_assign_base(expected_delete_assign_base &&) noexcept = + default; + expected_delete_assign_base & + operator=(const expected_delete_assign_base &) = delete; + expected_delete_assign_base & + operator=(expected_delete_assign_base &&) noexcept = delete; +}; + +// This is needed to be able to construct the expected_default_ctor_base which +// follows, while still conditionally deleting the default constructor. +struct default_constructor_tag { + explicit constexpr default_constructor_tag() = default; +}; + +// expected_default_ctor_base will ensure that expected has a deleted default +// constructor if T is not default constructible. +// This specialization is for when T is default constructible +template ::value || std::is_void::value> +struct expected_default_ctor_base { + constexpr expected_default_ctor_base() noexcept = default; + constexpr expected_default_ctor_base( + expected_default_ctor_base const &) noexcept = default; + constexpr expected_default_ctor_base(expected_default_ctor_base &&) noexcept = + default; + expected_default_ctor_base & + operator=(expected_default_ctor_base const &) noexcept = default; + expected_default_ctor_base & + operator=(expected_default_ctor_base &&) noexcept = default; + + constexpr explicit expected_default_ctor_base(default_constructor_tag) {} +}; + +// This specialization is for when T is not default constructible +template struct expected_default_ctor_base { + constexpr expected_default_ctor_base() noexcept = delete; + constexpr expected_default_ctor_base( + expected_default_ctor_base const &) noexcept = default; + constexpr expected_default_ctor_base(expected_default_ctor_base &&) noexcept = + default; + expected_default_ctor_base & + operator=(expected_default_ctor_base const &) noexcept = default; + expected_default_ctor_base & + operator=(expected_default_ctor_base &&) noexcept = default; + + constexpr explicit expected_default_ctor_base(default_constructor_tag) {} +}; +} // namespace detail + +template class bad_expected_access : public std::exception { +public: + explicit bad_expected_access(E e) : m_val(std::move(e)) {} + + virtual const char *what() const noexcept override { + return "Bad expected access"; + } + + const E &error() const & { return m_val; } + E &error() & { return m_val; } + const E &&error() const && { return std::move(m_val); } + E &&error() && { return std::move(m_val); } + +private: + E m_val; +}; + +/// An `expected` object is an object that contains the storage for +/// another object and manages the lifetime of this contained object `T`. +/// Alternatively it could contain the storage for another unexpected object +/// `E`. The contained object may not be initialized after the expected object +/// has been initialized, and may not be destroyed before the expected object +/// has been destroyed. The initialization state of the contained object is +/// tracked by the expected object. +template +class TL_EXPECTED_NODISCARD expected : + private detail::expected_move_assign_base, + private detail::expected_delete_ctor_base, + private detail::expected_delete_assign_base, + private detail::expected_default_ctor_base { + static_assert(!std::is_reference::value, "T must not be a reference"); + static_assert(!std::is_same::type>::value, + "T must not be in_place_t"); + static_assert(!std::is_same::type>::value, + "T must not be unexpect_t"); + static_assert( + !std::is_same>::type>::value, + "T must not be unexpected"); + static_assert(!std::is_reference::value, "E must not be a reference"); + + T *valptr() { return std::addressof(this->m_val); } + const T *valptr() const { return std::addressof(this->m_val); } + unexpected *errptr() { return std::addressof(this->m_unexpect); } + const unexpected *errptr() const { + return std::addressof(this->m_unexpect); + } + + template ::value> * = nullptr> + TL_EXPECTED_11_CONSTEXPR U &val() { + return this->m_val; + } + TL_EXPECTED_11_CONSTEXPR unexpected &err() { return this->m_unexpect; } + + template ::value> * = nullptr> + constexpr const U &val() const { + return this->m_val; + } + constexpr const unexpected &err() const { return this->m_unexpect; } + + using impl_base = detail::expected_move_assign_base; + using ctor_base = detail::expected_default_ctor_base; + +public: + typedef T value_type; + typedef E error_type; + typedef unexpected unexpected_type; + +#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ + !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) + template TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) & { + return and_then_impl(*this, std::forward(f)); + } + template TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) && { + return and_then_impl(std::move(*this), std::forward(f)); + } + template constexpr auto and_then(F &&f) const & { + return and_then_impl(*this, std::forward(f)); + } + +#ifndef TL_EXPECTED_NO_CONSTRR + template constexpr auto and_then(F &&f) const && { + return and_then_impl(std::move(*this), std::forward(f)); + } +#endif + +#else + template + TL_EXPECTED_11_CONSTEXPR auto + and_then(F &&f) & -> decltype(and_then_impl(std::declval(), + std::forward(f))) { + return and_then_impl(*this, std::forward(f)); + } + template + TL_EXPECTED_11_CONSTEXPR auto + and_then(F &&f) && -> decltype(and_then_impl(std::declval(), + std::forward(f))) { + return and_then_impl(std::move(*this), std::forward(f)); + } + template + constexpr auto and_then(F &&f) const & -> decltype(and_then_impl( + std::declval(), std::forward(f))) { + return and_then_impl(*this, std::forward(f)); + } + +#ifndef TL_EXPECTED_NO_CONSTRR + template + constexpr auto and_then(F &&f) const && -> decltype(and_then_impl( + std::declval(), std::forward(f))) { + return and_then_impl(std::move(*this), std::forward(f)); + } +#endif +#endif + +#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ + !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) + template TL_EXPECTED_11_CONSTEXPR auto map(F &&f) & { + return expected_map_impl(*this, std::forward(f)); + } + template TL_EXPECTED_11_CONSTEXPR auto map(F &&f) && { + return expected_map_impl(std::move(*this), std::forward(f)); + } + template constexpr auto map(F &&f) const & { + return expected_map_impl(*this, std::forward(f)); + } + template constexpr auto map(F &&f) const && { + return expected_map_impl(std::move(*this), std::forward(f)); + } +#else + template + TL_EXPECTED_11_CONSTEXPR decltype(expected_map_impl( + std::declval(), std::declval())) + map(F &&f) & { + return expected_map_impl(*this, std::forward(f)); + } + template + TL_EXPECTED_11_CONSTEXPR decltype(expected_map_impl(std::declval(), + std::declval())) + map(F &&f) && { + return expected_map_impl(std::move(*this), std::forward(f)); + } + template + constexpr decltype(expected_map_impl(std::declval(), + std::declval())) + map(F &&f) const & { + return expected_map_impl(*this, std::forward(f)); + } + +#ifndef TL_EXPECTED_NO_CONSTRR + template + constexpr decltype(expected_map_impl(std::declval(), + std::declval())) + map(F &&f) const && { + return expected_map_impl(std::move(*this), std::forward(f)); + } +#endif +#endif + +#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ + !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) + template TL_EXPECTED_11_CONSTEXPR auto transform(F &&f) & { + return expected_map_impl(*this, std::forward(f)); + } + template TL_EXPECTED_11_CONSTEXPR auto transform(F &&f) && { + return expected_map_impl(std::move(*this), std::forward(f)); + } + template constexpr auto transform(F &&f) const & { + return expected_map_impl(*this, std::forward(f)); + } + template constexpr auto transform(F &&f) const && { + return expected_map_impl(std::move(*this), std::forward(f)); + } +#else + template + TL_EXPECTED_11_CONSTEXPR decltype(expected_map_impl( + std::declval(), std::declval())) + transform(F &&f) & { + return expected_map_impl(*this, std::forward(f)); + } + template + TL_EXPECTED_11_CONSTEXPR decltype(expected_map_impl(std::declval(), + std::declval())) + transform(F &&f) && { + return expected_map_impl(std::move(*this), std::forward(f)); + } + template + constexpr decltype(expected_map_impl(std::declval(), + std::declval())) + transform(F &&f) const & { + return expected_map_impl(*this, std::forward(f)); + } + +#ifndef TL_EXPECTED_NO_CONSTRR + template + constexpr decltype(expected_map_impl(std::declval(), + std::declval())) + transform(F &&f) const && { + return expected_map_impl(std::move(*this), std::forward(f)); + } +#endif +#endif + +#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ + !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) + template TL_EXPECTED_11_CONSTEXPR auto map_error(F &&f) & { + return map_error_impl(*this, std::forward(f)); + } + template TL_EXPECTED_11_CONSTEXPR auto map_error(F &&f) && { + return map_error_impl(std::move(*this), std::forward(f)); + } + template constexpr auto map_error(F &&f) const & { + return map_error_impl(*this, std::forward(f)); + } + template constexpr auto map_error(F &&f) const && { + return map_error_impl(std::move(*this), std::forward(f)); + } +#else + template + TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval(), + std::declval())) + map_error(F &&f) & { + return map_error_impl(*this, std::forward(f)); + } + template + TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval(), + std::declval())) + map_error(F &&f) && { + return map_error_impl(std::move(*this), std::forward(f)); + } + template + constexpr decltype(map_error_impl(std::declval(), + std::declval())) + map_error(F &&f) const & { + return map_error_impl(*this, std::forward(f)); + } + +#ifndef TL_EXPECTED_NO_CONSTRR + template + constexpr decltype(map_error_impl(std::declval(), + std::declval())) + map_error(F &&f) const && { + return map_error_impl(std::move(*this), std::forward(f)); + } +#endif +#endif +#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ + !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) + template TL_EXPECTED_11_CONSTEXPR auto transform_error(F &&f) & { + return map_error_impl(*this, std::forward(f)); + } + template TL_EXPECTED_11_CONSTEXPR auto transform_error(F &&f) && { + return map_error_impl(std::move(*this), std::forward(f)); + } + template constexpr auto transform_error(F &&f) const & { + return map_error_impl(*this, std::forward(f)); + } + template constexpr auto transform_error(F &&f) const && { + return map_error_impl(std::move(*this), std::forward(f)); + } +#else + template + TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval(), + std::declval())) + transform_error(F &&f) & { + return map_error_impl(*this, std::forward(f)); + } + template + TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval(), + std::declval())) + transform_error(F &&f) && { + return map_error_impl(std::move(*this), std::forward(f)); + } + template + constexpr decltype(map_error_impl(std::declval(), + std::declval())) + transform_error(F &&f) const & { + return map_error_impl(*this, std::forward(f)); + } + +#ifndef TL_EXPECTED_NO_CONSTRR + template + constexpr decltype(map_error_impl(std::declval(), + std::declval())) + transform_error(F &&f) const && { + return map_error_impl(std::move(*this), std::forward(f)); + } +#endif +#endif + template expected TL_EXPECTED_11_CONSTEXPR or_else(F &&f) & { + return or_else_impl(*this, std::forward(f)); + } + + template expected TL_EXPECTED_11_CONSTEXPR or_else(F &&f) && { + return or_else_impl(std::move(*this), std::forward(f)); + } + + template expected constexpr or_else(F &&f) const & { + return or_else_impl(*this, std::forward(f)); + } + +#ifndef TL_EXPECTED_NO_CONSTRR + template expected constexpr or_else(F &&f) const && { + return or_else_impl(std::move(*this), std::forward(f)); + } +#endif + constexpr expected() = default; + constexpr expected(const expected &rhs) = default; + constexpr expected(expected &&rhs) = default; + expected &operator=(const expected &rhs) = default; + expected &operator=(expected &&rhs) = default; + + template ::value> * = + nullptr> + constexpr expected(in_place_t, Args &&...args) + : impl_base(in_place, std::forward(args)...), + ctor_base(detail::default_constructor_tag{}) {} + + template &, Args &&...>::value> * = nullptr> + constexpr expected(in_place_t, std::initializer_list il, Args &&...args) + : impl_base(in_place, il, std::forward(args)...), + ctor_base(detail::default_constructor_tag{}) {} + + template ::value> * = + nullptr, + detail::enable_if_t::value> * = + nullptr> + explicit constexpr expected(const unexpected &e) + : impl_base(unexpect, e.value()), + ctor_base(detail::default_constructor_tag{}) {} + + template < + class G = E, + detail::enable_if_t::value> * = + nullptr, + detail::enable_if_t::value> * = nullptr> + constexpr expected(unexpected const &e) + : impl_base(unexpect, e.value()), + ctor_base(detail::default_constructor_tag{}) {} + + template < + class G = E, + detail::enable_if_t::value> * = nullptr, + detail::enable_if_t::value> * = nullptr> + explicit constexpr expected(unexpected &&e) noexcept( + std::is_nothrow_constructible::value) + : impl_base(unexpect, std::move(e.value())), + ctor_base(detail::default_constructor_tag{}) {} + + template < + class G = E, + detail::enable_if_t::value> * = nullptr, + detail::enable_if_t::value> * = nullptr> + constexpr expected(unexpected &&e) noexcept( + std::is_nothrow_constructible::value) + : impl_base(unexpect, std::move(e.value())), + ctor_base(detail::default_constructor_tag{}) {} + + template ::value> * = + nullptr> + constexpr explicit expected(unexpect_t, Args &&...args) + : impl_base(unexpect, std::forward(args)...), + ctor_base(detail::default_constructor_tag{}) {} + + template &, Args &&...>::value> * = nullptr> + constexpr explicit expected(unexpect_t, std::initializer_list il, + Args &&...args) + : impl_base(unexpect, il, std::forward(args)...), + ctor_base(detail::default_constructor_tag{}) {} + + template ::value && + std::is_convertible::value)> * = + nullptr, + detail::expected_enable_from_other + * = nullptr> + explicit TL_EXPECTED_11_CONSTEXPR expected(const expected &rhs) + : ctor_base(detail::default_constructor_tag{}) { + if (rhs.has_value()) { + this->construct(*rhs); + } else { + this->construct_error(rhs.error()); + } + } + + template ::value && + std::is_convertible::value)> * = + nullptr, + detail::expected_enable_from_other + * = nullptr> + TL_EXPECTED_11_CONSTEXPR expected(const expected &rhs) + : ctor_base(detail::default_constructor_tag{}) { + if (rhs.has_value()) { + this->construct(*rhs); + } else { + this->construct_error(rhs.error()); + } + } + + template < + class U, class G, + detail::enable_if_t::value && + std::is_convertible::value)> * = nullptr, + detail::expected_enable_from_other * = nullptr> + explicit TL_EXPECTED_11_CONSTEXPR expected(expected &&rhs) + : ctor_base(detail::default_constructor_tag{}) { + if (rhs.has_value()) { + this->construct(std::move(*rhs)); + } else { + this->construct_error(std::move(rhs.error())); + } + } + + template < + class U, class G, + detail::enable_if_t<(std::is_convertible::value && + std::is_convertible::value)> * = nullptr, + detail::expected_enable_from_other * = nullptr> + TL_EXPECTED_11_CONSTEXPR expected(expected &&rhs) + : ctor_base(detail::default_constructor_tag{}) { + if (rhs.has_value()) { + this->construct(std::move(*rhs)); + } else { + this->construct_error(std::move(rhs.error())); + } + } + + template < + class U = T, + detail::enable_if_t::value> * = nullptr, + detail::expected_enable_forward_value * = nullptr> + explicit TL_EXPECTED_MSVC2015_CONSTEXPR expected(U &&v) + : expected(in_place, std::forward(v)) {} + + template < + class U = T, + detail::enable_if_t::value> * = nullptr, + detail::expected_enable_forward_value * = nullptr> + TL_EXPECTED_MSVC2015_CONSTEXPR expected(U &&v) + : expected(in_place, std::forward(v)) {} + + template < + class U = T, class G = T, + detail::enable_if_t::value> * = + nullptr, + detail::enable_if_t::value> * = nullptr, + detail::enable_if_t< + (!std::is_same, detail::decay_t>::value && + !detail::conjunction, + std::is_same>>::value && + std::is_constructible::value && + std::is_assignable::value && + std::is_nothrow_move_constructible::value)> * = nullptr> + expected &operator=(U &&v) { + if (has_value()) { + val() = std::forward(v); + } else { + err().~unexpected(); + ::new (valptr()) T(std::forward(v)); + this->m_has_val = true; + } + + return *this; + } + + template < + class U = T, class G = T, + detail::enable_if_t::value> * = + nullptr, + detail::enable_if_t::value> * = nullptr, + detail::enable_if_t< + (!std::is_same, detail::decay_t>::value && + !detail::conjunction, + std::is_same>>::value && + std::is_constructible::value && + std::is_assignable::value && + std::is_nothrow_move_constructible::value)> * = nullptr> + expected &operator=(U &&v) { + if (has_value()) { + val() = std::forward(v); + } else { + auto tmp = std::move(err()); + err().~unexpected(); + +#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED + try { + ::new (valptr()) T(std::forward(v)); + this->m_has_val = true; + } catch (...) { + err() = std::move(tmp); + throw; + } +#else + ::new (valptr()) T(std::forward(v)); + this->m_has_val = true; +#endif + } + + return *this; + } + + template ::value && + std::is_assignable::value> * = nullptr> + expected &operator=(const unexpected &rhs) { + if (!has_value()) { + err() = rhs; + } else { + this->destroy_val(); + ::new (errptr()) unexpected(rhs); + this->m_has_val = false; + } + + return *this; + } + + template ::value && + std::is_move_assignable::value> * = nullptr> + expected &operator=(unexpected &&rhs) noexcept { + if (!has_value()) { + err() = std::move(rhs); + } else { + this->destroy_val(); + ::new (errptr()) unexpected(std::move(rhs)); + this->m_has_val = false; + } + + return *this; + } + + template ::value> * = nullptr> + void emplace(Args &&...args) { + if (has_value()) { + val().~T(); + } else { + err().~unexpected(); + this->m_has_val = true; + } + ::new (valptr()) T(std::forward(args)...); + } + + template ::value> * = nullptr> + void emplace(Args &&...args) { + if (has_value()) { + val().~T(); + ::new (valptr()) T(std::forward(args)...); + } else { + auto tmp = std::move(err()); + err().~unexpected(); + +#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED + try { + ::new (valptr()) T(std::forward(args)...); + this->m_has_val = true; + } catch (...) { + err() = std::move(tmp); + throw; + } +#else + ::new (valptr()) T(std::forward(args)...); + this->m_has_val = true; +#endif + } + } + + template &, Args &&...>::value> * = nullptr> + void emplace(std::initializer_list il, Args &&...args) { + if (has_value()) { + T t(il, std::forward(args)...); + val() = std::move(t); + } else { + err().~unexpected(); + ::new (valptr()) T(il, std::forward(args)...); + this->m_has_val = true; + } + } + + template &, Args &&...>::value> * = nullptr> + void emplace(std::initializer_list il, Args &&...args) { + if (has_value()) { + T t(il, std::forward(args)...); + val() = std::move(t); + } else { + auto tmp = std::move(err()); + err().~unexpected(); + +#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED + try { + ::new (valptr()) T(il, std::forward(args)...); + this->m_has_val = true; + } catch (...) { + err() = std::move(tmp); + throw; + } +#else + ::new (valptr()) T(il, std::forward(args)...); + this->m_has_val = true; +#endif + } + } + +private: + using t_is_void = std::true_type; + using t_is_not_void = std::false_type; + using t_is_nothrow_move_constructible = std::true_type; + using move_constructing_t_can_throw = std::false_type; + using e_is_nothrow_move_constructible = std::true_type; + using move_constructing_e_can_throw = std::false_type; + + void swap_where_both_have_value(expected & /*rhs*/, t_is_void) noexcept { + // swapping void is a no-op + } + + void swap_where_both_have_value(expected &rhs, t_is_not_void) { + using std::swap; + swap(val(), rhs.val()); + } + + void swap_where_only_one_has_value(expected &rhs, t_is_void) noexcept( + std::is_nothrow_move_constructible::value) { + ::new (errptr()) unexpected_type(std::move(rhs.err())); + rhs.err().~unexpected_type(); + std::swap(this->m_has_val, rhs.m_has_val); + } + + void swap_where_only_one_has_value(expected &rhs, t_is_not_void) { + swap_where_only_one_has_value_and_t_is_not_void( + rhs, typename std::is_nothrow_move_constructible::type{}, + typename std::is_nothrow_move_constructible::type{}); + } + + void swap_where_only_one_has_value_and_t_is_not_void( + expected &rhs, t_is_nothrow_move_constructible, + e_is_nothrow_move_constructible) noexcept { + auto temp = std::move(val()); + val().~T(); + ::new (errptr()) unexpected_type(std::move(rhs.err())); + rhs.err().~unexpected_type(); + ::new (rhs.valptr()) T(std::move(temp)); + std::swap(this->m_has_val, rhs.m_has_val); + } + + void swap_where_only_one_has_value_and_t_is_not_void( + expected &rhs, t_is_nothrow_move_constructible, + move_constructing_e_can_throw) { + auto temp = std::move(val()); + val().~T(); +#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED + try { + ::new (errptr()) unexpected_type(std::move(rhs.err())); + rhs.err().~unexpected_type(); + ::new (rhs.valptr()) T(std::move(temp)); + std::swap(this->m_has_val, rhs.m_has_val); + } catch (...) { + val() = std::move(temp); + throw; + } +#else + ::new (errptr()) unexpected_type(std::move(rhs.err())); + rhs.err().~unexpected_type(); + ::new (rhs.valptr()) T(std::move(temp)); + std::swap(this->m_has_val, rhs.m_has_val); +#endif + } + + void swap_where_only_one_has_value_and_t_is_not_void( + expected &rhs, move_constructing_t_can_throw, + e_is_nothrow_move_constructible) { + auto temp = std::move(rhs.err()); + rhs.err().~unexpected_type(); +#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED + try { + ::new (rhs.valptr()) T(std::move(val())); + val().~T(); + ::new (errptr()) unexpected_type(std::move(temp)); + std::swap(this->m_has_val, rhs.m_has_val); + } catch (...) { + rhs.err() = std::move(temp); + throw; + } +#else + ::new (rhs.valptr()) T(std::move(val())); + val().~T(); + ::new (errptr()) unexpected_type(std::move(temp)); + std::swap(this->m_has_val, rhs.m_has_val); +#endif + } + +public: + template + detail::enable_if_t::value && + detail::is_swappable::value && + (std::is_nothrow_move_constructible::value || + std::is_nothrow_move_constructible::value)> + swap(expected &rhs) noexcept( + std::is_nothrow_move_constructible::value + &&detail::is_nothrow_swappable::value + &&std::is_nothrow_move_constructible::value + &&detail::is_nothrow_swappable::value) { + if (has_value() && rhs.has_value()) { + swap_where_both_have_value(rhs, typename std::is_void::type{}); + } else if (!has_value() && rhs.has_value()) { + rhs.swap(*this); + } else if (has_value()) { + swap_where_only_one_has_value(rhs, typename std::is_void::type{}); + } else { + using std::swap; + swap(err(), rhs.err()); + } + } + + constexpr const T *operator->() const { + TL_ASSERT(has_value()); + return valptr(); + } + TL_EXPECTED_11_CONSTEXPR T *operator->() { + TL_ASSERT(has_value()); + return valptr(); + } + + template ::value> * = nullptr> + constexpr const U &operator*() const & { + TL_ASSERT(has_value()); + return val(); + } + template ::value> * = nullptr> + TL_EXPECTED_11_CONSTEXPR U &operator*() & { + TL_ASSERT(has_value()); + return val(); + } + template ::value> * = nullptr> + constexpr const U &&operator*() const && { + TL_ASSERT(has_value()); + return std::move(val()); + } + template ::value> * = nullptr> + TL_EXPECTED_11_CONSTEXPR U &&operator*() && { + TL_ASSERT(has_value()); + return std::move(val()); + } + + constexpr bool has_value() const noexcept { return this->m_has_val; } + constexpr explicit operator bool() const noexcept { return this->m_has_val; } + + template ::value> * = nullptr> + TL_EXPECTED_11_CONSTEXPR const U &value() const & { + if (!has_value()) + TL_EXPECTED_THROW_EXCEPTION(bad_expected_access(err().value())); + return val(); + } + template ::value> * = nullptr> + TL_EXPECTED_11_CONSTEXPR U &value() & { + if (!has_value()) + TL_EXPECTED_THROW_EXCEPTION(bad_expected_access(err().value())); + return val(); + } + template ::value> * = nullptr> + TL_EXPECTED_11_CONSTEXPR const U &&value() const && { + if (!has_value()) + TL_EXPECTED_THROW_EXCEPTION(bad_expected_access(std::move(err()).value())); + return std::move(val()); + } + template ::value> * = nullptr> + TL_EXPECTED_11_CONSTEXPR U &&value() && { + if (!has_value()) + TL_EXPECTED_THROW_EXCEPTION(bad_expected_access(std::move(err()).value())); + return std::move(val()); + } + + constexpr const E &error() const & { + TL_ASSERT(!has_value()); + return err().value(); + } + TL_EXPECTED_11_CONSTEXPR E &error() & { + TL_ASSERT(!has_value()); + return err().value(); + } + constexpr const E &&error() const && { + TL_ASSERT(!has_value()); + return std::move(err().value()); + } + TL_EXPECTED_11_CONSTEXPR E &&error() && { + TL_ASSERT(!has_value()); + return std::move(err().value()); + } + + template constexpr T value_or(U &&v) const & { + static_assert(std::is_copy_constructible::value && + std::is_convertible::value, + "T must be copy-constructible and convertible to from U&&"); + return bool(*this) ? **this : static_cast(std::forward(v)); + } + template TL_EXPECTED_11_CONSTEXPR T value_or(U &&v) && { + static_assert(std::is_move_constructible::value && + std::is_convertible::value, + "T must be move-constructible and convertible to from U&&"); + return bool(*this) ? std::move(**this) : static_cast(std::forward(v)); + } +}; + +namespace detail { +template using exp_t = typename detail::decay_t::value_type; +template using err_t = typename detail::decay_t::error_type; +template using ret_t = expected>; + +#ifdef TL_EXPECTED_CXX14 +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + *std::declval()))> +constexpr auto and_then_impl(Exp &&exp, F &&f) { + static_assert(detail::is_expected::value, "F must return an expected"); + + return exp.has_value() + ? detail::invoke(std::forward(f), *std::forward(exp)) + : Ret(unexpect, std::forward(exp).error()); +} + +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval()))> +constexpr auto and_then_impl(Exp &&exp, F &&f) { + static_assert(detail::is_expected::value, "F must return an expected"); + + return exp.has_value() ? detail::invoke(std::forward(f)) + : Ret(unexpect, std::forward(exp).error()); +} +#else +template struct TC; +template (), + *std::declval())), + detail::enable_if_t>::value> * = nullptr> +auto and_then_impl(Exp &&exp, F &&f) -> Ret { + static_assert(detail::is_expected::value, "F must return an expected"); + + return exp.has_value() + ? detail::invoke(std::forward(f), *std::forward(exp)) + : Ret(unexpect, std::forward(exp).error()); +} + +template ())), + detail::enable_if_t>::value> * = nullptr> +constexpr auto and_then_impl(Exp &&exp, F &&f) -> Ret { + static_assert(detail::is_expected::value, "F must return an expected"); + + return exp.has_value() ? detail::invoke(std::forward(f)) + : Ret(unexpect, std::forward(exp).error()); +} +#endif + +#ifdef TL_EXPECTED_CXX14 +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + *std::declval())), + detail::enable_if_t::value> * = nullptr> +constexpr auto expected_map_impl(Exp &&exp, F &&f) { + using result = ret_t>; + return exp.has_value() ? result(detail::invoke(std::forward(f), + *std::forward(exp))) + : result(unexpect, std::forward(exp).error()); +} + +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + *std::declval())), + detail::enable_if_t::value> * = nullptr> +auto expected_map_impl(Exp &&exp, F &&f) { + using result = expected>; + if (exp.has_value()) { + detail::invoke(std::forward(f), *std::forward(exp)); + return result(); + } + + return result(unexpect, std::forward(exp).error()); +} + +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval())), + detail::enable_if_t::value> * = nullptr> +constexpr auto expected_map_impl(Exp &&exp, F &&f) { + using result = ret_t>; + return exp.has_value() ? result(detail::invoke(std::forward(f))) + : result(unexpect, std::forward(exp).error()); +} + +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval())), + detail::enable_if_t::value> * = nullptr> +auto expected_map_impl(Exp &&exp, F &&f) { + using result = expected>; + if (exp.has_value()) { + detail::invoke(std::forward(f)); + return result(); + } + + return result(unexpect, std::forward(exp).error()); +} +#else +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + *std::declval())), + detail::enable_if_t::value> * = nullptr> + +constexpr auto expected_map_impl(Exp &&exp, F &&f) + -> ret_t> { + using result = ret_t>; + + return exp.has_value() ? result(detail::invoke(std::forward(f), + *std::forward(exp))) + : result(unexpect, std::forward(exp).error()); +} + +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + *std::declval())), + detail::enable_if_t::value> * = nullptr> + +auto expected_map_impl(Exp &&exp, F &&f) -> expected> { + if (exp.has_value()) { + detail::invoke(std::forward(f), *std::forward(exp)); + return {}; + } + + return unexpected>(std::forward(exp).error()); +} + +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval())), + detail::enable_if_t::value> * = nullptr> + +constexpr auto expected_map_impl(Exp &&exp, F &&f) + -> ret_t> { + using result = ret_t>; + + return exp.has_value() ? result(detail::invoke(std::forward(f))) + : result(unexpect, std::forward(exp).error()); +} + +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval())), + detail::enable_if_t::value> * = nullptr> + +auto expected_map_impl(Exp &&exp, F &&f) -> expected> { + if (exp.has_value()) { + detail::invoke(std::forward(f)); + return {}; + } + + return unexpected>(std::forward(exp).error()); +} +#endif + +#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ + !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +constexpr auto map_error_impl(Exp &&exp, F &&f) { + using result = expected, detail::decay_t>; + return exp.has_value() + ? result(*std::forward(exp)) + : result(unexpect, detail::invoke(std::forward(f), + std::forward(exp).error())); +} +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +auto map_error_impl(Exp &&exp, F &&f) { + using result = expected, monostate>; + if (exp.has_value()) { + return result(*std::forward(exp)); + } + + detail::invoke(std::forward(f), std::forward(exp).error()); + return result(unexpect, monostate{}); +} +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +constexpr auto map_error_impl(Exp &&exp, F &&f) { + using result = expected, detail::decay_t>; + return exp.has_value() + ? result() + : result(unexpect, detail::invoke(std::forward(f), + std::forward(exp).error())); +} +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +auto map_error_impl(Exp &&exp, F &&f) { + using result = expected, monostate>; + if (exp.has_value()) { + return result(); + } + + detail::invoke(std::forward(f), std::forward(exp).error()); + return result(unexpect, monostate{}); +} +#else +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +constexpr auto map_error_impl(Exp &&exp, F &&f) + -> expected, detail::decay_t> { + using result = expected, detail::decay_t>; + + return exp.has_value() + ? result(*std::forward(exp)) + : result(unexpect, detail::invoke(std::forward(f), + std::forward(exp).error())); +} + +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +auto map_error_impl(Exp &&exp, F &&f) -> expected, monostate> { + using result = expected, monostate>; + if (exp.has_value()) { + return result(*std::forward(exp)); + } + + detail::invoke(std::forward(f), std::forward(exp).error()); + return result(unexpect, monostate{}); +} + +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +constexpr auto map_error_impl(Exp &&exp, F &&f) + -> expected, detail::decay_t> { + using result = expected, detail::decay_t>; + + return exp.has_value() + ? result() + : result(unexpect, detail::invoke(std::forward(f), + std::forward(exp).error())); +} + +template >::value> * = nullptr, + class Ret = decltype(detail::invoke(std::declval(), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +auto map_error_impl(Exp &&exp, F &&f) -> expected, monostate> { + using result = expected, monostate>; + if (exp.has_value()) { + return result(); + } + + detail::invoke(std::forward(f), std::forward(exp).error()); + return result(unexpect, monostate{}); +} +#endif + +#ifdef TL_EXPECTED_CXX14 +template (), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +constexpr auto or_else_impl(Exp &&exp, F &&f) { + static_assert(detail::is_expected::value, "F must return an expected"); + return exp.has_value() ? std::forward(exp) + : detail::invoke(std::forward(f), + std::forward(exp).error()); +} + +template (), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +detail::decay_t or_else_impl(Exp &&exp, F &&f) { + return exp.has_value() ? std::forward(exp) + : (detail::invoke(std::forward(f), + std::forward(exp).error()), + std::forward(exp)); +} +#else +template (), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +auto or_else_impl(Exp &&exp, F &&f) -> Ret { + static_assert(detail::is_expected::value, "F must return an expected"); + return exp.has_value() ? std::forward(exp) + : detail::invoke(std::forward(f), + std::forward(exp).error()); +} + +template (), + std::declval().error())), + detail::enable_if_t::value> * = nullptr> +detail::decay_t or_else_impl(Exp &&exp, F &&f) { + return exp.has_value() ? std::forward(exp) + : (detail::invoke(std::forward(f), + std::forward(exp).error()), + std::forward(exp)); +} +#endif +} // namespace detail + +template +constexpr bool operator==(const expected &lhs, + const expected &rhs) { + return (lhs.has_value() != rhs.has_value()) + ? false + : (!lhs.has_value() ? lhs.error() == rhs.error() : *lhs == *rhs); +} +template +constexpr bool operator!=(const expected &lhs, + const expected &rhs) { + return (lhs.has_value() != rhs.has_value()) + ? true + : (!lhs.has_value() ? lhs.error() != rhs.error() : *lhs != *rhs); +} +template +constexpr bool operator==(const expected &lhs, + const expected &rhs) { + return (lhs.has_value() != rhs.has_value()) + ? false + : (!lhs.has_value() ? lhs.error() == rhs.error() : true); +} +template +constexpr bool operator!=(const expected &lhs, + const expected &rhs) { + return (lhs.has_value() != rhs.has_value()) + ? true + : (!lhs.has_value() ? lhs.error() != rhs.error() : false); +} + +template +constexpr bool operator==(const expected &x, const U &v) { + return x.has_value() ? *x == v : false; +} +template +constexpr bool operator==(const U &v, const expected &x) { + return x.has_value() ? *x == v : false; +} +template +constexpr bool operator!=(const expected &x, const U &v) { + return x.has_value() ? *x != v : true; +} +template +constexpr bool operator!=(const U &v, const expected &x) { + return x.has_value() ? *x != v : true; +} + +template +constexpr bool operator==(const expected &x, const unexpected &e) { + return x.has_value() ? false : x.error() == e.value(); +} +template +constexpr bool operator==(const unexpected &e, const expected &x) { + return x.has_value() ? false : x.error() == e.value(); +} +template +constexpr bool operator!=(const expected &x, const unexpected &e) { + return x.has_value() ? true : x.error() != e.value(); +} +template +constexpr bool operator!=(const unexpected &e, const expected &x) { + return x.has_value() ? true : x.error() != e.value(); +} + +template ::value || + std::is_move_constructible::value) && + detail::is_swappable::value && + std::is_move_constructible::value && + detail::is_swappable::value> * = nullptr> +void swap(expected &lhs, + expected &rhs) noexcept(noexcept(lhs.swap(rhs))) { + lhs.swap(rhs); +} +} // namespace tl + +#endif diff --git a/script/build.sh b/script/build.sh new file mode 100755 index 0000000..7b3e976 --- /dev/null +++ b/script/build.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +build_dir="${BUILD_DIR:-build}" +build_type="${BUILD_TYPE:-Release}" + +cmake_args=( + -S . + -B "${build_dir}" + -DCMAKE_BUILD_TYPE="${build_type}" + -DENABLE_TESTS=OFF + -DENABLE_BENCHMARKS=OFF +) + +if command -v ninja >/dev/null 2>&1; then + cmake_args+=(-G Ninja) +fi + +cmake "${cmake_args[@]}" + +cmake --build "${build_dir}" --target columnar_engine + +if [[ -x "${build_dir}/src/columnar_engine" ]]; then + ln -sf "src/columnar_engine" "${build_dir}/columnar_engine" +fi diff --git a/script/convert.sh b/script/convert.sh new file mode 100755 index 0000000..9cd90cc --- /dev/null +++ b/script/convert.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +input_csv="$1" +columnar_path="$2" +build_dir="${BUILD_DIR:-build}" +row_group_size="${ROW_GROUP_SIZE:-50000}" +schema_path="${COLUMNAR_SCHEMA:-${columnar_path}.schema.csv}" +binary="./${build_dir}/columnar_engine" + +if [[ ! -x "${binary}" ]]; then + binary="./${build_dir}/src/columnar_engine" +fi + +if [[ ! -x "${binary}" ]]; then + echo "columnar_engine binary not found; run ./script/build.sh first." >&2 + exit 1 +fi + +mkdir -p "$(dirname "${columnar_path}")" +mkdir -p "$(dirname "${schema_path}")" + +"${binary}" infer-schema \ + --input "${input_csv}" \ + --output "${schema_path}" + +"${binary}" convert \ + --schema "${schema_path}" \ + --input "${input_csv}" \ + --output "${columnar_path}" \ + --row-group-size "${row_group_size}" diff --git a/script/run_query.sh b/script/run_query.sh new file mode 100755 index 0000000..2996f56 --- /dev/null +++ b/script/run_query.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 4 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +query_num="$1" +columnar_path="$2" +output_csv="$3" +log_path="$4" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(cd "${script_dir}/.." && pwd)" +build_dir="${BUILD_DIR:-build}" +binary="${repo_dir}/${build_dir}/columnar_engine" +query_file="${repo_dir}/benchmarks/queries/query_${query_num}.sql" + +mkdir -p "$(dirname "${output_csv}")" +mkdir -p "$(dirname "${log_path}")" + +: > "${output_csv}" + +if [[ ! -f "${query_file}" ]]; then + cat > "${log_path}" < "${log_path}" < "${log_path}" 2>&1; then + printf 'query_num=%s\nstatus=ok\ncolumnar_path=%s\nquery_file=%s\n' \ + "${query_num}" "${columnar_path}" "${query_file}" >> "${log_path}" +else + status=$? + printf 'query_num=%s\nstatus=failed\ncolumnar_path=%s\nquery_file=%s\nexit_code=%s\n' \ + "${query_num}" "${columnar_path}" "${query_file}" "${status}" >> "${log_path}" + exit "${status}" +fi diff --git a/script/setup.sh b/script/setup.sh new file mode 100755 index 0000000..3e6f8f2 --- /dev/null +++ b/script/setup.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +packages=( + build-essential + cmake + ninja-build + git + ca-certificates +) + +if ! command -v apt-get >/dev/null 2>&1; then + echo "apt-get not found; assuming build dependencies are already installed." + exit 0 +fi + +missing=() +for package in "${packages[@]}"; do + if ! dpkg -s "${package}" >/dev/null 2>&1; then + missing+=("${package}") + fi +done + +if ((${#missing[@]} == 0)); then + echo "Build dependencies already installed." + exit 0 +fi + +if [[ "$(id -u)" -eq 0 ]]; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${missing[@]}" + rm -rf /var/lib/apt/lists/* + exit 0 +fi + +sudo apt-get update +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${missing[@]}" diff --git a/scripts/Dockerfile b/scripts/Dockerfile new file mode 100644 index 0000000..e4d0f97 --- /dev/null +++ b/scripts/Dockerfile @@ -0,0 +1,13 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY commands.sh env.sh sample_input.csv /bench/ +RUN chmod +x /bench/commands.sh + +WORKDIR /work + +ENTRYPOINT ["/bench/commands.sh"] diff --git a/scripts/action.py b/scripts/action.py new file mode 100644 index 0000000..a6b61a5 --- /dev/null +++ b/scripts/action.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +""" +action.py — starts Docker (if not running) and then runs act +for a GitHub Actions workflow. + +Examples: + python3 scripts/action.py + python3 scripts/action.py --workflow .github/workflows/cmake-single-platform.yml + python3 scripts/action.py -- --verbose +""" + +from __future__ import annotations + +import argparse +import os +import platform +import shutil +import subprocess +import sys +import time +from pathlib import Path +from typing import List, Optional + + +DEFAULT_WORKFLOW = ".github/workflows/cmake-single-platform.yml" +DEFAULT_PLATFORM_MAP = "ubuntu-latest=ghcr.io/catthehacker/ubuntu:act-latest" +DEFAULT_ARCH = "linux/amd64" +DEFAULT_DOCKER_TIMEOUT_SECONDS = 180 + +DEFAULT_REPO_ROOT_SENTINEL = "" + +DOCKER_DESKTOP_APP_NAME_MAC = "Docker" + +ACT_EXE = "act" +DOCKER_EXE = "docker" +GIT_EXE = "git" +SYSTEMCTL_EXE = "systemctl" +SERVICE_EXE = "service" +SUDO_EXE = "sudo" +OPEN_EXE_MAC = "open" + +APT_PACKAGES_FILE = ".github/apt-packages.txt" + + +ERR_DOCKER_NOT_FOUND = ( + "Docker executable not found in PATH. Install Docker Desktop (macOS/Windows) " + "or Docker Engine (Linux)." +) +ERR_ACT_NOT_FOUND = ( + "'act' executable not found in PATH. Install nektos/act (e.g. 'brew install act') " + "or follow the official installation docs." +) +ERR_WORKFLOW_NOT_FOUND = "Workflow file not found: {path}" +ERR_DOCKER_NOT_READY_TIMEOUT = ( + "Docker did not become ready in time. Ensure Docker Desktop is running and does not " + "require manual confirmation." +) +ERR_APT_PACKAGES_NOT_FOUND = ( + "Required file not found: {path}\n" + "Create it (repo-root/.github/apt-packages.txt) or update the workflow to not depend on it." +) + +INFO_DOCKER_STARTING_MAC = "Docker daemon is not responding — starting Docker Desktop…" +INFO_DOCKER_STARTING_LINUX = ( + "Docker daemon is not responding — trying to start docker service…" +) +INFO_DOCKER_UNSUPPORTED_OS = ( + "Docker daemon is not responding. OS '{os_name}'. Start Docker manually and retry." +) +INFO_RUNNING_ACT = "Running:" +INFO_USING_REPO_ROOT = "Using repo root:" +INFO_AUTO_DETECT_REPO_ROOT_FAILED = ( + "Auto-detecting repo root via git failed; falling back to script parent directory." +) + + +def run( + cmd: List[str], check: bool = False, capture: bool = False +) -> subprocess.CompletedProcess: + return subprocess.run( + cmd, + check=check, + text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.STDOUT if capture else None, + ) + + +def have_exe(name: str) -> bool: + return shutil.which(name) is not None + + +def resolve_path(base: Path, maybe_path: str) -> Path: + p = Path(maybe_path) + return p if p.is_absolute() else (base / p) + + +def detect_repo_root() -> Path: + if have_exe(GIT_EXE): + p = run([GIT_EXE, "rev-parse", "--show-toplevel"], check=False, capture=True) + if p.returncode == 0 and p.stdout: + return Path(p.stdout.strip()).resolve() + + print(INFO_AUTO_DETECT_REPO_ROOT_FAILED, flush=True) + return Path(__file__).resolve().parent.parent + + +def docker_daemon_ready() -> bool: + if not have_exe(DOCKER_EXE): + return False + p = run([DOCKER_EXE, "info"], check=False, capture=True) + return p.returncode == 0 + + +def start_docker_desktop_mac() -> None: + run( + [OPEN_EXE_MAC, "-g", "-a", DOCKER_DESKTOP_APP_NAME_MAC], + check=False, + capture=False, + ) + + +def start_docker_linux() -> None: + if have_exe(SYSTEMCTL_EXE): + p = run([SYSTEMCTL_EXE, "start", DOCKER_EXE], check=False, capture=True) + if p.returncode != 0 and have_exe(SUDO_EXE): + run( + [SUDO_EXE, SYSTEMCTL_EXE, "start", DOCKER_EXE], + check=False, + capture=False, + ) + elif have_exe(SERVICE_EXE): + p = run([SERVICE_EXE, DOCKER_EXE, "start"], check=False, capture=True) + if p.returncode != 0 and have_exe(SUDO_EXE): + run( + [SUDO_EXE, SERVICE_EXE, DOCKER_EXE, "start"], check=False, capture=False + ) + + +def ensure_docker_running( + timeout_seconds: int = DEFAULT_DOCKER_TIMEOUT_SECONDS, +) -> None: + if not have_exe(DOCKER_EXE): + raise RuntimeError(ERR_DOCKER_NOT_FOUND) + + if docker_daemon_ready(): + return + + system = platform.system().lower() + if system == "darwin": + print(INFO_DOCKER_STARTING_MAC, flush=True) + start_docker_desktop_mac() + elif system == "linux": + print(INFO_DOCKER_STARTING_LINUX, flush=True) + start_docker_linux() + else: + print(INFO_DOCKER_UNSUPPORTED_OS.format(os_name=platform.system()), flush=True) + + start = time.time() + while time.time() - start < timeout_seconds: + if docker_daemon_ready(): + return + time.sleep(2) + + raise RuntimeError(ERR_DOCKER_NOT_READY_TIMEOUT) + + +def ensure_act_installed() -> None: + if have_exe(ACT_EXE): + return + raise RuntimeError(ERR_ACT_NOT_FOUND) + + +def run_act(workflow: str, arch: str, platform_map: str, extra_args: List[str]) -> int: + cmd = [ + ACT_EXE, + "-W", + workflow, + "--container-architecture", + arch, + "-P", + platform_map, + ] + cmd.extend(extra_args) + + print(f"\n{INFO_RUNNING_ACT} {' '.join(cmd)}\n", flush=True) + p = subprocess.run(cmd) + return p.returncode + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Start Docker (if needed) and run act workflow." + ) + parser.add_argument( + "--workflow", + default=DEFAULT_WORKFLOW, + help="Path to workflow yml (relative to repo root)", + ) + parser.add_argument( + "--arch", default=DEFAULT_ARCH, help="act --container-architecture value" + ) + parser.add_argument( + "--platform-map", + default=DEFAULT_PLATFORM_MAP, + help='act -P mapping, e.g. "ubuntu-latest=ghcr.io/catthehacker/ubuntu:act-latest"', + ) + parser.add_argument( + "--repo-root", + default=DEFAULT_REPO_ROOT_SENTINEL, + help="Repo root. If empty, auto-detect using git (default).", + ) + parser.add_argument( + "--docker-timeout", + type=int, + default=DEFAULT_DOCKER_TIMEOUT_SECONDS, + help="Seconds to wait for Docker to become ready", + ) + parser.add_argument( + "act_args", + nargs=argparse.REMAINDER, + help="Extra args passed to act (prefix with --), e.g. -- -v", + ) + return parser + + +def main() -> int: + parser = build_arg_parser() + args = parser.parse_args() + + extra = args.act_args + if extra and extra[0] == "--": + extra = extra[1:] + + repo_root = Path(args.repo_root).resolve() if args.repo_root else detect_repo_root() + print(f"{INFO_USING_REPO_ROOT} {repo_root}", flush=True) + os.chdir(repo_root) + + apt_file = resolve_path(repo_root, APT_PACKAGES_FILE) + if not apt_file.exists(): + raise RuntimeError(ERR_APT_PACKAGES_NOT_FOUND.format(path=apt_file)) + + workflow_path = resolve_path(repo_root, args.workflow) + if not workflow_path.exists(): + raise RuntimeError(ERR_WORKFLOW_NOT_FOUND.format(path=workflow_path.resolve())) + + ensure_act_installed() + ensure_docker_running(timeout_seconds=args.docker_timeout) + + return run_act(str(workflow_path), args.arch, args.platform_map, extra) + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as e: + print(f"\nERROR: {e}\n", file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/commands.sh b/scripts/commands.sh new file mode 100755 index 0000000..8309487 --- /dev/null +++ b/scripts/commands.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${script_dir}/env.sh" + +log() { + printf '[%s] %s\n' "$(date -Iseconds)" "$*" +} + +required_vars=( + REPO_URL + BRANCH + INPUT_CSV + COLUMNAR + RESULTS +) + +for var_name in "${required_vars[@]}"; do + if [[ -z "${!var_name:-}" ]]; then + echo "Missing required environment variable: ${var_name}" >&2 + exit 1 + fi +done + +log "Cloning ${REPO_URL} (branch: ${BRANCH})" +git clone --branch "${BRANCH}" "${REPO_URL}" repo +cd repo + +mkdir -p "$(dirname "${COLUMNAR}")" "${RESULTS}" + +log "Installing dependencies" +./script/setup.sh + +log "Building project" +./script/build.sh + +log "Converting CSV ${INPUT_CSV} -> ${COLUMNAR}" +./script/convert.sh "${INPUT_CSV}" "${COLUMNAR}" + +for query_num in {0..42}; do + output="${RESULTS}/query_${query_num}.csv" + logs="${RESULTS}/query_${query_num}.log" + log "Generating placeholder output for query ${query_num}" + ./script/run_query.sh "${query_num}" "${COLUMNAR}" "${output}" "${logs}" +done + +log "Completed. Columnar file: ${COLUMNAR}; results dir: ${RESULTS}" diff --git a/scripts/dashboard.py b/scripts/dashboard.py new file mode 100644 index 0000000..9d4f052 --- /dev/null +++ b/scripts/dashboard.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import pathlib +import statistics +import subprocess +import sys +import tempfile +import time + +README_START = "" +README_END = "" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run the first N benchmark queries, write CSV stats, and refresh the README benchmark table." + ) + parser.add_argument("--count", type=int, default=17, help="Number of first queries to benchmark.") + parser.add_argument("--warm-runs", type=int, default=4, help="Number of warm runs after the first measured run.") + parser.add_argument( + "--query-dir", + type=pathlib.Path, + default=pathlib.Path("benchmarks/queries"), + help="Directory with query_.sql files.", + ) + parser.add_argument( + "--input", + type=pathlib.Path, + default=pathlib.Path("benchmarks/hits_sample.columnar"), + help="Columnar input file.", + ) + parser.add_argument( + "--binary", + type=pathlib.Path, + default=pathlib.Path("build/src/columnar_engine"), + help="Path to the columnar_engine binary.", + ) + parser.add_argument( + "--csv-out", + type=pathlib.Path, + default=pathlib.Path("benchmarks/results/readme_benchmarks.csv"), + help="Where to write the benchmark CSV.", + ) + parser.add_argument( + "--readme", + type=pathlib.Path, + default=pathlib.Path("README.md"), + help="README file to update.", + ) + parser.add_argument( + "--table-name", + default="hits", + help="Logical table name passed to run-query.", + ) + return parser.parse_args() + + +def repo_root() -> pathlib.Path: + return pathlib.Path(__file__).resolve().parent.parent + + +def resolve_path(path: pathlib.Path, root: pathlib.Path) -> pathlib.Path: + return path if path.is_absolute() else root / path + + +def format_ms(value: float) -> str: + return f"{value:.2f}" + + +def compact_sql(sql: str, max_len: int = 88) -> str: + normalized = " ".join(sql.split()) + if len(normalized) <= max_len: + return normalized + return normalized[: max_len - 1] + "…" + + +def markdown_escape(text: str) -> str: + return text.replace("|", "\\|") + + +def run_query( + binary: pathlib.Path, + input_path: pathlib.Path, + query_path: pathlib.Path, + output_path: pathlib.Path, + table_name: str, +) -> float: + command = [ + str(binary), + "run-query", + "--input", + str(input_path), + "--output", + str(output_path), + "--table-name", + table_name, + "--query-file", + str(query_path), + ] + + started_at = time.perf_counter() + completed = subprocess.run(command, capture_output=True, text=True) + elapsed_ms = (time.perf_counter() - started_at) * 1000.0 + + if completed.returncode != 0: + stderr = completed.stderr.strip() + stdout = completed.stdout.strip() + message = stderr or stdout or f"exit code {completed.returncode}" + raise RuntimeError(message) + + return elapsed_ms + + +def collect_rows( + *, + root: pathlib.Path, + binary: pathlib.Path, + input_path: pathlib.Path, + query_dir: pathlib.Path, + count: int, + warm_runs: int, + table_name: str, +) -> list[dict[str, str]]: + rows: list[dict[str, str]] = [] + + with tempfile.TemporaryDirectory(prefix="readme_benchmarks_") as temp_dir_name: + temp_dir = pathlib.Path(temp_dir_name) + + for query_id in range(count): + query_path = query_dir / f"query_{query_id}.sql" + if not query_path.exists(): + break + + sql = query_path.read_text(encoding="utf-8").strip() + output_path = temp_dir / f"query_{query_id}.csv" + + first_error = "" + status = "ok" + first_run_ms = "" + warm_avg_ms = "" + warm_min_ms = "" + warm_max_ms = "" + warm_median_ms = "" + + try: + first_run_value = run_query(binary, input_path, query_path, output_path, table_name) + first_run_ms = format_ms(first_run_value) + + warm_values: list[float] = [] + for _ in range(warm_runs): + warm_values.append(run_query(binary, input_path, query_path, output_path, table_name)) + + if warm_values: + warm_avg_ms = format_ms(sum(warm_values) / len(warm_values)) + warm_min_ms = format_ms(min(warm_values)) + warm_max_ms = format_ms(max(warm_values)) + warm_median_ms = format_ms(statistics.median(warm_values)) + except RuntimeError as error: + status = "failed" + first_error = str(error) + + rows.append( + { + "query_id": str(query_id), + "query_file": str(query_path.relative_to(root)), + "sql": sql, + "first_run_ms": first_run_ms, + "warm_avg_ms": warm_avg_ms, + "warm_median_ms": warm_median_ms, + "warm_min_ms": warm_min_ms, + "warm_max_ms": warm_max_ms, + "status": status, + "error": first_error, + } + ) + + return rows + + +def write_csv(csv_path: pathlib.Path, rows: list[dict[str, str]]) -> None: + csv_path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = [ + "query_id", + "query_file", + "sql", + "first_run_ms", + "warm_avg_ms", + "warm_median_ms", + "warm_min_ms", + "warm_max_ms", + "status", + "error", + ] + with csv_path.open("w", encoding="utf-8", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def render_markdown(csv_path: pathlib.Path, rows: list[dict[str, str]], count: int, warm_runs: int) -> str: + generated_at = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + lines = [ + "## Dashboard", + "", + "", + "| Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status |", + "| --- | ---: | ---: | ---: | ---: | ---: | --- |", + ] + + for row in rows: + query_file = row["query_file"] + query_label = f"[{row['query_id']}]({query_file})" + status = row["status"] + if row["error"]: + status = f"{status}: {markdown_escape(compact_sql(row['error'], max_len=48))}" + + lines.append( + "| " + + " | ".join( + [ + query_label, + row["first_run_ms"] or "—", + row["warm_avg_ms"] or "—", + row["warm_median_ms"] or "—", + row["warm_min_ms"] or "—", + row["warm_max_ms"] or "—", + status, + ] + ) + + " |" + ) + + return "\n".join(lines) + + +def update_readme(readme_path: pathlib.Path, markdown_block: str) -> None: + if readme_path.exists(): + content = readme_path.read_text(encoding="utf-8") + else: + content = "" + + wrapped = f"{README_START}\n{markdown_block}\n{README_END}" + + if README_START in content and README_END in content: + start = content.index(README_START) + end = content.index(README_END) + len(README_END) + updated = content[:start] + wrapped + content[end:] + else: + suffix = "" if not content or content.endswith("\n") else "\n" + updated = content + suffix + "\n" + wrapped + "\n" + + readme_path.write_text(updated, encoding="utf-8") + + +def main() -> int: + args = parse_args() + if args.count <= 0: + raise SystemExit("--count must be > 0") + if args.warm_runs < 0: + raise SystemExit("--warm-runs must be >= 0") + + root = repo_root() + binary = resolve_path(args.binary, root) + input_path = resolve_path(args.input, root) + query_dir = resolve_path(args.query_dir, root) + csv_path = resolve_path(args.csv_out, root) + readme_path = resolve_path(args.readme, root) + + if not binary.exists(): + raise SystemExit(f"binary not found: {binary}") + if not input_path.exists(): + raise SystemExit(f"input file not found: {input_path}") + if not query_dir.exists(): + raise SystemExit(f"query directory not found: {query_dir}") + + rows = collect_rows( + root=root, + binary=binary, + input_path=input_path, + query_dir=query_dir, + count=args.count, + warm_runs=args.warm_runs, + table_name=args.table_name, + ) + write_csv(csv_path, rows) + markdown_block = render_markdown(csv_path.relative_to(root), rows, args.count, args.warm_runs) + update_readme(readme_path, markdown_block) + print(f"updated {readme_path.relative_to(root)} from {csv_path.relative_to(root)} with {len(rows)} row(s)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/env.sh b/scripts/env.sh new file mode 100644 index 0000000..29dd087 --- /dev/null +++ b/scripts/env.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +: "${REPO_URL:=https://github.com/ik4rd/columnar-engine}" +: "${BRANCH:=main}" +: "${INPUT_CSV:=/bench/sample_input.csv}" +: "${COLUMNAR:=/work/results/output.columnar}" +: "${RESULTS:=/work/results}" + +export REPO_URL +export BRANCH +export INPUT_CSV +export COLUMNAR +export RESULTS diff --git a/scripts/install_dependencies.sh b/scripts/install_dependencies.sh new file mode 100755 index 0000000..7ecacf8 --- /dev/null +++ b/scripts/install_dependencies.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +APT_PACKAGES_FILE=".github/apt-packages.txt" +APT_CACHE_DIR="${HOME}/.cache/apt" +APT_ARCHIVES_DIR="${APT_CACHE_DIR}/archives" +APT_PARTIAL_DIR="${APT_ARCHIVES_DIR}/partial" + +missing=0 +while read -r pkg; do + [[ -z "${pkg}" ]] && continue + dpkg -s "${pkg}" >/dev/null 2>&1 || missing=1 +done < "${APT_PACKAGES_FILE}" + +if [[ "${missing}" -eq 0 ]]; then + echo "All dependencies already installed — skipping apt install." + exit 0 +fi + +sudo apt-get update +mkdir -p "${APT_PARTIAL_DIR}" + +sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o Dir::Cache::archives="${APT_ARCHIVES_DIR}" \ + -o Dir::Cache::archives::partial="${APT_PARTIAL_DIR}" \ + install -y --no-install-recommends $(tr '\n' ' ' < "${APT_PACKAGES_FILE}") diff --git a/scripts/sample_input.columnar b/scripts/sample_input.columnar new file mode 100644 index 0000000000000000000000000000000000000000..a6151e99661714b367b33b3802c0ca908cda4dee GIT binary patch literal 154 zcmZQ%U}9iqU}a!naLma}PGtr%obr>{fs}JbVo^?JDi?zYgB(x^0|N&TC+Fvs=H|s4 mvLJDcn2 +#include +#include +#include + +#include "io/csv_batch_io.h" +#include "convert/csv_columnar.h" +#include "support/error.h" +#include "executor/executor.h" +#include "io/fileio.h" +#include "model/schema.h" + +void ConfigureInferSchemaCommand(argparse::ArgumentParser& command) { + command.add_description("Infer a schema from a CSV file."); + command.add_argument("--input").required(); + command.add_argument("--output").required(); +} + +void ConfigureConvertCommand(argparse::ArgumentParser& command) { + command.add_description("Convert CSV to the internal columnar format."); + command.add_argument("--schema").required(); + command.add_argument("--input").required(); + command.add_argument("--output").required(); + command.add_argument("--row-group-size").scan<'u', size_t>().default_value(size_t{50000}); +} + +void ConfigureRoundtripCommand(argparse::ArgumentParser& command) { + command.add_description("Convert a columnar file back to schema CSV and data CSV."); + command.add_argument("--input").required(); + command.add_argument("--schema-output").required(); + command.add_argument("--csv-output").required(); +} + +void ConfigureRunQueryCommand(argparse::ArgumentParser& command) { + command.add_description("Execute a SQL query against a columnar file and write the result to CSV."); + command.add_argument("--input").required(); + command.add_argument("--output").required(); + command.add_argument("--table-name").default_value(std::string("hits")); + command.add_argument("--query"); + command.add_argument("--query-file"); +} + +int RunInferSchema(const argparse::ArgumentParser& command) { + const auto input_path = std::filesystem::path(command.get("--input")); + const auto output_path = std::filesystem::path(command.get("--output")); + EnsureParentDirectory(output_path); + WriteSchemaCsv(output_path, InferSchemaCsv(input_path)); + return 0; +} + +int RunConvert(const argparse::ArgumentParser& command) { + const auto output_path = std::filesystem::path(command.get("--output")); + EnsureParentDirectory(output_path); + ConvertCsvToColumnar(std::filesystem::path(command.get("--schema")), + std::filesystem::path(command.get("--input")), output_path, + command.get("--row-group-size")); + return 0; +} + +int RunRoundtrip(const argparse::ArgumentParser& command) { + const auto schema_output_path = std::filesystem::path(command.get("--schema-output")); + const auto csv_output_path = std::filesystem::path(command.get("--csv-output")); + EnsureParentDirectory(schema_output_path); + EnsureParentDirectory(csv_output_path); + ConvertColumnarToCsv(std::filesystem::path(command.get("--input")), schema_output_path, + csv_output_path); + return 0; +} + +int RunQuery(const argparse::ArgumentParser& command) { + const bool has_query = command.is_used("--query"); + const bool has_query_file = command.is_used("--query-file"); + + if (has_query == has_query_file) { + throw Error::InvalidArgument("app", "exactly one of --query or --query-file must be specified"); + } + + const std::string query = has_query ? command.get("--query") + : ReadTextFile(std::filesystem::path(command.get("--query-file"))); + + const auto output_path = std::filesystem::path(command.get("--output")); + EnsureParentDirectory(output_path); + + Executor executor; + executor.RegisterTable(command.get("--table-name"), + std::filesystem::path(command.get("--input"))); + + const ExecuteExpected result = executor.Execute(query); + if (!result.has_value()) { + throw result.error(); + } + + WriteBatchCsv(output_path, result.value()); + + return 0; +} + +int main(const int argc, char** argv) { + try { + argparse::ArgumentParser program("columnar_engine"); + argparse::ArgumentParser infer_schema_command("infer-schema"); + argparse::ArgumentParser convert_command("convert"); + argparse::ArgumentParser roundtrip_command("roundtrip"); + argparse::ArgumentParser run_query_command("run-query"); + + ConfigureInferSchemaCommand(infer_schema_command); + ConfigureConvertCommand(convert_command); + ConfigureRoundtripCommand(roundtrip_command); + ConfigureRunQueryCommand(run_query_command); + + program.add_subparser(infer_schema_command); + program.add_subparser(convert_command); + program.add_subparser(roundtrip_command); + program.add_subparser(run_query_command); + program.parse_args(argc, argv); + + if (program.is_subcommand_used("infer-schema")) { + return RunInferSchema(infer_schema_command); + } + if (program.is_subcommand_used("convert")) { + return RunConvert(convert_command); + } + if (program.is_subcommand_used("roundtrip")) { + return RunRoundtrip(roundtrip_command); + } + if (program.is_subcommand_used("run-query")) { + return RunQuery(run_query_command); + } + + throw Error::InvalidArgument("app", "a subcommand is required"); + } catch (const Error& ex) { + std::cerr << ex.what() << std::endl; + } catch (const std::exception& ex) { + std::cerr << ex.what() << std::endl; + } + + return 1; +} diff --git a/src/convert/csv_columnar.cpp b/src/convert/csv_columnar.cpp new file mode 100644 index 0000000..590ee89 --- /dev/null +++ b/src/convert/csv_columnar.cpp @@ -0,0 +1,44 @@ +#include + +#include "convert/csv_columnar.h" + +#include "io/columnar_batch_io.h" +#include "io/csv_batch_io.h" +#include "support/error.h" +#include "model/schema.h" + +void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, + const std::filesystem::path& output_path, size_t max_rows_per_group) { + if (max_rows_per_group == 0) { + throw Error::InvalidArgument("columnar", "row group size must be > 0"); + } + + const Schema schema = ReadSchemaCsv(schema_path); + + BatchSizing sizing; + sizing.max_rows = max_rows_per_group; + + CsvBatchReader batch_reader(data_path, schema, sizing); + ColumnarBatchWriter batch_writer(output_path, schema); + + while (auto batch = batch_reader.ReadNext()) { + batch_writer.Write(*batch); + } + + std::move(batch_writer).Finalize(); +} + +void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, + const std::filesystem::path& data_path) { + ColumnarBatchReader batch_reader(columnar_path); + + WriteSchemaCsv(schema_path, batch_reader.GetSchema()); + + CsvBatchWriter batch_writer(data_path, batch_reader.GetSchema()); + + while (auto batch = batch_reader.ReadNext()) { + batch_writer.Write(*batch); + } + + batch_writer.Flush(); +} diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp new file mode 100644 index 0000000..bda9e89 --- /dev/null +++ b/src/executor/aggregate_states.cpp @@ -0,0 +1,342 @@ +#include +#include +#include +#include +#include + +#include "executor/aggregate_function.h" +#include "support/ascii.h" +#include "executor/executor.h" +#include "support/int128.h" +#include "support/parsing.h" + +static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view candidate, + const std::string_view current, const bool is_min) { + switch (type) { + case ColumnType::Boolean: { + const bool left = ParseBoolean(std::string(candidate)); + const bool right = ParseBoolean(std::string(current)); + return is_min ? left < right : left > right; + } + case ColumnType::Int16: { + const int16_t left = ParseInt16(std::string(candidate)); + const int16_t right = ParseInt16(std::string(current)); + return is_min ? left < right : left > right; + } + case ColumnType::Int32: { + const int32_t left = ParseInt32(std::string(candidate)); + const int32_t right = ParseInt32(std::string(current)); + return is_min ? left < right : left > right; + } + case ColumnType::Int64: { + const int64_t left = ParseInt64(std::string(candidate)); + const int64_t right = ParseInt64(std::string(current)); + return is_min ? left < right : left > right; + } + case ColumnType::Int128: { + const Int128 left = ParseInt128(std::string(candidate)); + const Int128 right = ParseInt128(std::string(current)); + return is_min ? left < right : left > right; + } + case ColumnType::String: + return is_min ? candidate < current : candidate > current; + case ColumnType::Date: { + const int32_t left = ParseDate(std::string(candidate)); + const int32_t right = ParseDate(std::string(current)); + return is_min ? left < right : left > right; + } + case ColumnType::Timestamp: { + const int64_t left = ParseTimestamp(std::string(candidate)); + const int64_t right = ParseTimestamp(std::string(current)); + return is_min ? left < right : left > right; + } + case ColumnType::Character: { + const char left = ParseCharacter(std::string(candidate)); + const char right = ParseCharacter(std::string(current)); + return is_min ? left < right : left > right; + } + } + throw Error::Unsupported("executor", "unsupported column type in comparison"); +} + +static Int128 ParseToInt128(const ColumnType type, const std::string_view value) { + switch (type) { + case ColumnType::Boolean: + return ParseBoolean(std::string(value)) ? 1 : 0; + case ColumnType::Int16: + return ParseInt16(std::string(value)); + case ColumnType::Int32: + return ParseInt32(std::string(value)); + case ColumnType::Int64: + return ParseInt64(std::string(value)); + case ColumnType::Int128: + return ParseInt128(std::string(value)); + case ColumnType::Character: + return ParseCharacter(std::string(value)); + case ColumnType::String: + case ColumnType::Date: + case ColumnType::Timestamp: + break; + } + throw Error::Unsupported("executor", "unsupported numeric conversion"); +} + +static std::string FormatAverage(const Int128 sum, const uint64_t count) { + if (count == 0) { + return "0"; + } + return Int128ToString(sum / static_cast(count)); +} + +class CountAS final : public AggState { + public: + void Consume(const std::optional value) override { + (void)value; + ++count_; + } + + std::string Finalize() const override { return std::to_string(count_); } + + private: + uint64_t count_ = 0; +}; + +class SumAS final : public AggState { + public: + explicit SumAS(const ColumnType type) : type_(type) {} + + public: + void Consume(const std::optional value) override { + if (!value.has_value()) { + throw Error::InvalidState("executor", "SUM requires a value"); + } + sum_ += ParseToInt128(type_, *value); + } + + std::string Finalize() const override { return Int128ToString(sum_); } + + private: + ColumnType type_; + Int128 sum_ = 0; +}; + +class AvgAS final : public AggState { + public: + explicit AvgAS(const ColumnType type) : type_(type) {} + + public: + void Consume(const std::optional value) override { + if (!value.has_value()) { + throw Error::InvalidState("executor", "AVG requires a value"); + } + sum_ += ParseToInt128(type_, *value); + ++count_; + } + + std::string Finalize() const override { return FormatAverage(sum_, count_); } + + private: + ColumnType type_; + Int128 sum_ = 0; + uint64_t count_ = 0; +}; + +class ExtremumAS final : public AggState { + public: + ExtremumAS(const ColumnType type, const bool is_min) : type_(type), is_min_(is_min) {} + + public: + void Consume(const std::optional value) override { + if (!value.has_value()) { + throw Error::InvalidState("executor", "MIN/MAX requires a value"); + } + + if (!extremum_.has_value()) { + extremum_ = std::string(*value); + return; + } + + if (ShouldReplaceExtremum(type_, *value, *extremum_, is_min_)) { + extremum_ = std::string(*value); + } + } + + std::string Finalize() const override { return extremum_.value_or(""); } + + private: + ColumnType type_; + bool is_min_ = true; + std::optional extremum_; +}; + +class DistinctAS final : public AggState { + public: + explicit DistinctAS(std::unique_ptr nested) : nested_(std::move(nested)) {} + + public: + void Consume(const std::optional value) override { + if (!value.has_value()) { + throw Error::InvalidState("executor", "DISTINCT requires a value"); + } + + if (seen_.insert(std::string(*value)).second) { + nested_->Consume(value); + } + } + + std::string Finalize() const override { return nested_->Finalize(); } + + private: + std::unique_ptr nested_; + std::unordered_set seen_; +}; + +static bool SupportsAnyType(const ColumnType type) { + (void)type; + return true; +} + +static bool SupportsNumericType(const ColumnType type) { + switch (type) { + case ColumnType::Boolean: + case ColumnType::Int16: + case ColumnType::Int32: + case ColumnType::Int64: + case ColumnType::Int128: + case ColumnType::Character: + return true; + case ColumnType::String: + case ColumnType::Date: + case ColumnType::Timestamp: + return false; + } + return false; +} + +static std::unique_ptr CreateCountState(const PlannedAgg& aggregate) { + (void)aggregate; + return std::make_unique(); +} + +static std::unique_ptr CreateSumState(const PlannedAgg& aggregate) { + return std::make_unique(aggregate.input_type); +} + +static std::unique_ptr CreateAvgState(const PlannedAgg& aggregate) { + return std::make_unique(aggregate.input_type); +} + +static std::unique_ptr CreateMinState(const PlannedAgg& aggregate) { + return std::make_unique(aggregate.input_type, true); +} + +static std::unique_ptr CreateMaxState(const PlannedAgg& aggregate) { + return std::make_unique(aggregate.input_type, false); +} + +AggRegistry& AggRegistry::Instance() { + static AggRegistry instance; + return instance; +} + +void AggRegistry::Register(const AggFuncDefinition& def) { registry_[ToUpperAscii(def.canonical_name)] = def; } + +const AggFuncDefinition* AggRegistry::Find(const std::string_view name) const { + const auto it = registry_.find(ToUpperAscii(name)); + if (it == registry_.end()) { + return nullptr; + } + return &it->second; +} + +AggRegistrar::AggRegistrar(const AggFuncDefinition& def) { AggRegistry::Instance().Register(def); } + +[[maybe_unused]] static AggRegistrar register_count({ + .canonical_name = "COUNT", + .distinct = true, + .star = true, + .supports_type = &SupportsAnyType, + .factory = &CreateCountState, +}); + +[[maybe_unused]] static AggRegistrar register_sum({ + .canonical_name = "SUM", + .distinct = false, + .star = false, + .supports_type = &SupportsNumericType, + .factory = &CreateSumState, +}); + +[[maybe_unused]] static AggRegistrar register_avg({ + .canonical_name = "AVG", + .distinct = false, + .star = false, + .supports_type = &SupportsNumericType, + .factory = &CreateAvgState, +}); + +[[maybe_unused]] static AggRegistrar register_min({ + .canonical_name = "MIN", + .distinct = false, + .star = false, + .supports_type = &SupportsAnyType, + .factory = &CreateMinState, +}); + +[[maybe_unused]] static AggRegistrar register_max({ + .canonical_name = "MAX", + .distinct = false, + .star = false, + .supports_type = &SupportsAnyType, + .factory = &CreateMaxState, +}); + +const AggFuncDefinition& ResolveAggFunc(const std::string_view name) { + const AggFuncDefinition* definition = AggRegistry::Instance().Find(name); + if (definition == nullptr) { + throw Error::Unsupported("executor", "aggregate '" + std::string(name) + "' is not registered"); + } + return *definition; +} + +bool AggSupportsInputType(const AggFuncDefinition& definition, const ColumnType type) { + if (definition.supports_type == nullptr) { + return false; + } + return definition.supports_type(type); +} + +std::string FormatAgg(const AggFuncDefinition& definition, const std::string_view column_name, const bool distinct, + const bool star) { + std::string output(definition.canonical_name); + output += '('; + if (star) { + output += '*'; + } else { + if (distinct) { + output += "DISTINCT "; + } + output += column_name; + } + output += ')'; + return output; +} + +std::unique_ptr CreateAggState(const PlannedAgg& aggregate) { + if (aggregate.function == nullptr || aggregate.function->factory == nullptr) { + throw Error::Unsupported("executor", "aggregate is not registered"); + } + if (aggregate.distinct && !aggregate.function->distinct) { + throw Error::Unsupported("executor", "DISTINCT is not supported for aggregate '" + + std::string(aggregate.function->canonical_name) + "'"); + } + if (aggregate.star && !aggregate.function->star) { + throw Error::Unsupported( + "executor", "aggregate '" + std::string(aggregate.function->canonical_name) + "' does not support '*'"); + } + std::unique_ptr state = aggregate.function->factory(aggregate); + if (aggregate.distinct) { + state = std::make_unique(std::move(state)); + } + return state; +} diff --git a/src/executor/executor.cpp b/src/executor/executor.cpp new file mode 100644 index 0000000..91e222d --- /dev/null +++ b/src/executor/executor.cpp @@ -0,0 +1,27 @@ +#include "executor/executor.h" + +#include + +#include "support/ascii.h" + +void Executor::RegisterTable(const std::string& name, std::filesystem::path path) { + tables_[ToLowerAscii(name)] = std::move(path); +} + +ExecuteExpected Executor::Execute(const std::string_view query) const { + try { + const ParsedQuery parsed = ParseQuery(query); + const PlannedQuery planned = PlanQuery(parsed, tables_); + + std::unique_ptr root = BuildPlan(planned); + + auto batch = root->Next(); + if (!batch.has_value()) { + return Batch{}; + } + + return std::move(*batch); + } catch (const Error& error) { + return tl::unexpected(error); + } +} diff --git a/src/executor/operator_utils.cpp b/src/executor/operator_utils.cpp new file mode 100644 index 0000000..6b047f4 --- /dev/null +++ b/src/executor/operator_utils.cpp @@ -0,0 +1,83 @@ +#include "executor/operator_utils.h" + +#include + +#include "support/ascii.h" +#include "support/parsing.h" + +template +static bool EvaluateComparison(const T left, const T right, const ComparisonKind kind) { + if (kind == ComparisonKind::Equal) { + return left == right; + } + if (kind == ComparisonKind::NotEqual) { + return left != right; + } + if (kind == ComparisonKind::Less) { + return left < right; + } + if (kind == ComparisonKind::LessOrEqual) { + return left <= right; + } + if (kind == ComparisonKind::Greater) { + return left > right; + } + if (kind == ComparisonKind::GreaterOrEqual) { + return left >= right; + } + throw Error::Unsupported("executor", "unsupported comparison kind"); +} + +template +static int CompareValuesImpl(const T left, const T right) { + if (left < right) { + return -1; + } + if (left > right) { + return 1; + } + return 0; +} + +int CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { + switch (type) { + case ColumnType::Boolean: + return CompareValuesImpl(ParseBoolean(std::string(lhs)), ParseBoolean(std::string(rhs))); + case ColumnType::Int16: + return CompareValuesImpl(ParseInt16(std::string(lhs)), ParseInt16(std::string(rhs))); + case ColumnType::Int32: + return CompareValuesImpl(ParseInt32(std::string(lhs)), ParseInt32(std::string(rhs))); + case ColumnType::Int64: + return CompareValuesImpl(ParseInt64(std::string(lhs)), ParseInt64(std::string(rhs))); + case ColumnType::Int128: + return CompareValuesImpl(ParseInt128(std::string(lhs)), ParseInt128(std::string(rhs))); + case ColumnType::String: + return CompareValuesImpl(std::string(lhs), std::string(rhs)); + case ColumnType::Date: + return CompareValuesImpl(ParseDate(std::string(lhs)), ParseDate(std::string(rhs))); + case ColumnType::Timestamp: + return CompareValuesImpl(ParseTimestamp(std::string(lhs)), ParseTimestamp(std::string(rhs))); + case ColumnType::Character: + return CompareValuesImpl(ParseCharacter(std::string(lhs)), ParseCharacter(std::string(rhs))); + } + throw Error::Unsupported("executor", "unsupported column type in comparison"); +} + +bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, + const ComparisonKind kind) { + return EvaluateComparison(CompareValues(type, lhs, rhs), 0, kind); +} + +ColumnType AggregateOutputType(const PlannedAgg& aggregate) { + const std::string name = ToUpperAscii(aggregate.function->canonical_name); + if (name == "COUNT") { + return ColumnType::Int64; + } + if (name == "SUM" || name == "AVG") { + return ColumnType::Int128; + } + if (name == "MIN" || name == "MAX") { + return aggregate.input_type; + } + throw Error::Unsupported("executor", "unsupported aggregate result type"); +} diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp new file mode 100644 index 0000000..886708e --- /dev/null +++ b/src/executor/operators.cpp @@ -0,0 +1,356 @@ +#include +#include +#include +#include +#include + +#include "executor/executor.h" +#include "executor/operator_utils.h" +#include "io/columnar_batch_io.h" +#include "io/fileio.h" + +class ScanOperator final : public Operator { + public: + ScanOperator(std::filesystem::path path, std::vector projection_indexes) + : path_(std::move(path)), in_(OpenInputFile(path_)), metadata_(ColumnarBatchReader(path_).GetMetadata()) { + if (metadata_.schema.columns.empty()) { + throw Error::InvalidData("operators", "columnar schema is empty", path_.string()); + } + + projection_indexes_ = std::move(projection_indexes); + projected_schema_.columns.reserve(projection_indexes_.size()); + + for (const size_t source_index : projection_indexes_) { + if (source_index >= metadata_.schema.columns.size()) { + throw Error::OutOfRange("operators", "projection index out of range", path_.string()); + } + projected_schema_.columns.push_back(metadata_.schema.columns[source_index]); + } + } + + public: + std::optional Next() override { + if (next_group_ >= metadata_.row_groups.size()) { + return std::nullopt; + } + + const auto& row_group = metadata_.row_groups[next_group_++]; + Batch batch(projected_schema_, row_group.row_count); + + for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { + const size_t source_index = projection_indexes_[projected_index]; + const auto& chunk = row_group.columns[source_index]; + SeekInputFile(in_, path_, chunk.offset); + batch.ColumnAt(projected_index).ReadFrom(in_, row_group.row_count, chunk.size); + } + + return batch; + } + + private: + std::filesystem::path path_; + std::ifstream in_; + + ColumnarMetadata metadata_; + std::vector projection_indexes_; + Schema projected_schema_; + size_t next_group_ = 0; +}; + +class FilterOperator final : public Operator { + public: + FilterOperator(std::unique_ptr child, PlannedFilter filter) + : child_(std::move(child)), filter_(std::move(filter)) {} + + public: + std::optional Next() override { + while (auto batch = child_->Next()) { + Batch filtered(batch->GetSchema()); + + for (size_t row = 0; row < batch->RowsCount(); ++row) { + const std::string value = batch->ColumnAt(filter_.column_index).ValueAsString(row); + + if (!MatchesComparison(filter_.column_type, value, filter_.literal_text, filter_.comparison)) { + continue; + } + + for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { + filtered.ColumnAt(column_index).AppendFromString(batch->ColumnAt(column_index).ValueAsString(row)); + } + } + + if (filtered.RowsCount() > 0) { + return filtered; + } + } + + return std::nullopt; + } + + private: + std::unique_ptr child_; + PlannedFilter filter_; +}; + +struct AggBinding { + PlannedAgg aggregate; + std::unique_ptr state; +}; + +class AggOperator final : public Operator { + public: + AggOperator(std::unique_ptr child, const std::vector& aggregates) : child_(std::move(child)) { + bindings_.reserve(aggregates.size()); + for (const auto& aggregate : aggregates) { + bindings_.push_back(AggBinding{ + .aggregate = aggregate, + .state = CreateAggState(aggregate), + }); + } + } + + public: + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + returned_ = true; + + while (auto batch = child_->Next()) { + for (size_t row = 0; row < batch->RowsCount(); ++row) { + for (auto& binding : bindings_) { + if (!binding.aggregate.star) { + std::string materialized = batch->ColumnAt(*binding.aggregate.column_index).ValueAsString(row); + binding.state->Consume(materialized); + } else { + binding.state->Consume(std::nullopt); + } + } + } + } + + Schema schema; + schema.columns.reserve(bindings_.size()); + + for (const auto& binding : bindings_) { + schema.columns.push_back(ColumnSchema{ + .name = binding.aggregate.output_name, + .type = AggregateOutputType(binding.aggregate), + }); + } + + Batch result(schema); + + for (size_t i = 0; i < bindings_.size(); ++i) { + result.ColumnAt(i).AppendFromString(bindings_[i].state->Finalize()); + } + + return result; + } + + private: + std::unique_ptr child_; + std::vector bindings_; + bool returned_ = false; +}; + +class GroupAggOperator final : public Operator { + public: + GroupAggOperator(std::unique_ptr child, std::vector group_keys, + std::vector aggregates, std::vector select_items, + std::vector order_by) + : child_(std::move(child)), + group_keys_(std::move(group_keys)), + aggregates_(std::move(aggregates)), + select_items_(std::move(select_items)), + order_by_(std::move(order_by)) {} + + public: + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + returned_ = true; + + while (auto batch = child_->Next()) { + for (size_t row = 0; row < batch->RowsCount(); ++row) { + std::vector key_values; + key_values.reserve(group_keys_.size()); + + std::string group_id; + for (const auto& group_key : group_keys_) { + std::string value = batch->ColumnAt(group_key.column_index).ValueAsString(row); + group_id += std::to_string(value.size()); + group_id.push_back(':'); + group_id += value; + group_id.push_back('|'); + key_values.push_back(std::move(value)); + } + + size_t group_index = 0; + const auto it = group_indexes_.find(group_id); + if (it == group_indexes_.end()) { + group_index = groups_.size(); + group_indexes_.emplace(std::move(group_id), group_index); + groups_.push_back(GroupState{ + .key_values = std::move(key_values), + .states = CreateStates(), + }); + } else { + group_index = it->second; + } + + GroupState& group = groups_[group_index]; + for (size_t i = 0; i < aggregates_.size(); ++i) { + const PlannedAgg& aggregate = aggregates_[i]; + if (!aggregate.star) { + std::string materialized = batch->ColumnAt(*aggregate.column_index).ValueAsString(row); + group.states[i]->Consume(materialized); + } else { + group.states[i]->Consume(std::nullopt); + } + } + } + } + + Schema schema; + schema.columns.reserve(select_items_.size()); + for (const auto& item : select_items_) { + schema.columns.push_back(ColumnSchema{ + .name = item.output_name, + .type = item.output_type, + }); + } + + std::vector> rows; + rows.reserve(groups_.size()); + for (const auto& group : groups_) { + rows.push_back(MaterializeRow(group)); + } + + if (!order_by_.empty()) { + std::ranges::stable_sort( + rows, [&](const std::vector& lhs, const std::vector& rhs) { + for (const auto& order : order_by_) { + const int comparison = CompareValues(order.value_type, lhs[order.result_column_index], + rhs[order.result_column_index]); + if (comparison != 0) { + return order.descending ? comparison > 0 : comparison < 0; + } + } + + for (size_t i = 0; i < select_items_.size(); ++i) { + if (select_items_[i].kind != SelectItemKind::GroupKey) { + continue; + } + const int comparison = CompareValues(select_items_[i].output_type, lhs[i], rhs[i]); + if (comparison != 0) { + return comparison > 0; + } + } + return false; + }); + } + + Batch result(schema, rows.size()); + for (const auto& row : rows) { + for (size_t i = 0; i < row.size(); ++i) { + result.ColumnAt(i).AppendFromString(row[i]); + } + } + + return result; + } + + private: + struct GroupState { + std::vector key_values; + std::vector> states; + }; + + std::vector> CreateStates() const { + std::vector> states; + states.reserve(aggregates_.size()); + for (const auto& aggregate : aggregates_) { + states.push_back(CreateAggState(aggregate)); + } + return states; + } + + std::vector MaterializeRow(const GroupState& group) const { + std::vector row; + row.reserve(select_items_.size()); + + for (const auto& item : select_items_) { + if (item.kind == SelectItemKind::GroupKey) { + row.push_back(group.key_values[item.index]); + } else { + row.push_back(group.states[item.index]->Finalize()); + } + } + + return row; + } + + private: + std::unique_ptr child_; + std::vector group_keys_; + std::vector aggregates_; + std::vector select_items_; + std::vector order_by_; + std::unordered_map group_indexes_; + std::vector groups_; + bool returned_ = false; +}; + +class LimitOperator final : public Operator { + public: + LimitOperator(std::unique_ptr child, const size_t limit) : child_(std::move(child)), remaining_(limit) {} + + public: + std::optional Next() override { + if (remaining_ == 0) { + return std::nullopt; + } + + while (auto batch = child_->Next()) { + if (batch->RowsCount() <= remaining_) { + remaining_ -= batch->RowsCount(); + return batch; + } + + Batch limited(batch->GetSchema(), remaining_); + for (size_t row = 0; row < remaining_; ++row) { + for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { + limited.ColumnAt(column_index).AppendFromString(batch->ColumnAt(column_index).ValueAsString(row)); + } + } + remaining_ = 0; + return limited; + } + + remaining_ = 0; + return std::nullopt; + } + + private: + std::unique_ptr child_; + size_t remaining_; +}; + +std::unique_ptr BuildPlan(const PlannedQuery& planned) { + std::unique_ptr root = std::make_unique(planned.table_path, planned.projection_indexes); + if (planned.filter.has_value()) { + root = std::make_unique(std::move(root), *planned.filter); + } + if (!planned.group_keys.empty()) { + root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, + planned.select_items, planned.order_by); + } else { + root = std::make_unique(std::move(root), planned.aggregates); + } + if (planned.limit.has_value()) { + root = std::make_unique(std::move(root), *planned.limit); + } + return root; +} diff --git a/src/executor/planner_utils.cpp b/src/executor/planner_utils.cpp new file mode 100644 index 0000000..1aa3eb2 --- /dev/null +++ b/src/executor/planner_utils.cpp @@ -0,0 +1,95 @@ +#include "executor/planner_utils.h" + +#include "support/ascii.h" + +size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { + const std::string needle = ToLowerAscii(column_name); + for (size_t i = 0; i < schema.columns.size(); ++i) { + if (ToLowerAscii(schema.columns[i].name) == needle) { + return i; + } + } + throw Error::InvalidArgument("query_planner", "unknown column '" + std::string(column_name) + "'"); +} + +static bool SameQualifier(const std::string_view lhs, const std::string_view rhs) { + return ToLowerAscii(lhs) == ToLowerAscii(rhs); +} + +void ValidateColumnQualifier(const ParsedQuery& parsed, const ColumnRef& column) { + if (column.qualifier.empty()) { + return; + } + + if (SameQualifier(column.qualifier, parsed.table_name)) { + return; + } + if (!parsed.table_alias.empty() && SameQualifier(column.qualifier, parsed.table_alias)) { + return; + } + + throw Error::InvalidArgument("query_planner", "unknown table qualifier '" + column.qualifier + "'"); +} + +static std::string UnescapeSqlString(const std::string_view text) { + if (text.size() < 2 || text.front() != '\'' || text.back() != '\'') { + throw Error::InvalidArgument("query_planner", "invalid string literal"); + } + + std::string result; + result.reserve(text.size() - 2); + + for (size_t i = 1; i + 1 < text.size(); ++i) { + if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { + result.push_back('\''); + ++i; + continue; + } + result.push_back(text[i]); + } + + return result; +} + +std::string NormalizeLiteral(const ParsedLiteral& literal, const ColumnType type) { + switch (literal.token_type) { + case Tokens::StringLiteral: + return UnescapeSqlString(literal.text); + case Tokens::NumericLiteral: + case Tokens::NameToken: + if (type == ColumnType::String && literal.token_type == Tokens::NameToken) { + return literal.text; + } + return literal.text; + default: + throw Error::InvalidArgument("query_planner", "unsupported literal type"); + } +} + +bool SameColumnName(const std::string_view lhs, const std::string_view rhs) { return ToLowerAscii(lhs) == ToLowerAscii(rhs); } + +bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs) { + if (!SameColumnName(lhs.name, rhs.name)) { + return false; + } + if (lhs.qualifier.empty() || rhs.qualifier.empty()) { + return true; + } + return SameQualifier(lhs.qualifier, rhs.qualifier); +} + +bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { + if (lhs.kind != rhs.kind) { + return false; + } + + if (lhs.kind == SelectItemKind::GroupKey) { + return SameColumnRef(lhs.column, rhs.column); + } + + return ToLowerAscii(lhs.aggregate.output_name) == ToLowerAscii(rhs.aggregate.output_name); +} + +bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { + return SameColumnName(lhs.output_name, rhs.output_name); +} diff --git a/src/executor/query_parser.cpp b/src/executor/query_parser.cpp new file mode 100644 index 0000000..86a1e87 --- /dev/null +++ b/src/executor/query_parser.cpp @@ -0,0 +1,297 @@ +#include +#include +#include + +#include "executor/executor.h" +#include "support/ascii.h" + +class TokenCursor { + public: + explicit TokenCursor(std::vector tokens) : tokens_(std::move(tokens)) {} + + public: + const Token& Peek(const size_t lookahead = 0) const { + if (pos_ + lookahead >= tokens_.size()) { + throw Error::InvalidArgument("query_parser", "unexpected end of query"); + } + return *tokens_[pos_ + lookahead]; + } + + bool Match(const Tokens type, const size_t lookahead = 0) const { + return pos_ + lookahead < tokens_.size() && tokens_[pos_ + lookahead]->GetType() == type; + } + + const Token& Consume(const Tokens type, const std::string_view message) { + if (!Match(type)) { + throw Error::InvalidArgument("query_parser", std::string(message)); + } + return *tokens_[pos_++]; + } + + bool TryConsume(const Tokens type) { + if (!Match(type)) { + return false; + } + ++pos_; + return true; + } + + const Token& ConsumeFunctionToken(const std::string_view message) { + if (pos_ >= tokens_.size()) { + throw Error::InvalidArgument("query_parser", std::string(message)); + } + + const Tokens type = tokens_[pos_]->GetType(); + switch (type) { + case Tokens::NumericLiteral: + case Tokens::StringLiteral: + case Tokens::OpenBracket: + case Tokens::CloseBracket: + case Tokens::Comma: + case Tokens::Dot: + case Tokens::Semicolon: + case Tokens::Equal: + case Tokens::NotEqual: + case Tokens::Less: + case Tokens::LessOrEqual: + case Tokens::Greater: + case Tokens::GreaterOrEqual: + case Tokens::EndOfInput: + throw Error::InvalidArgument("query_parser", std::string(message)); + default: + break; + } + + return *tokens_[pos_++]; + } + + private: + std::vector tokens_; + size_t pos_ = 0; +}; + +static ComparisonKind ComparisonKindFromToken(const Tokens type) { + switch (type) { + case Tokens::Equal: + return ComparisonKind::Equal; + case Tokens::NotEqual: + return ComparisonKind::NotEqual; + case Tokens::Less: + return ComparisonKind::Less; + case Tokens::LessOrEqual: + return ComparisonKind::LessOrEqual; + case Tokens::Greater: + return ComparisonKind::Greater; + case Tokens::GreaterOrEqual: + return ComparisonKind::GreaterOrEqual; + default: + throw Error::InvalidArgument("query_parser", "expected comparison operator in WHERE"); + } +} + +static std::string FormatColumnRef(const ColumnRef& column) { + if (column.qualifier.empty()) { + return column.name; + } + return column.qualifier + "." + column.name; +} + +static ColumnRef ParseColumnRef(TokenCursor& cursor, const std::string_view message) { + ColumnRef column{ + .qualifier = {}, + .name = std::string(cursor.Consume(Tokens::NameToken, message).GetText()), + }; + + if (cursor.TryConsume(Tokens::Dot)) { + column.qualifier = std::move(column.name); + column.name = std::string(cursor.Consume(Tokens::NameToken, "expected column name after '.'").GetText()); + } + + return column; +} + +static std::optional ParseOptionalLimit(TokenCursor& cursor) { + if (!cursor.TryConsume(Tokens::Limit)) { + return std::nullopt; + } + + const Token& limit_token = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after LIMIT"); + size_t limit = 0; + const std::string_view text = limit_token.GetText(); + const char* begin = text.data(); + const char* end = begin + text.size(); + if (const auto [ptr, ec] = std::from_chars(begin, end, limit); ec != std::errc() || ptr != end) { + throw Error::InvalidArgument("query_parser", "invalid LIMIT value"); + } + + return limit; +} + +static AggSpec ParseAggregate(TokenCursor& cursor) { + const Token& function_token = cursor.ConsumeFunctionToken("expected aggregate function"); + + AggSpec spec; + spec.function = &ResolveAggFunc(function_token.GetText()); + + cursor.Consume(Tokens::OpenBracket, "expected '(' after aggregate function"); + + if (cursor.TryConsume(Tokens::Asterisk)) { + if (!spec.function->star) { + throw Error::InvalidArgument("query_parser", + std::string(spec.function->canonical_name) + " does not support '*'"); + } + spec.star = true; + } else { + spec.distinct = cursor.TryConsume(Tokens::Distinct); + if (spec.distinct && !spec.function->distinct) { + throw Error::InvalidArgument("query_parser", + std::string(spec.function->canonical_name) + " does not support DISTINCT"); + } + spec.column = ParseColumnRef(cursor, "expected column name in aggregate"); + } + + cursor.Consume(Tokens::CloseBracket, "expected ')' after aggregate arguments"); + spec.output_name = FormatAgg(*spec.function, FormatColumnRef(spec.column), spec.distinct, spec.star); + + return spec; +} + +static SelectItemSpec ParseSelectExpression(TokenCursor& cursor) { + if (cursor.Match(Tokens::NameToken) && !cursor.Match(Tokens::OpenBracket, 1)) { + const ColumnRef column = ParseColumnRef(cursor, "expected column name in SELECT"); + return SelectItemSpec{ + .kind = SelectItemKind::GroupKey, + .column = column, + .aggregate = {}, + .output_name = column.name, + }; + } + + return SelectItemSpec{ + .kind = SelectItemKind::Aggregate, + .column = {}, + .aggregate = ParseAggregate(cursor), + .output_name = {}, + }; +} + +static std::string ParseSelectItemAlias(TokenCursor& cursor) { + if (cursor.TryConsume(Tokens::As)) { + return std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); + } + if (cursor.Match(Tokens::NameToken)) { + return std::string(cursor.Consume(Tokens::NameToken, "expected alias").GetText()); + } + return {}; +} + +static SelectItemSpec ParseSelectItem(TokenCursor& cursor) { + SelectItemSpec item = ParseSelectExpression(cursor); + if (const std::string alias = ParseSelectItemAlias(cursor); !alias.empty()) { + item.output_name = alias; + } else if (item.kind == SelectItemKind::Aggregate) { + item.output_name = item.aggregate.output_name; + } + return item; +} + +static OrderBySpec ParseOrderByItem(TokenCursor& cursor) { + SelectItemSpec item = ParseSelectExpression(cursor); + item.output_name = item.kind == SelectItemKind::Aggregate ? item.aggregate.output_name : item.column.name; + + bool descending = false; + if (cursor.Match(Tokens::NameToken)) { + const std::string direction = ToUpperAscii(cursor.Peek().GetText()); + if (direction == "DESC") { + cursor.Consume(Tokens::NameToken, "expected DESC"); + descending = true; + } else if (direction == "ASC") { + cursor.Consume(Tokens::NameToken, "expected ASC"); + } + } + + return OrderBySpec{ + .item = std::move(item), + .descending = descending, + }; +} + +ParsedQuery ParseQuery(const std::string_view query) { + auto tokenized = TokenizeSql(query); + if (!tokenized.has_value()) { + throw tokenized.error(); + } + + TokenCursor cursor(std::move(tokenized.value())); + + ParsedQuery parsed; + cursor.Consume(Tokens::Select, "expected SELECT"); + + parsed.select_items.push_back(ParseSelectItem(cursor)); + while (cursor.TryConsume(Tokens::Comma)) { + parsed.select_items.push_back(ParseSelectItem(cursor)); + } + + cursor.Consume(Tokens::From, "expected FROM"); + parsed.table_name = std::string(cursor.Consume(Tokens::NameToken, "expected table name").GetText()); + if (cursor.TryConsume(Tokens::As)) { + parsed.table_alias = std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); + } else if (cursor.Match(Tokens::NameToken)) { + parsed.table_alias = std::string(cursor.Consume(Tokens::NameToken, "expected table alias").GetText()); + } + + if (cursor.TryConsume(Tokens::Where)) { + FilterSpec filter; + filter.column = ParseColumnRef(cursor, "expected column name after WHERE"); + + const Tokens comparison_token = cursor.Peek().GetType(); + filter.comparison = ComparisonKindFromToken(comparison_token); + + cursor.Consume(comparison_token, "expected comparison operator in WHERE"); + + if (cursor.TryConsume(Tokens::Minus)) { + const Token& literal = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after '-'"); + filter.literal = + ParsedLiteral{.text = "-" + std::string(literal.GetText()), .token_type = Tokens::NumericLiteral}; + } else { + const Token& literal = cursor.Peek(); + switch (literal.GetType()) { + case Tokens::NumericLiteral: + case Tokens::StringLiteral: + case Tokens::NameToken: + filter.literal = ParsedLiteral{ + .text = std::string(literal.GetText()), + .token_type = literal.GetType(), + }; + cursor.Consume(literal.GetType(), "expected literal after comparison operator"); + break; + default: + throw Error::InvalidArgument("query_parser", "unsupported literal in WHERE"); + } + } + + parsed.filter = std::move(filter); + } + + if (cursor.TryConsume(Tokens::Group)) { + cursor.Consume(Tokens::By, "expected BY after GROUP"); + parsed.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); + while (cursor.TryConsume(Tokens::Comma)) { + parsed.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); + } + } + + if (cursor.TryConsume(Tokens::Order)) { + cursor.Consume(Tokens::By, "expected BY after ORDER"); + parsed.order_by.push_back(ParseOrderByItem(cursor)); + while (cursor.TryConsume(Tokens::Comma)) { + parsed.order_by.push_back(ParseOrderByItem(cursor)); + } + } + + parsed.limit = ParseOptionalLimit(cursor); + cursor.TryConsume(Tokens::Semicolon); + cursor.Consume(Tokens::EndOfInput, "unexpected trailing tokens"); + + return parsed; +} diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp new file mode 100644 index 0000000..9de1d36 --- /dev/null +++ b/src/executor/query_planner.cpp @@ -0,0 +1,151 @@ +#include + +#include "executor/executor.h" +#include "executor/operator_utils.h" +#include "executor/planner_utils.h" +#include "io/columnar_batch_io.h" +#include "support/ascii.h" + +PlannedQuery PlanQuery(const ParsedQuery& parsed, + const std::unordered_map& tables) { + PlannedQuery planned; + + const auto table_it = tables.find(ToLowerAscii(parsed.table_name)); + if (table_it == tables.end()) { + throw Error::NotFound("query_planner", "unknown table", parsed.table_name); + } + + planned.table_path = table_it->second; + + const ColumnarBatchReader reader(planned.table_path); + const Schema& schema = reader.GetSchema(); + + const auto register_projection = [&](const size_t source_index) -> size_t { + const auto it = std::ranges::find(planned.projection_indexes, source_index); + if (it != planned.projection_indexes.end()) { + return static_cast(std::distance(planned.projection_indexes.begin(), it)); + } + planned.projection_indexes.push_back(source_index); + return planned.projection_indexes.size() - 1; + }; + + if (parsed.filter.has_value()) { + ValidateColumnQualifier(parsed, parsed.filter->column); + const size_t source_index = FindColumnIndex(schema, parsed.filter->column.name); + const ColumnType column_type = schema.columns[source_index].type; + + planned.filter = PlannedFilter{ + .column_index = register_projection(source_index), + .column_type = column_type, + .comparison = parsed.filter->comparison, + .literal_text = NormalizeLiteral(parsed.filter->literal, column_type), + }; + } + + planned.group_keys.reserve(parsed.group_by.size()); + for (const ColumnRef& group_column : parsed.group_by) { + ValidateColumnQualifier(parsed, group_column); + const size_t source_index = FindColumnIndex(schema, group_column.name); + planned.group_keys.push_back(PlannedGroupKey{ + .column_index = register_projection(source_index), + .column_type = schema.columns[source_index].type, + .output_name = schema.columns[source_index].name, + }); + } + + bool has_aggregate = false; + for (const auto& item : parsed.select_items) { + if (item.kind == SelectItemKind::Aggregate) { + has_aggregate = true; + break; + } + } + + planned.select_items.reserve(parsed.select_items.size()); + planned.aggregates.reserve(parsed.select_items.size()); + + for (const auto& item : parsed.select_items) { + if (item.kind == SelectItemKind::GroupKey) { + if (parsed.group_by.empty()) { + throw Error::Unsupported("query_planner", "plain column SELECT items require GROUP BY"); + } + + ValidateColumnQualifier(parsed, item.column); + const auto it = std::ranges::find_if(planned.group_keys, [&](const PlannedGroupKey& group_key) { + return SameColumnName(group_key.output_name, item.column.name); + }); + if (it == planned.group_keys.end()) { + throw Error::InvalidArgument("query_planner", + "column '" + item.column.name + "' must appear in GROUP BY"); + } + + planned.select_items.push_back(PlannedSelectItem{ + .kind = SelectItemKind::GroupKey, + .index = static_cast(std::distance(planned.group_keys.begin(), it)), + .output_type = it->column_type, + .output_name = item.output_name, + }); + continue; + } + + const AggSpec& aggregate = item.aggregate; + PlannedAgg planned_aggregate{ + .function = aggregate.function, + .distinct = aggregate.distinct, + .star = aggregate.star, + .column_index = std::nullopt, + .input_type = ColumnType::String, + .output_name = item.output_name, + }; + + if (!aggregate.star) { + ValidateColumnQualifier(parsed, aggregate.column); + const size_t source_index = FindColumnIndex(schema, aggregate.column.name); + planned_aggregate.column_index = register_projection(source_index); + planned_aggregate.input_type = schema.columns[source_index].type; + if (!AggSupportsInputType(*aggregate.function, planned_aggregate.input_type)) { + throw Error::Unsupported("query_planner", + "aggregate " + std::string(aggregate.function->canonical_name) + + " does not support column '" + aggregate.column.name + "'"); + } + } + + const size_t aggregate_index = planned.aggregates.size(); + planned.aggregates.push_back(std::move(planned_aggregate)); + planned.select_items.push_back(PlannedSelectItem{ + .kind = SelectItemKind::Aggregate, + .index = aggregate_index, + .output_type = AggregateOutputType(planned.aggregates.back()), + .output_name = item.output_name, + }); + } + + if (parsed.group_by.empty() && !has_aggregate) { + throw Error::Unsupported("query_planner", "plain SELECT without aggregates is not supported"); + } + + planned.order_by.reserve(parsed.order_by.size()); + for (const auto& order_item : parsed.order_by) { + const auto it = std::ranges::find_if(parsed.select_items, [&](const SelectItemSpec& select_item) { + return SameSelectItem(select_item, order_item.item) || SameOutputName(select_item, order_item.item); + }); + if (it == parsed.select_items.end()) { + throw Error::InvalidArgument("query_planner", "ORDER BY expression must appear in SELECT"); + } + + const size_t result_column_index = static_cast(std::distance(parsed.select_items.begin(), it)); + planned.order_by.push_back(PlannedOrderBy{ + .result_column_index = result_column_index, + .descending = order_item.descending, + .value_type = planned.select_items[result_column_index].output_type, + }); + } + + planned.limit = parsed.limit; + + if (planned.projection_indexes.empty() && !schema.columns.empty()) { + planned.projection_indexes.push_back(0); + } + + return planned; +} diff --git a/src/io/batch_io.cpp b/src/io/batch_io.cpp new file mode 100644 index 0000000..786caf5 --- /dev/null +++ b/src/io/batch_io.cpp @@ -0,0 +1,323 @@ +#include +#include +#include + +#include "io/columnar_batch_io.h" +#include "io/csv.h" +#include "io/csv_batch_io.h" +#include "io/stream.h" +#include "model/metadata.h" +#include "support/error.h" + +static constexpr std::string_view ColumnarMagic = "CLMN"; + +static uint64_t TellWrite(const std::filesystem::path& path, std::ofstream& out) { + const auto pos = out.tellp(); + if (pos == -1) { + throw Error::PathIo("batch_io", path, "tell file"); + } + return pos; +} + +static void FlushWrite(const std::filesystem::path& path, std::ofstream& out) { + out.flush(); + if (!out) { + throw Error::PathIo("batch_io", path, "write file"); + } +} + +static uint64_t AddChecked(const uint64_t current, const uint64_t add) { + if (add > std::numeric_limits::max() - current) { + throw Error::Overflow("batch_io", "batch byte size overflow"); + } + return current + add; +} + +static uint64_t EstimateRowBytes(const std::vector>& columns, + const std::vector& row) { + uint64_t bytes = 0; + for (size_t i = 0; i < columns.size(); ++i) { + bytes = AddChecked(bytes, columns[i]->EstimateSizeFromString(row[i])); + } + return bytes; +} + +static void ValidateSizing(const BatchSizing& sizing) { + if (sizing.max_rows && *sizing.max_rows == 0) { + throw Error::InvalidArgument("batch_io", "max rows must be > 0"); + } + if (sizing.max_values && *sizing.max_values == 0) { + throw Error::InvalidArgument("batch_io", "max values must be > 0"); + } + if (sizing.max_bytes && *sizing.max_bytes == 0) { + throw Error::InvalidArgument("batch_io", "max bytes must be > 0"); + } +} + +static void ReserveForSizing(const Batch& batch, const BatchSizing& sizing, const size_t column_count) { + if (sizing.max_rows) { + batch.Reserve(*sizing.max_rows); + return; + } + if (sizing.max_values && column_count > 0) { + batch.Reserve(std::max(1, *sizing.max_values / column_count)); + } +} + +bool BatchSizing::WouldExceed(const size_t next_rows, const size_t column_count, const uint64_t next_bytes) const { + if (max_rows && next_rows > *max_rows) { + return true; + } + if (max_values && column_count > 0) { + if (next_rows > *max_values / column_count) { + return true; + } + } + if (max_bytes && next_bytes > *max_bytes) { + return true; + } + return false; +} + +CsvBatchReader::CsvBatchReader(const std::filesystem::path& path, Schema schema, BatchSizing sizing) + : csv_reader_(path), schema_(std::move(schema)), sizing_(std::move(sizing)) { + if (schema_.columns.empty()) { + throw Error::InvalidArgument("batch_io", "schema has no columns"); + } + ValidateSizing(sizing_); +} + +std::optional CsvBatchReader::ReadNext() { + if (reached_eof_ && !pending_row_) { + return std::nullopt; + } + + Batch batch(schema_); + + const size_t column_count = schema_.columns.size(); + ReserveForSizing(batch, sizing_, column_count); + const auto& columns = batch.GetColumns(); + + std::vector row; + row.reserve(column_count); + + size_t rows = 0; + uint64_t bytes = 0; + + while (true) { + if (pending_row_) { + row = std::move(*pending_row_); + pending_row_.reset(); + } else { + if (!csv_reader_.ReadRow(row)) { + reached_eof_ = true; + break; + } + } + + if (row.size() != column_count) { + throw Error::Mismatch("batch_io", "data csv column count mismatch"); + } + + const size_t next_rows = rows + 1; + uint64_t next_bytes = bytes; + + if (sizing_.max_bytes) { + next_bytes = AddChecked(bytes, EstimateRowBytes(columns, row)); + } + + if (rows > 0 && sizing_.WouldExceed(next_rows, column_count, next_bytes)) { + pending_row_ = std::move(row); + break; + } + + for (size_t col = 0; col < column_count; ++col) { + columns[col]->AppendFromString(row[col]); + } + + rows = next_rows; + bytes = next_bytes; + } + + if (rows == 0 && reached_eof_ && !pending_row_) { + return std::nullopt; + } + + return batch; +} + +CsvBatchWriter::CsvBatchWriter(const std::filesystem::path& path, Schema schema) + : csv_writer_(path), schema_(std::move(schema)) { + if (schema_.columns.empty()) { + throw Error::InvalidArgument("batch_io", "schema has no columns"); + } +} + +void CsvBatchWriter::Write(const Batch& batch) { + batch.Validate(); + if (batch.GetSchema() != schema_) { + throw Error::Mismatch("batch_io", "batch schema mismatch"); + } + + const size_t column_count = schema_.columns.size(); + const size_t row_count = batch.RowsCount(); + + std::vector row(column_count); + + for (size_t row_index = 0; row_index < row_count; ++row_index) { + for (size_t col = 0; col < column_count; ++col) { + row[col] = batch.ColumnAt(col).ValueAsString(row_index); + } + csv_writer_.WriteRow(row); + } +} + +void CsvBatchWriter::Flush() { csv_writer_.Flush(); } + +void AppendBatchRows(const Batch& batch, std::vector>& rows) { + const size_t row_count = batch.RowsCount(); + const size_t column_count = batch.ColumnsCount(); + + for (size_t row = 0; row < row_count; ++row) { + std::vector values; + values.reserve(column_count); + + for (size_t col = 0; col < column_count; ++col) { + values.push_back(batch.ColumnAt(col).ValueAsString(row)); + } + + rows.push_back(std::move(values)); + } +} + +void WriteBatchCsv(const std::filesystem::path& path, const Batch& batch) { + CsvBatchWriter writer(path, batch.GetSchema()); + writer.Write(batch); + writer.Flush(); +} + +ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::path& path) { + const auto file_metadata = TryGetFileMetadata(path); + + if (!file_metadata || !file_metadata->is_regular) { + throw Error::NotFound("batch_io", "columnar file not found", path.string()); + } + + const uint64_t file_size = file_metadata->size; + constexpr uint64_t FooterSize = sizeof(uint64_t) + ColumnarMagic.size(); + + if (file_size < FooterSize) { + throw Error::InvalidData("batch_io", "columnar file is too small", path.string()); + } + + std::ifstream in = OpenInputFile(path); + SeekInputFile(in, path, file_size - FooterSize); + + const uint64_t metadata_size = ReadStream(in); + std::string magic_read(ColumnarMagic.size(), '\0'); + ReadBytes(in, magic_read.data(), magic_read.size()); + + if (magic_read != ColumnarMagic) { + throw Error::InvalidData("batch_io", "invalid columnar magic", path.string()); + } + + if (metadata_size > file_size - FooterSize) { + throw Error::InvalidData("batch_io", "metadata size exceeds file size", path.string()); + } + + SeekInputFile(in, path, file_size - FooterSize - metadata_size); + + return ReadMetadata(in); +} + +ColumnarBatchReader::ColumnarBatchReader(const std::filesystem::path& path) + : path_(path), in_(OpenInputFile(path)), metadata_(ReadFileMetadata(path)) { + if (metadata_.schema.columns.empty()) { + throw Error::InvalidData("batch_io", "columnar schema is empty", path.string()); + } +} + +std::optional ColumnarBatchReader::ReadNext() { + if (next_group_ >= metadata_.row_groups.size()) { + return std::nullopt; + } + + const auto& [row_count, row_group_columns] = metadata_.row_groups[next_group_++]; + Batch batch(metadata_.schema, row_count); + + const auto& columns = batch.GetColumns(); + + if (row_group_columns.size() != columns.size()) { + throw Error::Mismatch("batch_io", "row group column count mismatch", path_.string()); + } + + for (size_t col = 0; col < columns.size(); ++col) { + const auto& [offset, size] = row_group_columns[col]; + SeekInputFile(in_, path_, offset); + columns[col]->ReadFrom(in_, row_count, size); + } + + return batch; +} + +ColumnarBatchWriter::ColumnarBatchWriter(const std::filesystem::path& path, Schema schema) + : path_(path), out_(OpenOutputFile(path)) { + if (schema.columns.empty()) { + throw Error::InvalidArgument("batch_io", "schema has no columns", path.string()); + } + metadata_.schema = std::move(schema); +} + +void ColumnarBatchWriter::Write(const Batch& batch) { + if (finalized_) { + throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); + } + + batch.Validate(); + if (batch.GetSchema() != metadata_.schema) { + throw Error::Mismatch("batch_io", "batch schema mismatch", path_.string()); + } + + const size_t row_count = batch.RowsCount(); + + if (row_count == 0) { + return; + } + if (row_count > std::numeric_limits::max()) { + throw Error::Overflow("batch_io", "row group exceeds supported size", path_.string()); + } + + RowGroupMetadata group; + group.row_count = row_count; + group.columns.reserve(batch.ColumnsCount()); + + for (const auto& column : batch.GetColumns()) { + const uint64_t offset = TellWrite(path_, out_); + column->WriteTo(out_); + const uint64_t end = TellWrite(path_, out_); + group.columns.push_back(ColumnChunkMetadata{offset, end - offset}); + } + + metadata_.row_groups.push_back(std::move(group)); +} + +void ColumnarBatchWriter::Flush() { FlushWrite(path_, out_); } + +void ColumnarBatchWriter::Finalize() && { FinalizeImpl(); } + +void ColumnarBatchWriter::FinalizeImpl() { + if (finalized_) { + throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); + } + + const uint64_t metadata_start = TellWrite(path_, out_); + WriteMetadata(out_, metadata_); + const uint64_t metadata_end = TellWrite(path_, out_); + const uint64_t metadata_size = metadata_end - metadata_start; + + WriteStream(out_, metadata_size); + WriteBytes(out_, ColumnarMagic); + + FlushWrite(path_, out_); + finalized_ = true; +} diff --git a/src/io/csv.cpp b/src/io/csv.cpp new file mode 100644 index 0000000..0291c4a --- /dev/null +++ b/src/io/csv.cpp @@ -0,0 +1,196 @@ +#include "io/csv.h" + +#include + +#include "io/fileio.h" +#include "support/error.h" + +static bool NeedsCsvQuotes(const std::string_view value) { + return value.find_first_of(",\"\n\r") != std::string_view::npos; +} + +CsvReader::CsvReader(std::istream& in) : in_(&in) {} + +CsvReader::CsvReader(const std::filesystem::path& path) : owned_in_(OpenInputFile(path)), in_(&owned_in_) {} + +CsvReader::CsvReader(CsvReader&& other) noexcept : owned_in_(std::move(other.owned_in_)) { + RebindAfterMove(other.in_ == &other.owned_in_, other.in_); + other.in_ = nullptr; +} + +CsvReader& CsvReader::operator=(CsvReader&& other) noexcept { + if (this != &other) { + owned_in_ = std::move(other.owned_in_); + RebindAfterMove(other.in_ == &other.owned_in_, other.in_); + other.in_ = nullptr; + } + return *this; +} + +void CsvReader::RebindAfterMove(const bool uses_owned_stream, std::istream* source_stream) noexcept { + if (uses_owned_stream) { + in_ = &owned_in_; + return; + } + in_ = source_stream; +} + +bool CsvReader::ReadRow(std::vector& row) const { + row.clear(); + row.emplace_back(); + + auto* buffer = in_->rdbuf(); + using Traits = std::streambuf::traits_type; + /* До этого чтение происходило через istream::get() и peek() + * Traits требуется только как вспомогательный тип для: + * - eof() + * - to_chat_type() + * - eq_int_type() + */ + + bool in_quotes = false; + bool saw_data = false; + + while (true) { + const Traits::int_type next = buffer->sbumpc(); + + if (Traits::eq_int_type(next, Traits::eof())) { + if (!saw_data) { + return false; + } + if (in_quotes) { + throw Error::InvalidData("csv", "unexpected EOF inside quoted field"); + } + return true; + } + + saw_data = true; + const char ch = Traits::to_char_type(next); + + if (in_quotes) { + if (ch == '"') { + const Traits::int_type peek = buffer->sgetc(); + if (Traits::to_char_type(peek) == '"') { + buffer->sbumpc(); + row.back().push_back('"'); + } else { + in_quotes = false; + } + } else { + row.back().push_back(ch); + } + continue; + } + + if (ch == '"') { + if (row.back().empty()) { + in_quotes = true; + } else { + row.back().push_back('"'); + } + continue; + } + + if (ch == ',') { + row.emplace_back(); + continue; + } + + if (ch == '\n') { + return true; + } + + if (ch == '\r') { + if (Traits::to_char_type(buffer->sgetc()) == '\n') { + buffer->sbumpc(); + } + return true; + } + + row.back().push_back(ch); + } +} + +CsvWriter::CsvWriter(std::ostream& out) : out_(&out) {} + +CsvWriter::CsvWriter(const std::filesystem::path& path) : owned_out_(OpenOutputFile(path)), out_(&owned_out_) {} + +CsvWriter::CsvWriter(CsvWriter&& other) noexcept : owned_out_(std::move(other.owned_out_)) { + RebindAfterMove(other.out_ == &other.owned_out_, other.out_); + other.out_ = nullptr; +} + +CsvWriter& CsvWriter::operator=(CsvWriter&& other) noexcept { + if (this != &other) { + owned_out_ = std::move(other.owned_out_); + RebindAfterMove(other.out_ == &other.owned_out_, other.out_); + other.out_ = nullptr; + } + return *this; +} + +void CsvWriter::RebindAfterMove(const bool uses_owned_stream, std::ostream* source_stream) noexcept { + if (uses_owned_stream) { + out_ = &owned_out_; + return; + } + out_ = source_stream; +} + +void CsvWriter::WriteRow(const std::vector& row) const { + for (size_t i = 0; i < row.size(); ++i) { + if (i > 0) { + *out_ << ','; + } + + const std::string& value = row[i]; + if (!NeedsCsvQuotes(value)) { + *out_ << value; + continue; + } + + *out_ << '"'; + for (const char ch : value) { + if (ch == '"') { + *out_ << "\"\""; + } else { + *out_ << ch; + } + } + *out_ << '"'; + } + *out_ << '\n'; + if (!*out_) { + throw Error::Io("csv", "failed to write csv row"); + } +} + +void CsvWriter::Flush() const { + out_->flush(); + if (!*out_) { + throw Error::Io("csv", "failed to write csv"); + } +} + +std::vector> ReadRows(const std::filesystem::path& path) { + const CsvReader reader(path); + + std::vector> rows; + std::vector row; + + while (reader.ReadRow(row)) { + rows.push_back(row); + } + + return rows; +} + +void WriteRows(const std::filesystem::path& path, const std::vector>& rows) { + const CsvWriter writer(path); + + for (const auto& row : rows) { + writer.WriteRow(row); + } + + writer.Flush(); +} diff --git a/src/io/fileio.cpp b/src/io/fileio.cpp new file mode 100644 index 0000000..b332524 --- /dev/null +++ b/src/io/fileio.cpp @@ -0,0 +1,133 @@ +#include "io/fileio.h" + +#include + +#include "support/error.h" + +std::optional TryGetFileMetadata(const std::filesystem::path& path) { + std::error_code ec; + if (!std::filesystem::exists(path, ec)) { + if (ec) { + throw Error::PathIo("fileio", path, "check existence"); + } + return std::nullopt; + } + + FileMetadata metadata; + const auto status = std::filesystem::status(path, ec); + if (ec) { + throw Error::PathIo("fileio", path, "read status"); + } + + metadata.is_regular = std::filesystem::is_regular_file(status); + metadata.is_directory = std::filesystem::is_directory(status); + if (metadata.is_regular) { + metadata.size = std::filesystem::file_size(path, ec); + if (ec) { + throw Error::PathIo("fileio", path, "read file size"); + } + } + + metadata.last_write_time = std::filesystem::last_write_time(path, ec); + if (ec) { + throw Error::PathIo("fileio", path, "read last write time"); + } + + return metadata; +} + +bool FileExists(const std::filesystem::path& path) { + std::error_code ec; + const bool exists = std::filesystem::exists(path, ec); + if (ec) { + throw Error::PathIo("fileio", path, "check existence"); + } + return exists; +} + +void EnsureParentDirectory(const std::filesystem::path& path) { + const auto parent = path.parent_path(); + if (parent.empty()) { + return; + } + + std::error_code ec; + std::filesystem::create_directories(parent, ec); + if (ec) { + throw Error::PathIo("fileio", path, "create parent directories"); + } +} + +std::ifstream OpenInputFile(const std::filesystem::path& path) { + std::ifstream file(path, std::ios::binary); + if (!file.is_open()) { + throw Error::PathIo("fileio", path, "open for read"); + } + return file; +} + +void SeekInputFile(std::istream& in, const std::filesystem::path& path, const uint64_t offset) { + in.seekg(static_cast(offset)); + if (!in) { + throw Error::PathIo("fileio", path, "seek for read"); + } +} + +std::ofstream OpenOutputFile(const std::filesystem::path& path, const FileOpenMode mode) { + std::ofstream file(path, std::ios::binary | (mode == FileOpenMode::Append ? std::ios::app : std::ios::trunc)); + if (!file.is_open()) { + throw Error::PathIo("fileio", path, mode == FileOpenMode::Append ? "open for append" : "open for write"); + } + return file; +} + +std::string ReadTextFile(const std::filesystem::path& path) { + std::ifstream file = OpenInputFile(path); + return std::string(std::istreambuf_iterator(file), std::istreambuf_iterator()); +} + +std::vector ReadFileBytes(const std::filesystem::path& path) { + std::ifstream file = OpenInputFile(path); + file.seekg(0, std::ios::end); + if (!file) { + throw Error::PathIo("fileio", path, "read file size"); + } + + const auto end_pos = file.tellg(); + if (end_pos < 0) { + throw Error::PathIo("fileio", path, "read file size"); + } + + std::vector data(end_pos); + if (end_pos == 0) { + return data; + } + + file.seekg(0, std::ios::beg); + file.read(reinterpret_cast(data.data()), data.size()); + + if (!file) { + throw Error::PathIo("fileio", path, "read file"); + } + + return data; +} + +void WriteFileBytes(const std::filesystem::path& path, const std::span bytes) { + std::ofstream file = OpenOutputFile(path); + if (!bytes.empty()) { + file.write(reinterpret_cast(bytes.data()), bytes.size()); + if (!file) { + throw Error::PathIo("fileio", path, "write file"); + } + } + + file.flush(); + if (!file) { + throw Error::PathIo("fileio", path, "write file"); + } +} + +void WriteFileBytes(const std::filesystem::path& path, const std::vector& bytes) { + WriteFileBytes(path, std::span(bytes.data(), bytes.size())); +} diff --git a/src/io/stream.cpp b/src/io/stream.cpp new file mode 100644 index 0000000..41d85f2 --- /dev/null +++ b/src/io/stream.cpp @@ -0,0 +1,23 @@ +#include "io/stream.h" + +#include "support/error.h" + +void ReadBytes(std::istream& in, char* dst, const size_t size) { + if (size == 0) { + return; + } + in.read(dst, size); + if (in.gcount() != static_cast(size) || !in) { + throw Error::Io("stream", "failed to read bytes"); + } +} + +void WriteBytes(std::ostream& out, const std::string_view bytes) { + if (bytes.empty()) { + return; + } + out.write(bytes.data(), bytes.size()); + if (!out) { + throw Error::Io("stream", "failed to write bytes"); + } +} diff --git a/src/model/batch.cpp b/src/model/batch.cpp new file mode 100644 index 0000000..09fe5ca --- /dev/null +++ b/src/model/batch.cpp @@ -0,0 +1,86 @@ +#include "model/batch.h" + +#include + +#include "support/error.h" + +static std::vector> CloneColumns(const std::vector>& columns) { + std::vector> clones; + clones.reserve(columns.size()); + for (const auto& column : columns) { + clones.push_back(column->Clone()); + } + return clones; +} + +Batch::Batch(Schema schema) : schema_(std::move(schema)) { + columns_.reserve(schema_.columns.size()); + for (const auto& [name, type] : schema_.columns) { + columns_.push_back(CreateColumn(type)); + } +} + +Batch::Batch(Schema schema, const size_t reserve_rows) : Batch(std::move(schema)) { Reserve(reserve_rows); } + +Batch::Batch(const Batch& other) : schema_(other.schema_), columns_(CloneColumns(other.columns_)) {} + +Batch& Batch::operator=(const Batch& other) { + if (this == &other) { + return *this; + } + + auto columns = CloneColumns(other.columns_); + schema_ = other.schema_; + columns_ = std::move(columns); + + return *this; +} + +size_t Batch::ColumnsCount() const { return columns_.size(); } + +size_t Batch::RowsCount() const { + if (columns_.empty()) { + return 0; + } + return columns_.front()->Size(); +} + +void Batch::Reserve(const size_t n) const { + for (auto& column : columns_) { + column->Reserve(n); + } +} + +const Column& Batch::ColumnAt(const size_t i) const { + if (i >= columns_.size()) { + throw Error::OutOfRange("batch", "column index out of range"); + } + return *columns_[i]; +} + +Column& Batch::ColumnAt(const size_t i) { + if (i >= columns_.size()) { + throw Error::OutOfRange("batch", "column index out of range"); + } + return *columns_[i]; +} + +void Batch::Validate() const { + if (schema_.columns.size() != columns_.size()) { + throw Error::Mismatch("batch", "column count mismatch"); + } + + if (columns_.empty()) { + return; + } + + const size_t rows = columns_.front()->Size(); + for (size_t i = 0; i < columns_.size(); ++i) { + if (columns_[i]->Type() != schema_.columns[i].type) { + throw Error::Mismatch("batch", "column type mismatch"); + } + if (columns_[i]->Size() != rows) { + throw Error::Mismatch("batch", "column row count mismatch"); + } + } +} diff --git a/src/model/column.cpp b/src/model/column.cpp new file mode 100644 index 0000000..ab11742 --- /dev/null +++ b/src/model/column.cpp @@ -0,0 +1,46 @@ +#include "model/column.h" + +#include + +#include "model/column_boolean.h" +#include "model/column_character.h" +#include "model/column_date.h" +#include "model/column_int128.h" +#include "model/column_int16.h" +#include "model/column_int32.h" +#include "model/column_int64.h" +#include "model/column_string.h" +#include "model/column_timestamp.h" +#include "support/error.h" + +std::unique_ptr CreateColumn(const ColumnType type) { + switch (type) { + case ColumnType::Int64: + return std::make_unique(); + case ColumnType::String: + return std::make_unique(); + case ColumnType::Boolean: + return std::make_unique(); + case ColumnType::Int16: + return std::make_unique(); + case ColumnType::Int32: + return std::make_unique(); + case ColumnType::Int128: + return std::make_unique(); + case ColumnType::Date: + return std::make_unique(); + case ColumnType::Timestamp: + return std::make_unique(); + case ColumnType::Character: + return std::make_unique(); + } + throw Error::Unsupported("column", "unsupported column type"); +} + +Column::Column(const ColumnType type) : type_(type) {} + +void Column::CheckRowIndex(const char* module, const size_t row, const size_t size) { + if (row >= size) { + throw Error::OutOfRange(module, "row index out of range"); + } +} diff --git a/src/model/column_boolean.cpp b/src/model/column_boolean.cpp new file mode 100644 index 0000000..c58ee59 --- /dev/null +++ b/src/model/column_boolean.cpp @@ -0,0 +1,7 @@ +#include "model/column_boolean.h" + +#include "support/parsing.h" + +void BooleanColumn::AppendFromString(const std::string& value) { AppendValue(ParseBoolean(value) ? 1 : 0); } + +std::string BooleanColumn::ValueAsString(const size_t row) const { return BooleanToString(ValueAt(row) != 0); } diff --git a/src/model/column_character.cpp b/src/model/column_character.cpp new file mode 100644 index 0000000..e7312e4 --- /dev/null +++ b/src/model/column_character.cpp @@ -0,0 +1,7 @@ +#include "model/column_character.h" + +#include "support/parsing.h" + +void CharacterColumn::AppendFromString(const std::string& value) { AppendValue(ParseCharacter(value)); } + +std::string CharacterColumn::ValueAsString(const size_t row) const { return std::string(1, ValueAt(row)); } diff --git a/src/model/column_date.cpp b/src/model/column_date.cpp new file mode 100644 index 0000000..e9cc250 --- /dev/null +++ b/src/model/column_date.cpp @@ -0,0 +1,7 @@ +#include "model/column_date.h" + +#include "support/parsing.h" + +void DateColumn::AppendFromString(const std::string& value) { AppendValue(ParseDate(value)); } + +std::string DateColumn::ValueAsString(const size_t row) const { return DateToString(ValueAt(row)); } diff --git a/src/model/column_int128.cpp b/src/model/column_int128.cpp new file mode 100644 index 0000000..2102e6f --- /dev/null +++ b/src/model/column_int128.cpp @@ -0,0 +1,7 @@ +#include "model/column_int128.h" + +#include "support/parsing.h" + +void Int128Column::AppendFromString(const std::string& value) { AppendValue(ParseInt128(value)); } + +std::string Int128Column::ValueAsString(const size_t row) const { return Int128ToString(ValueAt(row)); } diff --git a/src/model/column_int16.cpp b/src/model/column_int16.cpp new file mode 100644 index 0000000..95c6cd5 --- /dev/null +++ b/src/model/column_int16.cpp @@ -0,0 +1,7 @@ +#include "model/column_int16.h" + +#include "support/parsing.h" + +void Int16Column::AppendFromString(const std::string& value) { AppendValue(ParseInt16(value)); } + +std::string Int16Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_int32.cpp b/src/model/column_int32.cpp new file mode 100644 index 0000000..d0d3639 --- /dev/null +++ b/src/model/column_int32.cpp @@ -0,0 +1,7 @@ +#include "model/column_int32.h" + +#include "support/parsing.h" + +void Int32Column::AppendFromString(const std::string& value) { AppendValue(ParseInt32(value)); } + +std::string Int32Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_int64.cpp b/src/model/column_int64.cpp new file mode 100644 index 0000000..df9be61 --- /dev/null +++ b/src/model/column_int64.cpp @@ -0,0 +1,7 @@ +#include "model/column_int64.h" + +#include "support/parsing.h" + +void Int64Column::AppendFromString(const std::string& value) { AppendValue(ParseInt64(value)); } + +std::string Int64Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_string.cpp b/src/model/column_string.cpp new file mode 100644 index 0000000..1a94bc2 --- /dev/null +++ b/src/model/column_string.cpp @@ -0,0 +1,5 @@ +#include "model/column_string.h" + +void StringColumn::AppendFromString(const std::string& value) { AppendValue(value); } + +std::string StringColumn::ValueAsString(const size_t row) const { return ValueAt(row); } diff --git a/src/model/column_timestamp.cpp b/src/model/column_timestamp.cpp new file mode 100644 index 0000000..c69a98a --- /dev/null +++ b/src/model/column_timestamp.cpp @@ -0,0 +1,7 @@ +#include "model/column_timestamp.h" + +#include "support/parsing.h" + +void TimestampColumn::AppendFromString(const std::string& value) { AppendValue(ParseTimestamp(value)); } + +std::string TimestampColumn::ValueAsString(const size_t row) const { return TimestampToString(ValueAt(row)); } diff --git a/src/model/metadata.cpp b/src/model/metadata.cpp new file mode 100644 index 0000000..17e1209 --- /dev/null +++ b/src/model/metadata.cpp @@ -0,0 +1,100 @@ +#include "model/metadata.h" + +#include +#include + +#include "support/error.h" +#include "io/stream.h" + +static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { + switch (static_cast(type_byte)) { + case ColumnType::Int64: + case ColumnType::String: + case ColumnType::Boolean: + case ColumnType::Int16: + case ColumnType::Int32: + case ColumnType::Int128: + case ColumnType::Date: + case ColumnType::Timestamp: + case ColumnType::Character: + return static_cast(type_byte); + } + + throw Error::InvalidData("columnar", "unknown column type in metadata"); +} + +ColumnarMetadata ReadMetadata(std::istream& in) { + ColumnarMetadata metadata; + + const uint32_t column_count = ReadStream(in); + metadata.schema.columns.reserve(column_count); + + for (uint32_t i = 0; i < column_count; ++i) { + const uint32_t name_size = ReadStream(in); + std::string name(name_size, '\0'); + + ReadBytes(in, name.data(), name.size()); + const uint8_t type_byte = ReadStream(in); + const ColumnType type = ColumnTypeFromByte(type_byte); + + metadata.schema.columns.push_back(ColumnSchema{std::move(name), type}); + } + + const uint32_t row_group_count = ReadStream(in); + metadata.row_groups.reserve(row_group_count); + + for (uint32_t i = 0; i < row_group_count; ++i) { + RowGroupMetadata group; + group.row_count = ReadStream(in); + + const uint32_t columns_in_group = ReadStream(in); + if (columns_in_group != column_count) { + throw Error::Mismatch("columnar", "row group column count mismatch"); + } + + group.columns.reserve(columns_in_group); + + for (uint32_t col = 0; col < columns_in_group; ++col) { + ColumnChunkMetadata chunk; + chunk.offset = ReadStream(in); + chunk.size = ReadStream(in); + group.columns.push_back(chunk); + } + + metadata.row_groups.push_back(std::move(group)); + } + + return metadata; +} + +void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { + if (metadata.schema.columns.size() > std::numeric_limits::max()) { + throw Error::Overflow("columnar", "too many columns for metadata"); + } + + WriteStream(out, metadata.schema.columns.size()); + + for (const auto& [name, type] : metadata.schema.columns) { + if (name.size() > std::numeric_limits::max()) { + throw Error::Overflow("columnar", "column name too long"); + } + WriteStream(out, name.size()); + WriteBytes(out, name); + WriteStream(out, static_cast(type)); + } + + if (metadata.row_groups.size() > std::numeric_limits::max()) { + throw Error::Overflow("columnar", "too many row groups for metadata"); + } + + WriteStream(out, metadata.row_groups.size()); + + for (const auto& [row_count, columns] : metadata.row_groups) { + WriteStream(out, row_count); + WriteStream(out, columns.size()); + for (const auto& [offset, size] : columns) { + WriteStream(out, offset); + WriteStream(out, size); + } + } +} diff --git a/src/model/schema.cpp b/src/model/schema.cpp new file mode 100644 index 0000000..c5344f1 --- /dev/null +++ b/src/model/schema.cpp @@ -0,0 +1,141 @@ +#include "model/schema.h" + +#include + +#include "io/csv.h" +#include "support/error.h" +#include "io/fileio.h" +#include "support/parsing.h" + +static bool CanParseAs(const ColumnType type, const std::string& value) { + try { + switch (type) { + case ColumnType::Boolean: + static_cast(ParseBoolean(value)); + return true; + case ColumnType::Int16: + static_cast(ParseInt16(value)); + return true; + case ColumnType::Int32: + static_cast(ParseInt32(value)); + return true; + case ColumnType::Int64: + static_cast(ParseInt64(value)); + return true; + case ColumnType::Int128: + static_cast(ParseInt128(value)); + return true; + case ColumnType::Date: + static_cast(ParseDate(value)); + return true; + case ColumnType::Timestamp: + static_cast(ParseTimestamp(value)); + return true; + case ColumnType::Character: + static_cast(ParseCharacter(value)); + return true; + case ColumnType::String: + return true; + } + } catch (const Error&) { + return false; + } + + throw Error::Unsupported("schema", "unknown column type"); +} + +static ColumnType InferColumnType(const std::vector& values) { + static constexpr std::array InferenceOrder = { + ColumnType::Boolean, ColumnType::Int16, ColumnType::Int32, ColumnType::Int64, ColumnType::Int128, + ColumnType::Date, ColumnType::Timestamp, ColumnType::Character, ColumnType::String, + }; + + for (const ColumnType type : InferenceOrder) { + bool matches = true; + for (const auto& value : values) { + if (!CanParseAs(type, value)) { + matches = false; + break; + } + } + if (matches) { + return type; + } + } + + return ColumnType::String; +} + +Schema ReadSchemaCsv(const std::filesystem::path& path) { + const CsvReader reader(path); + + Schema schema; + std::vector row; + + while (reader.ReadRow(row)) { + if (row.size() == 1 && row[0].empty()) { + continue; + } + while (row.size() > 2 && row.back().empty()) { + row.pop_back(); + } + if (row.size() != 2) { + throw Error::InvalidData("schema", "schema csv must have 2 columns", path.string()); + } + schema.columns.push_back(ColumnSchema{ + row[0], + ParseColumnType(row[1]), + }); + } + + if (schema.columns.empty()) { + throw Error::InvalidData("schema", "schema csv is empty", path.string()); + } + + return schema; +} + +Schema InferSchemaCsv(const std::filesystem::path& path) { + const CsvReader reader(path); + + std::vector> values_by_column; + std::vector row; + + while (reader.ReadRow(row)) { + if (values_by_column.empty()) { + values_by_column.resize(row.size()); + } else if (row.size() != values_by_column.size()) { + throw Error::InvalidData("schema", "csv rows have inconsistent column count", path.string()); + } + + for (size_t i = 0; i < row.size(); ++i) { + values_by_column[i].push_back(row[i]); + } + } + + if (values_by_column.empty()) { + throw Error::InvalidData("schema", "csv is empty", path.string()); + } + + Schema schema; + schema.columns.reserve(values_by_column.size()); + + for (size_t i = 0; i < values_by_column.size(); ++i) { + schema.columns.push_back(ColumnSchema{ + "column_" + std::to_string(i + 1), + InferColumnType(values_by_column[i]), + }); + } + + return schema; +} + +void WriteSchemaCsv(const std::filesystem::path& path, const Schema& schema) { + const CsvWriter writer(path); + + for (const auto& [name, type] : schema.columns) { + writer.WriteRow({name, ColumnTypeToString(type)}); + } + + writer.Flush(); +} diff --git a/src/sql_parser/tokenizer.cpp b/src/sql_parser/tokenizer.cpp new file mode 100644 index 0000000..c8980b1 --- /dev/null +++ b/src/sql_parser/tokenizer.cpp @@ -0,0 +1,174 @@ +#include "sql_parser/tokenizer.h" + +#include +#include +#include +#include +#include + +static bool IsIdentifierStart(const char ch) { + const auto uch = static_cast(ch); + return std::isalpha(uch) != 0 || ch == '_'; +} + +static bool IsIdentifierPart(const char ch) { + const auto uch = static_cast(ch); + return std::isalnum(uch) != 0 || ch == '_'; +} + +Tokenizer::Tokenizer(std::string data) : query_(std::move(data)) {} + +Expected Tokenizer::GetNext() { + while (pos_ < query_.size() && std::isspace(static_cast(query_[pos_])) != 0) { + ++pos_; + } + + if (pos_ >= query_.size()) { + return MakeToken(Tokens::EndOfInput, {}, query_.size()); + } + + const char ch = query_[pos_]; + if (IsIdentifierStart(ch)) { + const size_t start = pos_; + ++pos_; + + while (pos_ < query_.size() && IsIdentifierPart(query_[pos_])) { + ++pos_; + } + + const std::string_view text = std::string_view(query_).substr(start, pos_ - start); + + return MakeToken(ResolveIdentifierType(text), std::string(text), start); + } + + if (std::isdigit(static_cast(ch)) != 0) { + const size_t start = pos_; + ++pos_; + + while (pos_ < query_.size() && std::isdigit(static_cast(query_[pos_])) != 0) { + ++pos_; + } + + if (pos_ + 1 < query_.size() && query_[pos_] == '.' && + std::isdigit(static_cast(query_[pos_ + 1])) != 0) { + ++pos_; + while (pos_ < query_.size() && std::isdigit(static_cast(query_[pos_])) != 0) { + ++pos_; + } + } + + const std::string_view text = std::string_view(query_).substr(start, pos_ - start); + + return MakeToken(Tokens::NumericLiteral, std::string(text), start); + } + + if (ch == '\'') { + const size_t start = pos_; + ++pos_; + + bool terminated = false; + + while (pos_ < query_.size()) { + if (query_[pos_] != '\'') { + ++pos_; + continue; + } + + if (pos_ + 1 < query_.size() && query_[pos_ + 1] == '\'') { + pos_ += 2; + continue; + } + + ++pos_; + terminated = true; + + break; + } + + if (!terminated) { + return MakeUnterminatedString(start); + } + + const std::string_view text = std::string_view(query_).substr(start, pos_ - start); + + return MakeToken(Tokens::StringLiteral, std::string(text), start); + } + + const size_t start = pos_; + switch (ch) { + case '+': + ++pos_; + return MakeToken(Tokens::Plus, "+", start); + case '-': + ++pos_; + return MakeToken(Tokens::Minus, "-", start); + case '*': + ++pos_; + return MakeToken(Tokens::Asterisk, "*", start); + case '(': + ++pos_; + return MakeToken(Tokens::OpenBracket, "(", start); + case ')': + ++pos_; + return MakeToken(Tokens::CloseBracket, ")", start); + case ',': + ++pos_; + return MakeToken(Tokens::Comma, ",", start); + case '.': + ++pos_; + return MakeToken(Tokens::Dot, ".", start); + case ';': + ++pos_; + return MakeToken(Tokens::Semicolon, ";", start); + case '=': + ++pos_; + return MakeToken(Tokens::Equal, "=", start); + case '<': + ++pos_; + if (pos_ < query_.size() && query_[pos_] == '=') { + ++pos_; + return MakeToken(Tokens::LessOrEqual, "<=", start); + } + if (pos_ < query_.size() && query_[pos_] == '>') { + ++pos_; + return MakeToken(Tokens::NotEqual, "<>", start); + } + return MakeToken(Tokens::Less, "<", start); + case '>': + ++pos_; + if (pos_ < query_.size() && query_[pos_] == '=') { + ++pos_; + return MakeToken(Tokens::GreaterOrEqual, ">=", start); + } + return MakeToken(Tokens::Greater, ">", start); + case '!': + if (pos_ + 1 < query_.size() && query_[pos_ + 1] == '=') { + pos_ += 2; + return MakeToken(Tokens::NotEqual, "!=", start); + } + return MakeUnexpectedCharacter(ch, start); + default: + return MakeUnexpectedCharacter(ch, start); + } +} + +Expected> TokenizeSql(const std::string_view input) { + Tokenizer tokenizer{std::string(input)}; + + std::vector tokens; + tokens.reserve(input.size() / 2 + 1); + + while (true) { + auto token = tokenizer.GetNext(); + if (!token.has_value()) { + return tl::unexpected(token.error()); + } + + tokens.push_back(token.value()); + if (token.value()->GetType() == Tokens::EndOfInput) { + break; + } + } + + return tokens; +} diff --git a/src/sql_parser/tokenizer_factory.cpp b/src/sql_parser/tokenizer_factory.cpp new file mode 100644 index 0000000..dd99921 --- /dev/null +++ b/src/sql_parser/tokenizer_factory.cpp @@ -0,0 +1,130 @@ +#include +#include +#include +#include +#include + +#include "support/ascii.h" +#include "sql_parser/tokenizer.h" + +static const std::unordered_map kKeywords = { + {"AS", Tokens::As}, {"AND", Tokens::And}, {"AVG", Tokens::Avg}, {"BY", Tokens::By}, + {"COUNT", Tokens::Count}, {"CREATE", Tokens::Create}, {"DISTINCT", Tokens::Distinct}, {"FROM", Tokens::From}, + {"GROUP", Tokens::Group}, {"LENGTH", Tokens::Length}, {"LIMIT", Tokens::Limit}, {"MAX", Tokens::Max}, + {"MIN", Tokens::Min}, {"ORDER", Tokens::Order}, {"SELECT", Tokens::Select}, {"SUM", Tokens::Sum}, + {"WHERE", Tokens::Where}, +}; + +static std::string DescribeCharacter(const char ch) { + switch (ch) { + case '\n': + return "\\n"; + case '\r': + return "\\r"; + case '\t': + return "\\t"; + default: + break; + } + + if (std::isprint(static_cast(ch)) != 0) { + return std::string(1, ch); + } + + return "code " + std::to_string(static_cast(static_cast(ch))); +} + +Tokens ResolveIdentifierType(const std::string_view text) { + const std::string upper = ToUpperAscii(text); + const auto it = kKeywords.find(upper); + if (it != kKeywords.end()) { + return it->second; + } + return Tokens::NameToken; +} + +TokenPtr MakeToken(const Tokens type, std::string text, const size_t offset) { + switch (type) { + case Tokens::NameToken: + return std::make_shared(std::move(text), offset); + case Tokens::NumericLiteral: + return std::make_shared(std::move(text), offset); + case Tokens::StringLiteral: + return std::make_shared(std::move(text), offset); + case Tokens::Select: + return std::make_shared(std::move(text), offset); + case Tokens::From: + return std::make_shared(std::move(text), offset); + case Tokens::Create: + return std::make_shared(std::move(text), offset); + case Tokens::As: + return std::make_shared(std::move(text), offset); + case Tokens::Sum: + return std::make_shared(std::move(text), offset); + case Tokens::Count: + return std::make_shared(std::move(text), offset); + case Tokens::Avg: + return std::make_shared(std::move(text), offset); + case Tokens::Max: + return std::make_shared(std::move(text), offset); + case Tokens::Min: + return std::make_shared(std::move(text), offset); + case Tokens::And: + return std::make_shared(std::move(text), offset); + case Tokens::Distinct: + return std::make_shared(std::move(text), offset); + case Tokens::Length: + return std::make_shared(std::move(text), offset); + case Tokens::Where: + return std::make_shared(std::move(text), offset); + case Tokens::By: + return std::make_shared(std::move(text), offset); + case Tokens::Group: + return std::make_shared(std::move(text), offset); + case Tokens::Order: + return std::make_shared(std::move(text), offset); + case Tokens::Limit: + return std::make_shared(std::move(text), offset); + case Tokens::Plus: + return std::make_shared(std::move(text), offset); + case Tokens::Minus: + return std::make_shared(std::move(text), offset); + case Tokens::Asterisk: + return std::make_shared(std::move(text), offset); + case Tokens::OpenBracket: + return std::make_shared(std::move(text), offset); + case Tokens::CloseBracket: + return std::make_shared(std::move(text), offset); + case Tokens::Comma: + return std::make_shared(std::move(text), offset); + case Tokens::Dot: + return std::make_shared(std::move(text), offset); + case Tokens::Semicolon: + return std::make_shared(std::move(text), offset); + case Tokens::Equal: + return std::make_shared(std::move(text), offset); + case Tokens::NotEqual: + return std::make_shared(std::move(text), offset); + case Tokens::Less: + return std::make_shared(std::move(text), offset); + case Tokens::LessOrEqual: + return std::make_shared(std::move(text), offset); + case Tokens::Greater: + return std::make_shared(std::move(text), offset); + case Tokens::GreaterOrEqual: + return std::make_shared(std::move(text), offset); + case Tokens::EndOfInput: + return std::make_shared(std::move(text), offset); + } + return nullptr; +} + +Expected MakeUnexpectedCharacter(const char ch, const size_t offset) { + return tl::unexpected(Error::InvalidData("tokenizer_factory", "unexpected character '" + DescribeCharacter(ch) + + "' at position " + std::to_string(offset))); +} + +Expected MakeUnterminatedString(const size_t offset) { + return tl::unexpected( + Error::InvalidData("tokenizer_factory", "unterminated string literal at position " + std::to_string(offset))); +} diff --git a/src/sql_parser/tokenizer_tokens.cpp b/src/sql_parser/tokenizer_tokens.cpp new file mode 100644 index 0000000..0d07844 --- /dev/null +++ b/src/sql_parser/tokenizer_tokens.cpp @@ -0,0 +1,133 @@ +#include + +#include "sql_parser/tokenizer.h" + +std::string_view TokenTypeToString(const Tokens type) noexcept { + switch (type) { + case Tokens::NameToken: + return "name"; + case Tokens::NumericLiteral: + return "numeric_literal"; + case Tokens::StringLiteral: + return "string_literal"; + case Tokens::Select: + return "select"; + case Tokens::From: + return "from"; + case Tokens::Create: + return "create"; + case Tokens::As: + return "as"; + case Tokens::Sum: + return "sum"; + case Tokens::Count: + return "count"; + case Tokens::Avg: + return "avg"; + case Tokens::Max: + return "max"; + case Tokens::Min: + return "min"; + case Tokens::And: + return "and"; + case Tokens::Distinct: + return "distinct"; + case Tokens::Length: + return "length"; + case Tokens::Where: + return "where"; + case Tokens::By: + return "by"; + case Tokens::Group: + return "group"; + case Tokens::Order: + return "order"; + case Tokens::Limit: + return "limit"; + case Tokens::Plus: + return "plus"; + case Tokens::Minus: + return "minus"; + case Tokens::Asterisk: + return "asterisk"; + case Tokens::OpenBracket: + return "open_bracket"; + case Tokens::CloseBracket: + return "close_bracket"; + case Tokens::Comma: + return "comma"; + case Tokens::Dot: + return "dot"; + case Tokens::Semicolon: + return "semicolon"; + case Tokens::Equal: + return "equal"; + case Tokens::NotEqual: + return "not_equal"; + case Tokens::Less: + return "less"; + case Tokens::LessOrEqual: + return "less_or_equal"; + case Tokens::Greater: + return "greater"; + case Tokens::GreaterOrEqual: + return "greater_or_equal"; + case Tokens::EndOfInput: + return "end_of_input"; + } + return "unknown"; +} + +Token::Token(std::string text, const size_t offset) : text_(std::move(text)), offset_(offset) {} + +std::string_view Token::GetText() const noexcept { return text_; } + +size_t Token::GetOffset() const noexcept { return offset_; } + +void Command::AddArg(TokenPtr arg) { args_.push_back(std::move(arg)); } + +const std::vector& Command::GetArgs() const noexcept { return args_; } + +// alert: не типобезопасная вещь, лучше переписать +#define COLUMNAR_DEFINE_TOKEN_TYPE(token_class, token_value) \ + Tokens token_class::GetType() const noexcept { return Tokens::token_value; } + +COLUMNAR_DEFINE_TOKEN_TYPE(TSelectToken, Select) +COLUMNAR_DEFINE_TOKEN_TYPE(TCreateToken, Create) +COLUMNAR_DEFINE_TOKEN_TYPE(TFromToken, From) +COLUMNAR_DEFINE_TOKEN_TYPE(TLimitToken, Limit) +COLUMNAR_DEFINE_TOKEN_TYPE(TOrderToken, Order) +COLUMNAR_DEFINE_TOKEN_TYPE(TWhereToken, Where) +COLUMNAR_DEFINE_TOKEN_TYPE(TGroupToken, Group) +COLUMNAR_DEFINE_TOKEN_TYPE(TSumToken, Sum) +COLUMNAR_DEFINE_TOKEN_TYPE(TMinToken, Min) +COLUMNAR_DEFINE_TOKEN_TYPE(TMaxToken, Max) +COLUMNAR_DEFINE_TOKEN_TYPE(TCountToken, Count) +COLUMNAR_DEFINE_TOKEN_TYPE(TDistinctToken, Distinct) +COLUMNAR_DEFINE_TOKEN_TYPE(TAvgToken, Avg) +COLUMNAR_DEFINE_TOKEN_TYPE(TLengthToken, Length) +COLUMNAR_DEFINE_TOKEN_TYPE(TPlusToken, Plus) +COLUMNAR_DEFINE_TOKEN_TYPE(TMinusToken, Minus) +COLUMNAR_DEFINE_TOKEN_TYPE(TNameToken, NameToken) +COLUMNAR_DEFINE_TOKEN_TYPE(TNumericLiteralToken, NumericLiteral) +COLUMNAR_DEFINE_TOKEN_TYPE(TStringLiteralToken, StringLiteral) +COLUMNAR_DEFINE_TOKEN_TYPE(TAsToken, As) +COLUMNAR_DEFINE_TOKEN_TYPE(TAndToken, And) +COLUMNAR_DEFINE_TOKEN_TYPE(TByToken, By) +COLUMNAR_DEFINE_TOKEN_TYPE(TOpenBracketToken, OpenBracket) +COLUMNAR_DEFINE_TOKEN_TYPE(TCommaToken, Comma) +COLUMNAR_DEFINE_TOKEN_TYPE(TCloseBracketToken, CloseBracket) +COLUMNAR_DEFINE_TOKEN_TYPE(TAsteriskToken, Asterisk) +COLUMNAR_DEFINE_TOKEN_TYPE(TDotToken, Dot) +COLUMNAR_DEFINE_TOKEN_TYPE(TSemicolonToken, Semicolon) +COLUMNAR_DEFINE_TOKEN_TYPE(TEqualToken, Equal) +COLUMNAR_DEFINE_TOKEN_TYPE(TNotEqualToken, NotEqual) +COLUMNAR_DEFINE_TOKEN_TYPE(TLessToken, Less) +COLUMNAR_DEFINE_TOKEN_TYPE(TLessOrEqualToken, LessOrEqual) +COLUMNAR_DEFINE_TOKEN_TYPE(TGreaterToken, Greater) +COLUMNAR_DEFINE_TOKEN_TYPE(TGreaterOrEqualToken, GreaterOrEqual) +COLUMNAR_DEFINE_TOKEN_TYPE(TEndOfInputToken, EndOfInput) + +#undef COLUMNAR_DEFINE_TOKEN_TYPE + +std::string_view TNameToken::GetName() const noexcept { return GetText(); } diff --git a/src/support/ascii.cpp b/src/support/ascii.cpp new file mode 100644 index 0000000..779323f --- /dev/null +++ b/src/support/ascii.cpp @@ -0,0 +1,21 @@ +#include "support/ascii.h" + +#include + +std::string ToLowerAscii(const std::string_view input) { + std::string output; + output.reserve(input.size()); + for (const unsigned char ch : input) { + output.push_back(static_cast(std::tolower(ch))); + } + return output; +} + +std::string ToUpperAscii(const std::string_view input) { + std::string output; + output.reserve(input.size()); + for (const unsigned char ch : input) { + output.push_back(static_cast(std::toupper(ch))); + } + return output; +} diff --git a/src/support/error.cpp b/src/support/error.cpp new file mode 100644 index 0000000..0867b49 --- /dev/null +++ b/src/support/error.cpp @@ -0,0 +1,107 @@ +#include "support/error.h" + +#include + +std::string Error::BuildWhat(const std::string_view module, const std::string_view message) { + return std::string(module) + ": " + std::string(message); +} + +Error::Error(const Code code, std::string module, std::string module_instance, const Type type, std::string message) + : code_(code), + module_(std::move(module)), + module_instance_(std::move(module_instance)), + type_(type), + message_(std::move(message)), + what_(BuildWhat(module_, message_)) {} + +Error Error::InvalidArgument(std::string module, std::string message, std::string module_instance) { + return Error(Code::InvalidArgument, std::move(module), std::move(module_instance), Type::Validation, + std::move(message)); +} + +Error Error::InvalidData(std::string module, std::string message, std::string module_instance) { + return Error(Code::InvalidData, std::move(module), std::move(module_instance), Type::Data, std::move(message)); +} + +Error Error::NotFound(std::string module, std::string message, std::string module_instance) { + return Error(Code::NotFound, std::move(module), std::move(module_instance), Type::State, std::move(message)); +} + +Error Error::OutOfRange(std::string module, std::string message, std::string module_instance) { + return Error(Code::OutOfRange, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); +} + +Error Error::Mismatch(std::string module, std::string message, std::string module_instance) { + return Error(Code::Mismatch, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); +} + +Error Error::Overflow(std::string module, std::string message, std::string module_instance) { + return Error(Code::Overflow, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); +} + +Error Error::InvalidState(std::string module, std::string message, std::string module_instance) { + return Error(Code::InvalidState, std::move(module), std::move(module_instance), Type::State, std::move(message)); +} + +Error Error::Unsupported(std::string module, std::string message, std::string module_instance) { + return Error(Code::Unsupported, std::move(module), std::move(module_instance), Type::Validation, + std::move(message)); +} + +Error Error::Io(std::string module, std::string message, std::string module_instance) { + return Error(Code::IoFailure, std::move(module), std::move(module_instance), Type::Io, std::move(message)); +} + +Error Error::PathIo(std::string module, const std::filesystem::path& path, const std::string_view action) { + return Io(std::move(module), "failed to " + std::string(action) + " '" + path.string() + "'", path.string()); +} + +const char* Error::what() const noexcept { return what_.c_str(); } + +Error::Code Error::GetCode() const noexcept { return code_; } + +Error::Type Error::GetType() const noexcept { return type_; } + +const std::string& Error::GetModule() const noexcept { return module_; } + +const std::string& Error::GetModuleInstance() const noexcept { return module_instance_; } + +const std::string& Error::GetMessage() const noexcept { return message_; } + +std::string_view Error::CodeToString(const Code code) noexcept { + switch (code) { + case Code::InvalidArgument: + return "invalid_argument"; + case Code::InvalidData: + return "invalid_data"; + case Code::NotFound: + return "not_found"; + case Code::OutOfRange: + return "out_of_range"; + case Code::Mismatch: + return "mismatch"; + case Code::Overflow: + return "overflow"; + case Code::InvalidState: + return "invalid_state"; + case Code::Unsupported: + return "unsupported"; + case Code::IoFailure: + return "io_failure"; + } + return "unknown"; +} + +std::string_view Error::TypeToString(const Type type) noexcept { + switch (type) { + case Type::Validation: + return "validation"; + case Type::Data: + return "data"; + case Type::State: + return "state"; + case Type::Io: + return "io"; + } + return "unknown"; +} diff --git a/src/support/parsing.cpp b/src/support/parsing.cpp new file mode 100644 index 0000000..5737ef2 --- /dev/null +++ b/src/support/parsing.cpp @@ -0,0 +1,315 @@ +#include "support/parsing.h" + +#include +#include +#include +#include +#include + +#include "support/ascii.h" +#include "support/error.h" + +template +T ParseInteger(const std::string& value, const std::string_view type_name) { + if (value.empty()) { + throw Error::InvalidData("parsing", "empty value for " + std::string(type_name)); + } + + T result = 0; + const char* begin = value.data(); + const char* end = begin + value.size(); + + if (const auto [ptr, ec] = std::from_chars(begin, end, result); ec != std::errc() || ptr != end) { + throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); + } + + return result; +} + +static unsigned ParseUnsignedPart(const std::string_view input, const size_t offset, const size_t length, + const std::string_view type_name) { + unsigned value = 0; + const char* begin = input.data() + offset; + const char* end = begin + length; + + if (const auto [ptr, ec] = std::from_chars(begin, end, value); ec != std::errc() || ptr != end) { + throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); + } + + return value; +} + +static std::chrono::sys_days ParseDatePoint(const std::string_view value, const std::string_view type_name) { + if (value.size() != 10 || value[4] != '-' || value[7] != '-') { + throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); + } + + const int year = static_cast(ParseUnsignedPart(value, 0, 4, type_name)); + const unsigned month = ParseUnsignedPart(value, 5, 2, type_name); + const unsigned day = ParseUnsignedPart(value, 8, 2, type_name); + + const std::chrono::year_month_day ymd{ + std::chrono::year{year}, + std::chrono::month{month}, + std::chrono::day{day}, + }; + + if (!ymd.ok()) { + throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); + } + + return std::chrono::sys_days{ymd}; +} + +static std::string FormatDateParts(const std::chrono::year_month_day ymd) { + std::ostringstream out; + out << std::setfill('0') << std::setw(4) << static_cast(ymd.year()) << '-' << std::setw(2) + << static_cast(ymd.month()) << '-' << std::setw(2) << static_cast(ymd.day()); + return out.str(); +} + +bool ParseBoolean(const std::string& value) { + const std::string lowered = ToLowerAscii(value); + if (lowered == "true" || lowered == "1") { + return true; + } + if (lowered == "false" || lowered == "0") { + return false; + } + throw Error::InvalidData("parsing", "invalid bool value"); +} + +int16_t ParseInt16(const std::string& value) { return ParseInteger(value, "int16"); } + +int32_t ParseInt32(const std::string& value) { return ParseInteger(value, "int32"); } + +int64_t ParseInt64(const std::string& value) { return ParseInteger(value, "int64"); } + +Int128 ParseInt128(const std::string& value) { + if (value.empty()) { + throw Error::InvalidData("parsing", "empty value for int128"); + } + + size_t pos = 0; + bool negative = false; + + if (value[pos] == '+' || value[pos] == '-') { + negative = value[pos] == '-'; + ++pos; + } + + if (pos == value.size()) { + throw Error::InvalidData("parsing", "invalid int128 value"); + } + + const UInt128 positive_limit = ~UInt128{0} >> 1; + const UInt128 negative_limit = positive_limit + 1; + const UInt128 limit = negative ? negative_limit : positive_limit; + + UInt128 magnitude = 0; + + for (; pos < value.size(); ++pos) { + const char ch = value[pos]; + if (ch < '0' || ch > '9') { + throw Error::InvalidData("parsing", "invalid int128 value"); + } + + const auto digit = static_cast(ch - '0'); + if (magnitude > (limit - digit) / 10) { + throw Error::InvalidData("parsing", "invalid int128 value"); + } + magnitude = magnitude * 10 + digit; + } + + if (!negative) { + return static_cast(magnitude); + } + if (magnitude == negative_limit) { + return -static_cast(positive_limit) - 1; + } + + return -static_cast(magnitude); +} + +int32_t ParseDate(const std::string& value) { + const auto day_point = ParseDatePoint(value, "date"); + const auto days_since_epoch = day_point.time_since_epoch().count(); + + if (days_since_epoch < std::numeric_limits::min() || + days_since_epoch > std::numeric_limits::max()) { + throw Error::Overflow("parsing", "date value exceeds supported range"); + } + + return days_since_epoch; +} + +int64_t ParseTimestamp(const std::string& value) { + if (value.size() < 19) { + throw Error::InvalidData("parsing", "invalid timestamp value"); + } + + const auto day_point = ParseDatePoint(std::string_view(value).substr(0, 10), "timestamp"); + + if (value[10] != ' ' && value[10] != 'T') { + throw Error::InvalidData("parsing", "invalid timestamp value"); + } + if (value[13] != ':' || value[16] != ':') { + throw Error::InvalidData("parsing", "invalid timestamp value"); + } + + const unsigned hour = ParseUnsignedPart(value, 11, 2, "timestamp"); + const unsigned minute = ParseUnsignedPart(value, 14, 2, "timestamp"); + const unsigned second = ParseUnsignedPart(value, 17, 2, "timestamp"); + + if (hour > 23 || minute > 59 || second > 59) { + throw Error::InvalidData("parsing", "invalid timestamp value"); + } + + int64_t microseconds = 0; + + if (value.size() > 19) { + if (value[19] != '.') { + throw Error::InvalidData("parsing", "invalid timestamp value"); + } + + const size_t digits = value.size() - 20; + if (digits == 0 || digits > 6) { + throw Error::InvalidData("parsing", "invalid timestamp value"); + } + + const auto fractional = ParseUnsignedPart(value, 20, digits, "timestamp"); + microseconds = static_cast(fractional); + for (size_t i = digits; i < 6; ++i) { + microseconds *= 10; + } + } + + const auto timestamp = day_point + std::chrono::hours{hour} + std::chrono::minutes{minute} + + std::chrono::seconds{second} + std::chrono::microseconds{microseconds}; + + return std::chrono::duration_cast(timestamp.time_since_epoch()).count(); +} + +char ParseCharacter(const std::string& value) { + if (value.size() != 1) { + throw Error::InvalidData("parsing", "invalid char value"); + } + return value[0]; +} + +std::string BooleanToString(const bool value) { return value ? "true" : "false"; } + +std::string Int128ToString(const Int128 value) { + if (value == 0) { + return "0"; + } + + const bool negative = value < 0; + UInt128 magnitude = 0; + + if (negative) { + magnitude = static_cast(-(value + 1)); + ++magnitude; + } else { + magnitude = static_cast(value); + } + + std::string out; + + while (magnitude > 0) { + out.push_back(static_cast('0' + magnitude % 10)); + magnitude /= 10; + } + if (negative) { + out.push_back('-'); + } + + std::reverse(out.begin(), out.end()); + + return out; +} + +std::string DateToString(const int32_t value) { + const auto day_point = std::chrono::sys_days{} + std::chrono::days{value}; + return FormatDateParts(std::chrono::year_month_day{day_point}); +} + +std::string TimestampToString(const int64_t value) { + const std::chrono::sys_time timestamp{std::chrono::microseconds{value}}; + const auto day_point = std::chrono::floor(timestamp); + const std::chrono::hh_mm_ss tod{timestamp - day_point}; + + std::ostringstream out; + out << FormatDateParts(std::chrono::year_month_day{day_point}) << ' ' << std::setfill('0') << std::setw(2) + << tod.hours().count() << ':' << std::setw(2) << tod.minutes().count() << ':' << std::setw(2) + << tod.seconds().count(); + + const int64_t fractional = tod.subseconds().count(); + + if (fractional != 0) { + std::string frac = std::to_string(fractional); + frac.insert(frac.begin(), 6 - static_cast(frac.size()), '0'); + while (!frac.empty() && frac.back() == '0') { + frac.pop_back(); + } + out << '.' << frac; + } + + return out.str(); +} + +ColumnType ParseColumnType(const std::string_view input) { + const std::string lowered = ToLowerAscii(input); + if (lowered == "bool") { + return ColumnType::Boolean; + } + if (lowered == "int16") { + return ColumnType::Int16; + } + if (lowered == "int32") { + return ColumnType::Int32; + } + if (lowered == "int64") { + return ColumnType::Int64; + } + if (lowered == "int128") { + return ColumnType::Int128; + } + if (lowered == "string") { + return ColumnType::String; + } + if (lowered == "date") { + return ColumnType::Date; + } + if (lowered == "timestamp") { + return ColumnType::Timestamp; + } + if (lowered == "char") { + return ColumnType::Character; + } + throw Error::Unsupported("parsing", "unknown column type"); +} + +std::string ColumnTypeToString(const ColumnType type) { + switch (type) { + case ColumnType::Boolean: + return "bool"; + case ColumnType::Int16: + return "int16"; + case ColumnType::Int32: + return "int32"; + case ColumnType::Int64: + return "int64"; + case ColumnType::Int128: + return "int128"; + case ColumnType::String: + return "string"; + case ColumnType::Date: + return "date"; + case ColumnType::Timestamp: + return "timestamp"; + case ColumnType::Character: + return "char"; + } + throw Error::Unsupported("parsing", "unknown column type"); +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..988c0e4 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,16 @@ +include(GoogleTest) + +function(add_component_test name) + add_executable(${name} ${ARGN}) + target_link_libraries(${name} PRIVATE columnar_engine_lib GTest::gtest_main) + gtest_discover_tests(${name}) +endfunction() + +add_component_test(test_fileio test_fileio.cpp) +add_component_test(test_columnar test_columnar.cpp) +add_component_test(test_csv test_csv.cpp) +add_component_test(test_batch test_batch.cpp) +add_component_test(test_columns test_columns.cpp) +add_component_test(test_stream test_stream.cpp) +add_component_test(test_tokenizer test_tokenizer.cpp) +add_component_test(test_executor test_executor.cpp) diff --git a/tests/test_batch.cpp b/tests/test_batch.cpp new file mode 100644 index 0000000..cfbfb5a --- /dev/null +++ b/tests/test_batch.cpp @@ -0,0 +1,277 @@ +#include +#include +#include +#include + +#include "io/columnar_batch_io.h" +#include "io/csv.h" +#include "io/csv_batch_io.h" +#include "support/error.h" +#include "gtest/gtest.h" +#include "testing/temp_file.h" + +static_assert(std::is_copy_constructible_v); +static_assert(std::is_copy_assignable_v); +static_assert(std::is_move_constructible_v); +static_assert(std::is_move_assignable_v); + +TEST(batch, usage_example_csv_to_columnar_roundtrip) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + const TempFile csv_file("batch_usage_csv"); + const TempFile columnar_file("batch_usage_columnar"); + WriteRows(csv_file.Path(), { + {"1", "alpha"}, + {"2", "beta"}, + }); + + CsvBatchReader csv_reader(csv_file.Path(), schema, {}); + ColumnarBatchWriter columnar_writer(columnar_file.Path(), schema); + + while (auto batch = csv_reader.ReadNext()) { + columnar_writer.Write(*batch); + } + std::move(columnar_writer).Finalize(); + + ColumnarBatchReader columnar_reader(columnar_file.Path()); + auto batch = columnar_reader.ReadNext(); + ASSERT_TRUE(batch.has_value()); + EXPECT_EQ(batch->RowsCount(), 2u); + EXPECT_EQ(batch->ColumnAt(0).ValueAsString(0), "1"); + EXPECT_EQ(batch->ColumnAt(1).ValueAsString(1), "beta"); + EXPECT_FALSE(columnar_reader.ReadNext().has_value()); +} + +TEST(batch, csv_reader_respects_max_rows) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + const TempFile data_in("batch_csv"); + const std::vector> data_rows = { + {"1", "alpha"}, + {"2", "be,ta"}, + {"3", "gamma"}, + }; + + WriteRows(data_in.Path(), data_rows); + + BatchSizing sizing; + sizing.max_rows = 2; + CsvBatchReader reader(data_in.Path(), schema, sizing); + + auto first = reader.ReadNext(); + ASSERT_TRUE(first.has_value()); + EXPECT_EQ(first->RowsCount(), 2u); + EXPECT_EQ(first->ColumnAt(0).ValueAsString(0), "1"); + EXPECT_EQ(first->ColumnAt(1).ValueAsString(1), "be,ta"); + + auto second = reader.ReadNext(); + ASSERT_TRUE(second.has_value()); + EXPECT_EQ(second->RowsCount(), 1u); + EXPECT_EQ(second->ColumnAt(0).ValueAsString(0), "3"); + EXPECT_EQ(second->ColumnAt(1).ValueAsString(0), "gamma"); + + EXPECT_FALSE(reader.ReadNext().has_value()); +} + +TEST(batch, write_batch_csv_writes_single_batch) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + Batch batch(schema); + batch.ColumnAt(0).AppendFromString("1"); + batch.ColumnAt(1).AppendFromString("alpha"); + batch.ColumnAt(0).AppendFromString("2"); + batch.ColumnAt(1).AppendFromString("be,ta"); + + const TempFile data_out("batch_write_csv"); + WriteBatchCsv(data_out.Path(), batch); + + EXPECT_EQ(ReadRows(data_out.Path()), (std::vector>{ + {"1", "alpha"}, + {"2", "be,ta"}, + })); +} + +TEST(batch, columnar_roundtrip) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + const TempFile data_in("batch_roundtrip_csv"); + const TempFile columnar_file("batch_roundtrip_columnar"); + + const std::vector> data_rows = { + {"1", "alpha"}, + {"2", "be,ta"}, + {"3", "gamma"}, + }; + + WriteRows(data_in.Path(), data_rows); + + BatchSizing sizing; + sizing.max_rows = 2; + CsvBatchReader csv_reader(data_in.Path(), schema, sizing); + ColumnarBatchWriter columnar_writer(columnar_file.Path(), schema); + + while (auto batch = csv_reader.ReadNext()) { + columnar_writer.Write(*batch); + } + std::move(columnar_writer).Finalize(); + + ColumnarBatchReader columnar_reader(columnar_file.Path()); + EXPECT_EQ(columnar_reader.GetMetadata().row_groups.size(), 2u); + + std::vector> roundtrip_rows; + while (auto batch = columnar_reader.ReadNext()) { + AppendBatchRows(*batch, roundtrip_rows); + } + + EXPECT_EQ(roundtrip_rows, data_rows); +} + +TEST(batch, validate_detects_row_count_mismatch) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + Batch batch(schema); + batch.ColumnAt(0).AppendFromString("1"); + batch.ColumnAt(0).AppendFromString("2"); + batch.ColumnAt(1).AppendFromString("alpha"); + + EXPECT_THROW(batch.Validate(), Error); +} + +TEST(batch, copy_is_deep) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + Batch original(schema); + original.ColumnAt(0).AppendFromString("1"); + original.ColumnAt(1).AppendFromString("alpha"); + + Batch copied = original; + copied.ColumnAt(0).AppendFromString("2"); + copied.ColumnAt(1).AppendFromString("beta"); + + EXPECT_EQ(original.RowsCount(), 1u); + EXPECT_EQ(original.ColumnAt(0).ValueAsString(0), "1"); + EXPECT_EQ(original.ColumnAt(1).ValueAsString(0), "alpha"); + + EXPECT_EQ(copied.RowsCount(), 2u); + EXPECT_EQ(copied.ColumnAt(0).ValueAsString(1), "2"); + EXPECT_EQ(copied.ColumnAt(1).ValueAsString(1), "beta"); +} + +TEST(batch, csv_reader_respects_max_values) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + const TempFile data_in("batch_csv_values"); + const std::vector> data_rows = { + {"1", "alpha"}, + {"2", "beta"}, + {"3", "gamma"}, + }; + + WriteRows(data_in.Path(), data_rows); + + BatchSizing sizing; + sizing.max_values = 3; + CsvBatchReader reader(data_in.Path(), schema, sizing); + + for (size_t i = 0; i < data_rows.size(); ++i) { + auto batch = reader.ReadNext(); + ASSERT_TRUE(batch.has_value()); + EXPECT_EQ(batch->RowsCount(), 1u); + EXPECT_EQ(batch->ColumnAt(0).ValueAsString(0), data_rows[i][0]); + EXPECT_EQ(batch->ColumnAt(1).ValueAsString(0), data_rows[i][1]); + } + + EXPECT_FALSE(reader.ReadNext().has_value()); +} + +TEST(batch, csv_reader_respects_max_bytes) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + const TempFile data_in("batch_csv_bytes"); + const std::vector> data_rows = { + {"1", "aaaa"}, + {"2", "bbbb"}, + {"3", "cccc"}, + }; + + WriteRows(data_in.Path(), data_rows); + + BatchSizing sizing; + sizing.max_bytes = sizeof(int64_t) + 4u; + CsvBatchReader reader(data_in.Path(), schema, sizing); + + for (size_t i = 0; i < data_rows.size(); ++i) { + auto batch = reader.ReadNext(); + ASSERT_TRUE(batch.has_value()); + EXPECT_EQ(batch->RowsCount(), 1u); + EXPECT_EQ(batch->ColumnAt(0).ValueAsString(0), data_rows[i][0]); + EXPECT_EQ(batch->ColumnAt(1).ValueAsString(0), data_rows[i][1]); + } + + EXPECT_FALSE(reader.ReadNext().has_value()); +} + +TEST(batch, csv_reader_throws_on_column_mismatch) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + const TempFile data_in("batch_csv_mismatch"); + WriteRows(data_in.Path(), {{"1"}}); + + BatchSizing sizing; + sizing.max_rows = 2; + CsvBatchReader reader(data_in.Path(), schema, sizing); + + EXPECT_THROW(reader.ReadNext(), Error); +} + +TEST(batch, columnar_writer_rejects_after_finalize) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + const TempFile columnar_file("batch_finalize_columnar"); + ColumnarBatchWriter writer(columnar_file.Path(), schema); + std::move(writer).Finalize(); + + Batch batch(schema); + EXPECT_THROW(writer.Write(batch), Error); + EXPECT_THROW(std::move(writer).Finalize(), Error); +} diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp new file mode 100644 index 0000000..f6051f5 --- /dev/null +++ b/tests/test_columnar.cpp @@ -0,0 +1,208 @@ +#include +#include +#include + +#include "io/columnar_batch_io.h" +#include "convert/csv_columnar.h" +#include "io/csv.h" +#include "support/error.h" +#include "io/fileio.h" +#include "gtest/gtest.h" +#include "support/parsing.h" +#include "model/schema.h" +#include "testing/temp_file.h" + +TEST(columnar, csv_to_columnar_and_back) { + const TempFile schema_in("schema_in"); + const TempFile data_in("data_in"); + const TempFile columnar_file("columnar"); + const TempFile schema_out("schema_out"); + const TempFile data_out("data_out"); + + const std::vector> schema_rows = { + {"a", "int64"}, + {"b", "int64"}, + {"name", "string"}, + {"d", "int64"}, + }; + + const std::vector> data_rows = { + {"1", "2", "first", "4"}, + {"5", "1", "sec,ond", "2"}, + {"8", "17", "thi\"rd", "2"}, + {"9", "5", "line1\nline2", "10"}, + }; + + WriteRows(schema_in.Path(), schema_rows); + WriteRows(data_in.Path(), data_rows); + + ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 2); + ConvertColumnarToCsv(columnar_file.Path(), schema_out.Path(), data_out.Path()); + + const auto [columns] = ReadSchemaCsv(schema_out.Path()); + ASSERT_EQ(columns.size(), schema_rows.size()); + for (size_t i = 0; i < schema_rows.size(); ++i) { + EXPECT_EQ(columns[i].name, schema_rows[i][0]); + EXPECT_EQ(ColumnTypeToString(columns[i].type), schema_rows[i][1]); + } + + const auto data_roundtrip = ReadRows(data_out.Path()); + EXPECT_EQ(data_roundtrip, data_rows); +} + +TEST(columnar, metadata_offsets_and_sizes) { + const TempFile schema_in("schema_meta_in"); + const TempFile data_in("data_meta_in"); + const TempFile columnar_file("columnar_meta"); + + const std::vector> schema_rows = { + {"id", "int64"}, + {"name", "string"}, + }; + + const std::vector> data_rows = { + {"1", "a"}, + {"2", "bb"}, + {"3", "ccc"}, + }; + + WriteRows(schema_in.Path(), schema_rows); + WriteRows(data_in.Path(), data_rows); + + ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 2); + + ColumnarBatchReader reader(columnar_file.Path()); + const auto& [schema, row_groups] = reader.GetMetadata(); + ASSERT_EQ(schema.columns.size(), 2u); + EXPECT_EQ(schema.columns[0].name, "id"); + EXPECT_EQ(schema.columns[0].type, ColumnType::Int64); + EXPECT_EQ(schema.columns[1].name, "name"); + EXPECT_EQ(schema.columns[1].type, ColumnType::String); + + ASSERT_EQ(row_groups.size(), 2u); + EXPECT_EQ(row_groups[0].row_count, 2u); + EXPECT_EQ(row_groups[1].row_count, 1u); + + ASSERT_EQ(row_groups[0].columns.size(), 2u); + ASSERT_EQ(row_groups[1].columns.size(), 2u); + + auto string_chunk_size = [&data_rows](const size_t start, const size_t count) -> uint64_t { + uint64_t total = 0; + for (size_t i = 0; i < count; ++i) { + total += sizeof(uint32_t); + total += data_rows[start + i][1].size(); + } + return total; + }; + + EXPECT_EQ(row_groups[0].columns[0].size, sizeof(int64_t) * 2u); + EXPECT_EQ(row_groups[0].columns[1].size, string_chunk_size(0, 2)); + EXPECT_EQ(row_groups[1].columns[0].size, sizeof(int64_t) * 1u); + EXPECT_EQ(row_groups[1].columns[1].size, string_chunk_size(2, 1)); + + uint64_t expected_offset = 0; + for (const auto& [row_count, columns] : row_groups) { + for (const auto& [offset, size] : columns) { + EXPECT_EQ(offset, expected_offset); + expected_offset += size; + } + } + + const auto file_info = TryGetFileMetadata(columnar_file.Path()); + ASSERT_TRUE(file_info.has_value()); + EXPECT_TRUE(file_info->is_regular); + EXPECT_LT(expected_offset, file_info->size); +} + +TEST(columnar, invalid_int64_throws) { + const TempFile schema_in("schema_bad_int"); + const TempFile data_in("data_bad_int"); + const TempFile columnar_file("columnar_bad_int"); + + WriteRows(schema_in.Path(), {{"value", "int64"}}); + WriteRows(data_in.Path(), {{"not_a_number"}}); + + EXPECT_THROW(ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 4), Error); +} + +TEST(columnar, mixed_supported_types_roundtrip) { + const TempFile schema_in("schema_mixed_in"); + const TempFile data_in("data_mixed_in"); + const TempFile columnar_file("columnar_mixed"); + const TempFile schema_out("schema_mixed_out"); + const TempFile data_out("data_mixed_out"); + + const std::vector> schema_rows = { + {"flag", "bool"}, + {"small", "int16"}, + {"medium", "int32"}, + {"big", "int64"}, + {"huge", "int128"}, + {"name", "string"}, + {"birth_date", "date"}, + {"created_at", "timestamp"}, + {"grade", "char"}, + }; + + const std::vector> data_rows = { + {"true", "-7", "12345", "9999999999", "170141183460469231731687303715884105727", "alpha", "2024-02-29", + "2024-02-29 12:34:56.123456", "A"}, + {"false", "8", "-54321", "-42", "-170141183460469231731687303715884105728", "be,ta", "1970-01-01", + "1970-01-01 00:00:00", "Z"}, + }; + + WriteRows(schema_in.Path(), schema_rows); + WriteRows(data_in.Path(), data_rows); + + ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 1); + ConvertColumnarToCsv(columnar_file.Path(), schema_out.Path(), data_out.Path()); + + EXPECT_EQ(ReadRows(schema_out.Path()), schema_rows); + EXPECT_EQ(ReadRows(data_out.Path()), data_rows); +} + +TEST(columnar, infer_schema_from_csv_is_convert_compatible) { + const TempFile data_in("data_infer_in"); + const TempFile schema_out("schema_infer_out"); + const TempFile columnar_file("columnar_infer"); + const TempFile data_out("data_infer_out"); + + const std::vector> data_rows = { + {"true", "-7", "12345", "9999999999", "170141183460469231731687303715884105727", "alpha", "2024-02-29", + "2024-02-29 12:34:56.123456", "A"}, + {"false", "8", "-54321", "-42", "-170141183460469231731687303715884105728", "be,ta", "1970-01-01", + "1970-01-01 00:00:00", "Z"}, + }; + + WriteRows(data_in.Path(), data_rows); + WriteSchemaCsv(schema_out.Path(), InferSchemaCsv(data_in.Path())); + + EXPECT_EQ(ReadRows(schema_out.Path()), + (std::vector>{ + {"column_1", "bool"}, + {"column_2", "int16"}, + {"column_3", "int32"}, + {"column_4", "int64"}, + {"column_5", "int128"}, + {"column_6", "string"}, + {"column_7", "date"}, + {"column_8", "timestamp"}, + {"column_9", "char"}, + })); + + ConvertCsvToColumnar(schema_out.Path(), data_in.Path(), columnar_file.Path(), 1); + ConvertColumnarToCsv(columnar_file.Path(), schema_out.Path(), data_out.Path()); + + EXPECT_EQ(ReadRows(data_out.Path()), data_rows); +} + +TEST(columnar, schema_rows_with_trailing_empty_columns_are_accepted) { + const TempFile schema_in("schema_trailing_empty"); + const TempFile data_in("data_trailing_empty"); + const TempFile columnar_file("columnar_trailing_empty"); + + WriteRows(schema_in.Path(), {{"id", "int64", ""}, {"name", "string", ""}}); + WriteRows(data_in.Path(), {{"1", "alpha"}, {"2", "beta"}}); + + EXPECT_NO_THROW(ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 2)); +} diff --git a/tests/test_columns.cpp b/tests/test_columns.cpp new file mode 100644 index 0000000..8c32a40 --- /dev/null +++ b/tests/test_columns.cpp @@ -0,0 +1,138 @@ +#include +#include +#include + +#include "model/column.h" +#include "model/column_int64.h" +#include "model/column_string.h" +#include "support/error.h" +#include "gtest/gtest.h" +#include "support/parsing.h" + +static void ExpectColumnRoundtrip(const ColumnType type, const std::vector& values, + const std::vector& expected_values = {}) { + auto column = CreateColumn(type); + for (const auto& value : values) { + column->AppendFromString(value); + } + + std::stringstream buffer; + column->WriteTo(buffer); + const std::string bytes = buffer.str(); + + auto read_back = CreateColumn(type); + std::stringstream in(bytes); + read_back->ReadFrom(in, values.size(), bytes.size()); + + ASSERT_EQ(read_back->Size(), values.size()); + const auto& expected = expected_values.empty() ? values : expected_values; + ASSERT_EQ(expected.size(), values.size()); + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(read_back->ValueAsString(i), expected[i]); + } +} + +TEST(columns, int64_roundtrip) { + Int64Column values; + values.AppendFromString("10"); + values.AppendFromString("-5"); + + std::stringstream buffer; + values.WriteTo(buffer); + const std::string bytes = buffer.str(); + + Int64Column read_back; + std::stringstream in(bytes); + read_back.ReadFrom(in, 2, bytes.size()); + + EXPECT_EQ(read_back.Size(), 2u); + EXPECT_EQ(read_back.ValueAsString(0), "10"); + EXPECT_EQ(read_back.ValueAsString(1), "-5"); +} + +TEST(columns, string_roundtrip) { + StringColumn values; + values.AppendFromString("alpha"); + values.AppendFromString("line1\nline2"); + values.AppendFromString("be,ta"); + + std::stringstream buffer; + values.WriteTo(buffer); + const std::string bytes = buffer.str(); + + StringColumn read_back; + std::stringstream in(bytes); + read_back.ReadFrom(in, 3, bytes.size()); + + EXPECT_EQ(read_back.Size(), 3u); + EXPECT_EQ(read_back.ValueAsString(0), "alpha"); + EXPECT_EQ(read_back.ValueAsString(1), "line1\nline2"); + EXPECT_EQ(read_back.ValueAsString(2), "be,ta"); +} + +TEST(columns, int64_invalid_value_throws) { + Int64Column values; + EXPECT_THROW(values.AppendFromString("not_a_number"), Error); +} + +TEST(columns, string_out_of_range_throws) { + StringColumn values; + values.AppendFromString("alpha"); + EXPECT_THROW(values.ValueAsString(1), Error); +} + +TEST(columns, supported_scalar_types_roundtrip) { + ExpectColumnRoundtrip(ColumnType::Boolean, {"true", "false", "1"}, {"true", "false", "true"}); + ExpectColumnRoundtrip(ColumnType::Int16, {"-32768", "0", "32767"}); + ExpectColumnRoundtrip(ColumnType::Int32, {"-2147483648", "7", "2147483647"}); + ExpectColumnRoundtrip( + ColumnType::Int128, + {"-170141183460469231731687303715884105728", "0", "170141183460469231731687303715884105727"}); + ExpectColumnRoundtrip(ColumnType::Date, {"1970-01-01", "2024-02-29", "2030-12-31"}); + ExpectColumnRoundtrip( + ColumnType::Timestamp, + {"1970-01-01 00:00:00", "2024-02-29 12:34:56.123456", "2030-12-31T23:59:59.5"}, + {"1970-01-01 00:00:00", "2024-02-29 12:34:56.123456", "2030-12-31 23:59:59.5"}); + ExpectColumnRoundtrip(ColumnType::Character, {"A", ",", "\n"}); +} + +TEST(columns, scalar_invalid_values_throw) { + auto boolean = CreateColumn(ColumnType::Boolean); + EXPECT_THROW(boolean->AppendFromString("maybe"), Error); + + auto int16 = CreateColumn(ColumnType::Int16); + EXPECT_THROW(int16->AppendFromString("40000"), Error); + + auto int32 = CreateColumn(ColumnType::Int32); + EXPECT_THROW(int32->AppendFromString("2147483648"), Error); + + auto int128 = CreateColumn(ColumnType::Int128); + EXPECT_THROW(int128->AppendFromString("170141183460469231731687303715884105728"), Error); + + auto date = CreateColumn(ColumnType::Date); + EXPECT_THROW(date->AppendFromString("2024-02-30"), Error); + + auto timestamp = CreateColumn(ColumnType::Timestamp); + EXPECT_THROW(timestamp->AppendFromString("2024-02-29 24:00:00"), Error); + + auto character = CreateColumn(ColumnType::Character); + EXPECT_THROW(character->AppendFromString("ab"), Error); +} + +TEST(columns, type_names_roundtrip) { + const std::vector types = { + ColumnType::Boolean, + ColumnType::Int16, + ColumnType::Int32, + ColumnType::Int64, + ColumnType::Int128, + ColumnType::String, + ColumnType::Date, + ColumnType::Timestamp, + ColumnType::Character, + }; + + for (const auto type : types) { + EXPECT_EQ(ParseColumnType(ColumnTypeToString(type)), type); + } +} diff --git a/tests/test_csv.cpp b/tests/test_csv.cpp new file mode 100644 index 0000000..c892da5 --- /dev/null +++ b/tests/test_csv.cpp @@ -0,0 +1,65 @@ +#include +#include +#include +#include + +#include "io/csv.h" +#include "gtest/gtest.h" + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_move_constructible_v); +static_assert(std::is_move_assignable_v); +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_move_constructible_v); +static_assert(std::is_move_assignable_v); + +TEST(csv, read_write_roundtrip_with_quotes) { + std::stringstream buffer; + CsvWriter writer(buffer); + CsvReader reader(buffer); + + writer.WriteRow({"1", "2,3", "he\"llo", "line1\nline2"}); + writer.WriteRow({"", "plain"}); + + std::vector row; + ASSERT_TRUE(reader.ReadRow(row)); + EXPECT_EQ(row, (std::vector{"1", "2,3", "he\"llo", "line1\nline2"})); + + ASSERT_TRUE(reader.ReadRow(row)); + EXPECT_EQ(row, (std::vector{"", "plain"})); + + EXPECT_FALSE(reader.ReadRow(row)); +} + +TEST(csv, handles_crlf_newlines) { + std::istringstream input("a,b\r\nc,d\r\n"); + CsvReader reader(input); + std::vector row; + + ASSERT_TRUE(reader.ReadRow(row)); + EXPECT_EQ(row, (std::vector{"a", "b"})); + + ASSERT_TRUE(reader.ReadRow(row)); + EXPECT_EQ(row, (std::vector{"c", "d"})); + + EXPECT_FALSE(reader.ReadRow(row)); +} + +TEST(csv, moved_reader_and_writer_remain_usable) { + std::stringstream buffer; + CsvWriter writer(buffer); + CsvWriter moved_writer(std::move(writer)); + moved_writer.WriteRow({"1", "value"}); + moved_writer.Flush(); + + std::istringstream input(buffer.str()); + CsvReader reader(input); + CsvReader moved_reader(std::move(reader)); + + std::vector row; + ASSERT_TRUE(moved_reader.ReadRow(row)); + EXPECT_EQ(row, (std::vector{"1", "value"})); + EXPECT_FALSE(moved_reader.ReadRow(row)); +} diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp new file mode 100644 index 0000000..a17bbda --- /dev/null +++ b/tests/test_executor.cpp @@ -0,0 +1,203 @@ +#include +#include +#include + +#include "convert/csv_columnar.h" +#include "executor/executor.h" +#include "gtest/gtest.h" +#include "io/csv.h" +#include "testing/executor_test_utils.h" +#include "testing/temp_file.h" + +static Batch BuildHitsTable(const std::string_view query) { + const TempFile schema_file("executor_schema"); + const TempFile data_file("executor_data"); + const TempFile columnar_file("executor_columnar"); + + WriteRows(schema_file.Path(), { + {"AdvEngineID", "int64"}, + {"ResolutionWidth", "int32"}, + {"UserID", "int64"}, + {"RegionID", "int32"}, + {"MobilePhone", "bool"}, + {"MobilePhoneModel", "string"}, + {"SearchEngineID", "int32"}, + {"SearchPhrase", "string"}, + {"EventDate", "date"}, + }); + + WriteRows(data_file.Path(), { + {"0", "100", "1", "10", "false", "", "1", "alpha", "2024-01-05"}, + {"7", "200", "2", "20", "true", "iPhone", "1", "beta", "2024-01-03"}, + {"3", "301", "2", "10", "true", "Pixel", "2", "alpha", "2024-01-07"}, + {"7", "500", "5", "20", "true", "iPhone", "1", "gamma", "2024-01-01"}, + {"1", "400", "11", "30", "false", "Nokia", "2", "delta", "2024-01-09"}, + }); + + ConvertCsvToColumnar(schema_file.Path(), data_file.Path(), columnar_file.Path(), 2); + + Executor executor; + executor.RegisterTable("hits", columnar_file.Path()); + + auto result = executor.Execute(query); + if (!result.has_value()) { + throw result.error(); + } + + return std::move(result.value()); +} + +TEST(executor, usage_example_register_table_and_execute_query) { + const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"5"}); +} + +TEST(executor, groups_rows_and_orders_by_aggregate_descending) { + const Batch batch = BuildHitsTable( + "SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"7", "2"}, + {"1", "1"}, + {"3", "1"}, + }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); + std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_from_with_as_alias_and_qualified_columns) { + const Batch batch = BuildHitsTable( + "SELECT h.AdvEngineID, COUNT(*) FROM hits AS h WHERE h.AdvEngineID <> 0 GROUP BY h.AdvEngineID ORDER BY " + "COUNT(*) DESC;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"7", "2"}, + {"1", "1"}, + {"3", "1"}, + }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); + std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_from_with_bare_alias) { + const Batch batch = BuildHitsTable( + "SELECT h.AdvEngineID, COUNT(*) FROM hits h WHERE h.AdvEngineID <> 0 GROUP BY h.AdvEngineID ORDER BY " + "COUNT(*) DESC;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"7", "2"}, + {"1", "1"}, + {"3", "1"}, + }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); + std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_select_alias_order_by_alias_and_limit) { + const Batch batch = BuildHitsTable( + "SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 2;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"RegionID", "u"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"10", "2"}, + {"20", "2"}, + }; + std::ranges::sort(actual_rows); + std::ranges::sort(expected_rows); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_multiple_aggregates_with_alias_and_limit) { + const Batch batch = BuildHitsTable( + "SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) " + "FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 2;"); + + EXPECT_EQ(BatchColumnNames(batch), + (std::vector{"RegionID", "SUM(AdvEngineID)", "c", "AVG(ResolutionWidth)", + "COUNT(DISTINCT UserID)"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"10", "3", "2", "200", "2"}, + {"20", "14", "2", "350", "2"}, + }; + std::ranges::sort(actual_rows); + std::ranges::sort(expected_rows); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, executes_basic_aggregate_queries) { + { + const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); + } + + { + const Batch batch = BuildHitsTable("SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), + (std::vector{"SUM(AdvEngineID)", "COUNT(*)", "AVG(ResolutionWidth)"})); + EXPECT_EQ(SingleRowValues(batch), (std::vector{"18", "5", "300"})); + } + + { + const Batch batch = BuildHitsTable("SELECT AVG(UserID) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"AVG(UserID)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); + } + + { + const Batch batch = BuildHitsTable("SELECT COUNT(DISTINCT UserID) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(DISTINCT UserID)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); + } + + { + const Batch batch = BuildHitsTable("SELECT COUNT(DISTINCT SearchPhrase) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(DISTINCT SearchPhrase)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); + } + + { + const Batch batch = BuildHitsTable("SELECT MIN(EventDate), MAX(EventDate) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"MIN(EventDate)", "MAX(EventDate)"})); + EXPECT_EQ(SingleRowValues(batch), (std::vector{"2024-01-01", "2024-01-09"})); + } + + { + const Batch batch = BuildHitsTable("select sum(AdvEngineID), count(*) from hits where SearchPhrase = 'alpha';"); + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"SUM(AdvEngineID)", "COUNT(*)"})); + EXPECT_EQ(SingleRowValues(batch), (std::vector{"3", "2"})); + } +} + +TEST(executor, rejects_unknown_aggregate_functions) { + const TempFile schema_file("executor_schema"); + const TempFile data_file("executor_data"); + const TempFile columnar_file("executor_columnar"); + + WriteRows(schema_file.Path(), { + {"Value", "int64"}, + }); + WriteRows(data_file.Path(), { + {"1"}, + }); + ConvertCsvToColumnar(schema_file.Path(), data_file.Path(), columnar_file.Path(), 1); + + Executor executor; + executor.RegisterTable("hits", columnar_file.Path()); + + const ExecuteExpected result = executor.Execute("SELECT median(Value) FROM hits;"); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().GetCode(), Error::Code::Unsupported); +} diff --git a/tests/test_fileio.cpp b/tests/test_fileio.cpp new file mode 100644 index 0000000..5427f41 --- /dev/null +++ b/tests/test_fileio.cpp @@ -0,0 +1,153 @@ +#include +#include +#include +#include + +#include "support/error.h" +#include "io/fileio.h" +#include "gtest/gtest.h" +#include "testing/temp_file.h" + +TEST(fileio, write_and_read_roundtrip) { + const TempFile temp("fileio_roundtrip"); + const std::vector payload = {0, 1, 2, 3, 255}; + + WriteFileBytes(temp.Path(), payload); + const auto read_back = ReadFileBytes(temp.Path()); + + EXPECT_EQ(read_back, payload); +} + +TEST(fileio, append_bytes) { + const TempFile temp("fileio_append"); + const std::vector initial = {10, 11}; + const std::vector extra = {12, 13}; + + WriteFileBytes(temp.Path(), initial); + std::ofstream out = OpenOutputFile(temp.Path(), FileOpenMode::Append); + out.write(reinterpret_cast(extra.data()), extra.size()); + ASSERT_TRUE(out.good()); + out.flush(); + ASSERT_TRUE(out.good()); + + const auto read_back = ReadFileBytes(temp.Path()); + const std::vector expected = {10, 11, 12, 13}; + EXPECT_EQ(read_back, expected); +} + +TEST(fileio, metadata_for_file) { + const TempFile temp("fileio_meta"); + const std::vector payload = {42, 43, 44}; + + WriteFileBytes(temp.Path(), payload); + + const auto metadata = TryGetFileMetadata(temp.Path()); + ASSERT_TRUE(metadata.has_value()); + EXPECT_TRUE(metadata->is_regular); + EXPECT_FALSE(metadata->is_directory); + EXPECT_EQ(metadata->size, payload.size()); +} + +TEST(fileio, metadata_for_missing_file) { + const auto missing = UniqueTempPath("fileio_missing"); + const auto metadata = TryGetFileMetadata(missing); + + EXPECT_FALSE(metadata.has_value()); +} + +TEST(fileio, read_missing_file_throws) { + const auto missing = UniqueTempPath("fileio_missing_read"); + + EXPECT_THROW(ReadFileBytes(missing), Error); +} + +TEST(fileio, read_text_roundtrip) { + const TempFile temp("fileio_text_roundtrip"); + const std::string payload = "SELECT COUNT(*)\nFROM hits\nWHERE UserID = 42;\n"; + + std::ofstream out = OpenOutputFile(temp.Path()); + out << payload; + ASSERT_TRUE(out.good()); + out.flush(); + ASSERT_TRUE(out.good()); + + EXPECT_EQ(ReadTextFile(temp.Path()), payload); +} + +TEST(fileio, file_exists_checks) { + const TempFile temp("fileio_exists"); + EXPECT_FALSE(FileExists(temp.Path())); + + WriteFileBytes(temp.Path(), std::vector{1}); + + EXPECT_TRUE(FileExists(temp.Path())); +} + +TEST(fileio, ensure_parent_directory_creates_missing_directories) { + const auto file_path = UniqueTempPath("fileio_nested") / "a" / "b" / "query.sql"; + const auto parent = file_path.parent_path(); + + ASSERT_FALSE(std::filesystem::exists(parent)); + EnsureParentDirectory(file_path); + EXPECT_TRUE(std::filesystem::exists(parent)); + EXPECT_TRUE(std::filesystem::is_directory(parent)); +} + +TEST(fileio, stream_roundtrip) { + const TempFile temp("fileio_stream_roundtrip"); + const std::vector payload = {7, 8, 9, 10, 11}; + + std::ofstream out = OpenOutputFile(temp.Path()); + out.write(reinterpret_cast(payload.data()), payload.size()); + ASSERT_TRUE(out.good()); + out.flush(); + ASSERT_TRUE(out.good()); + + std::ifstream in = OpenInputFile(temp.Path()); + std::vector read_back(payload.size()); + in.read(reinterpret_cast(read_back.data()), read_back.size()); + ASSERT_EQ(static_cast(in.gcount()), read_back.size()); + EXPECT_EQ(read_back, payload); +} + +TEST(fileio, stream_append) { + const TempFile temp("fileio_stream_append"); + const std::vector initial = {1, 2}; + const std::vector extra = {3, 4, 5}; + + WriteFileBytes(temp.Path(), initial); + + std::ofstream out = OpenOutputFile(temp.Path(), FileOpenMode::Append); + out.write(reinterpret_cast(extra.data()), extra.size()); + ASSERT_TRUE(out.good()); + out.flush(); + ASSERT_TRUE(out.good()); + + const auto read_back = ReadFileBytes(temp.Path()); + const std::vector expected = {1, 2, 3, 4, 5}; + EXPECT_EQ(read_back, expected); +} + +TEST(fileio, stream_seek_and_tell) { + const TempFile temp("fileio_stream_seek"); + const std::vector payload = {100, 101, 102, 103}; + + WriteFileBytes(temp.Path(), payload); + + std::ifstream in = OpenInputFile(temp.Path()); + EXPECT_EQ(in.tellg(), std::streampos(0)); + in.seekg(std::streampos(2)); + ASSERT_TRUE(in.good()); + EXPECT_EQ(in.tellg(), std::streampos(2)); + + uint8_t value = 0; + in.read(reinterpret_cast(&value), 1); + ASSERT_EQ(in.gcount(), 1); + EXPECT_EQ(value, payload[2]); +} + +TEST(fileio, stream_open_missing_throws) { + const auto missing = UniqueTempPath("fileio_stream_missing"); + + EXPECT_THROW({ std::ifstream in = OpenInputFile(missing); }, Error); +} diff --git a/tests/test_stream.cpp b/tests/test_stream.cpp new file mode 100644 index 0000000..89a7d80 --- /dev/null +++ b/tests/test_stream.cpp @@ -0,0 +1,20 @@ +#include +#include +#include + +#include "gtest/gtest.h" +#include "io/stream.h" + +TEST(stream, roundtrip_signed_integrals) { + std::stringstream buffer; + + WriteStream(buffer, std::numeric_limits::min()); + WriteStream(buffer, -1); + WriteStream(buffer, 0); + WriteStream(buffer, std::numeric_limits::max()); + + EXPECT_EQ(ReadStream(buffer), std::numeric_limits::min()); + EXPECT_EQ(ReadStream(buffer), -1); + EXPECT_EQ(ReadStream(buffer), 0); + EXPECT_EQ(ReadStream(buffer), std::numeric_limits::max()); +} diff --git a/tests/test_tokenizer.cpp b/tests/test_tokenizer.cpp new file mode 100644 index 0000000..ca11a74 --- /dev/null +++ b/tests/test_tokenizer.cpp @@ -0,0 +1,117 @@ +#include +#include +#include + +#include "gtest/gtest.h" +#include "sql_parser/tokenizer.h" + +static std::vector CollectTypes(const std::vector& tokens) { + std::vector types; + types.reserve(tokens.size()); + + for (const TokenPtr& token : tokens) { + types.push_back(token->GetType()); + } + + return types; +} + +TEST(sql_tokenizer, tokenizes_select_query) { + const auto result = + TokenizeSql("SELECT DISTINCT name, COUNT(*), LENGTH(city) FROM sales GROUP BY name ORDER BY name LIMIT 10;"); + + ASSERT_TRUE(result.has_value()); + const auto& tokens = result.value(); + + EXPECT_EQ( + CollectTypes(tokens), + (std::vector{ + Tokens::Select, Tokens::Distinct, Tokens::NameToken, Tokens::Comma, Tokens::Count, + Tokens::OpenBracket, Tokens::Asterisk, Tokens::CloseBracket, Tokens::Comma, Tokens::Length, + Tokens::OpenBracket, Tokens::NameToken, Tokens::CloseBracket, Tokens::From, Tokens::NameToken, + Tokens::Group, Tokens::By, Tokens::NameToken, Tokens::Order, Tokens::By, + Tokens::NameToken, Tokens::Limit, Tokens::NumericLiteral, Tokens::Semicolon, Tokens::EndOfInput, + })); + + ASSERT_EQ(tokens.size(), 25U); + EXPECT_NE(std::dynamic_pointer_cast(tokens[0]), nullptr); + EXPECT_NE(std::dynamic_pointer_cast(tokens[1]), nullptr); + EXPECT_NE(std::dynamic_pointer_cast(tokens[2]), nullptr); + EXPECT_EQ(tokens[2]->GetText(), "name"); + EXPECT_EQ(tokens[14]->GetText(), "sales"); + EXPECT_EQ(tokens[22]->GetText(), "10"); + EXPECT_EQ(tokens[24]->GetOffset(), std::string_view("SELECT DISTINCT name, COUNT(*), LENGTH(city) FROM sales " + "GROUP BY name ORDER BY name LIMIT 10;") + .size()); +} + +TEST(sql_tokenizer, tokenizes_create_with_literals_and_case_insensitive_keywords) { + const auto result = TokenizeSql("create report as SeLeCt 'abc''def', total_sum - 10.5 FROM source_table"); + + ASSERT_TRUE(result.has_value()); + const auto& tokens = result.value(); + + EXPECT_EQ(CollectTypes(tokens), (std::vector{ + Tokens::Create, + Tokens::NameToken, + Tokens::As, + Tokens::Select, + Tokens::StringLiteral, + Tokens::Comma, + Tokens::NameToken, + Tokens::Minus, + Tokens::NumericLiteral, + Tokens::From, + Tokens::NameToken, + Tokens::EndOfInput, + })); + + ASSERT_EQ(tokens.size(), 12U); + EXPECT_NE(std::dynamic_pointer_cast(tokens[0]), nullptr); + EXPECT_NE(std::dynamic_pointer_cast(tokens[4]), nullptr); + EXPECT_EQ(tokens[1]->GetText(), "report"); + EXPECT_EQ(tokens[4]->GetText(), "'abc''def'"); + EXPECT_EQ(tokens[8]->GetText(), "10.5"); + EXPECT_EQ(tokens[10]->GetText(), "source_table"); +} + +TEST(sql_tokenizer, tokenizer_returns_tokens_one_by_one) { + Tokenizer tokenizer("WHERE value >= 10 AND value <> 20 AND value != 30"); + + std::vector types; + while (true) { + auto token = tokenizer.GetNext(); + ASSERT_TRUE(token.has_value()); + types.push_back(token.value()->GetType()); + + if (token.value()->GetType() == Tokens::EndOfInput) { + break; + } + } + + EXPECT_EQ(types, (std::vector{ + Tokens::Where, + Tokens::NameToken, + Tokens::GreaterOrEqual, + Tokens::NumericLiteral, + Tokens::And, + Tokens::NameToken, + Tokens::NotEqual, + Tokens::NumericLiteral, + Tokens::And, + Tokens::NameToken, + Tokens::NotEqual, + Tokens::NumericLiteral, + Tokens::EndOfInput, + })); +} + +TEST(sql_tokenizer, rejects_invalid_input) { + const auto unterminated = TokenizeSql("SELECT 'oops"); + ASSERT_FALSE(unterminated.has_value()); + EXPECT_EQ(unterminated.error().GetCode(), Error::Code::InvalidData); + + const auto invalid_character = TokenizeSql("SELECT @oops"); + ASSERT_FALSE(invalid_character.has_value()); + EXPECT_EQ(invalid_character.error().GetCode(), Error::Code::InvalidData); +} From 8ba8412efbcd83dde782e4e8f481443e95affbd1 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Tue, 5 May 2026 23:32:45 +0300 Subject: [PATCH 14/29] chore: create trivial review baseline --- .clang-format | 10 - .clang-tidy | 41 - .coderabbit.yaml | 30 - .github/apt-packages.txt | 7 - .github/workflows/cmake-single-platform.yml | 72 - .gitignore | 111 - CMakeLists.txt | 84 - README.md | 3 - benchmarks/CMakeLists.txt | 34 - benchmarks/benchmark_csv_to_columnar.cpp | 121 - benchmarks/benchmark_queries.cpp | 50 - benchmarks/queries/README.md | 1 - benchmarks/queries/query_0.sql | 1 - benchmarks/queries/query_1.sql | 1 - benchmarks/queries/query_2.sql | 1 - benchmarks/queries/query_3.sql | 1 - benchmarks/queries/query_4.sql | 1 - cmake/dependencies_package.cmake | 14 - cmake/sanitizers.cmake | 59 - include/convert/csv_columnar.h | 11 - include/executor/aggregate_function.h | 45 - include/executor/executor.h | 120 - include/io/batch_io.h | 42 - include/io/columnar_batch_io.h | 62 - include/io/csv.h | 55 - include/io/csv_batch_io.h | 54 - include/io/fileio.h | 35 - include/io/stream.h | 31 - include/model/batch.h | 41 - include/model/column.h | 44 - include/model/column_boolean.h | 15 - include/model/column_character.h | 15 - include/model/column_date.h | 15 - include/model/column_int128.h | 16 - include/model/column_int16.h | 15 - include/model/column_int32.h | 15 - include/model/column_int64.h | 15 - include/model/column_string.h | 15 - include/model/column_timestamp.h | 15 - include/model/fixed_column.h | 58 - include/model/metadata.h | 27 - include/model/schema.h | 37 - include/model/variant_column.h | 94 - include/sql_parser/tokenizer.h | 322 --- include/support/ascii.h | 7 - include/support/error.h | 72 - include/support/int128.h | 24 - include/support/parsing.h | 27 - include/testing/executor_test_utils.h | 33 - include/testing/temp_file.h | 52 - include/tl/expected.hpp | 2475 ------------------- script/build.sh | 25 - script/convert.sh | 36 - script/run_query.sh | 62 - script/setup.sh | 37 - scripts/Dockerfile | 13 - scripts/action.py | 258 -- scripts/commands.sh | 48 - scripts/env.sh | 13 - scripts/install_dependencies.sh | 26 - scripts/sample_input.columnar | Bin 154 -> 0 bytes scripts/sample_input.csv | 3 - scripts/sample_output.csv | 3 - scripts/schema_input.csv | 3 - src/CMakeLists.txt | 35 - src/app/main.cpp | 138 -- src/convert/csv_columnar.cpp | 42 - src/executor/aggregate_states.cpp | 406 --- src/executor/executor.cpp | 27 - src/executor/operators.cpp | 217 -- src/executor/query_parser.cpp | 175 -- src/executor/query_planner.cpp | 118 - src/io/batch_io.cpp | 328 --- src/io/csv.cpp | 196 -- src/io/fileio.cpp | 126 - src/io/stream.cpp | 23 - src/model/batch.cpp | 86 - src/model/column.cpp | 46 - src/model/column_boolean.cpp | 7 - src/model/column_character.cpp | 7 - src/model/column_date.cpp | 7 - src/model/column_int128.cpp | 7 - src/model/column_int16.cpp | 7 - src/model/column_int32.cpp | 7 - src/model/column_int64.cpp | 7 - src/model/column_string.cpp | 5 - src/model/column_timestamp.cpp | 7 - src/model/metadata.cpp | 100 - src/model/schema.cpp | 138 -- src/sql_parser/tokenizer.cpp | 174 -- src/sql_parser/tokenizer_factory.cpp | 130 - src/sql_parser/tokenizer_tokens.cpp | 133 - src/support/ascii.cpp | 21 - src/support/error.cpp | 107 - src/support/parsing.cpp | 315 --- tests/CMakeLists.txt | 16 - tests/test_batch.cpp | 245 -- tests/test_columnar.cpp | 197 -- tests/test_columns.cpp | 138 -- tests/test_csv.cpp | 65 - tests/test_executor.cpp | 114 - tests/test_fileio.cpp | 153 -- tests/test_stream.cpp | 20 - tests/test_tokenizer.cpp | 117 - 104 files changed, 9280 deletions(-) delete mode 100644 .clang-format delete mode 100644 .clang-tidy delete mode 100644 .coderabbit.yaml delete mode 100644 .github/apt-packages.txt delete mode 100644 .github/workflows/cmake-single-platform.yml delete mode 100644 .gitignore delete mode 100644 CMakeLists.txt delete mode 100644 README.md delete mode 100644 benchmarks/CMakeLists.txt delete mode 100644 benchmarks/benchmark_csv_to_columnar.cpp delete mode 100644 benchmarks/benchmark_queries.cpp delete mode 100644 benchmarks/queries/README.md delete mode 100644 benchmarks/queries/query_0.sql delete mode 100644 benchmarks/queries/query_1.sql delete mode 100644 benchmarks/queries/query_2.sql delete mode 100644 benchmarks/queries/query_3.sql delete mode 100644 benchmarks/queries/query_4.sql delete mode 100644 cmake/dependencies_package.cmake delete mode 100644 cmake/sanitizers.cmake delete mode 100644 include/convert/csv_columnar.h delete mode 100644 include/executor/aggregate_function.h delete mode 100644 include/executor/executor.h delete mode 100644 include/io/batch_io.h delete mode 100644 include/io/columnar_batch_io.h delete mode 100644 include/io/csv.h delete mode 100644 include/io/csv_batch_io.h delete mode 100644 include/io/fileio.h delete mode 100644 include/io/stream.h delete mode 100644 include/model/batch.h delete mode 100644 include/model/column.h delete mode 100644 include/model/column_boolean.h delete mode 100644 include/model/column_character.h delete mode 100644 include/model/column_date.h delete mode 100644 include/model/column_int128.h delete mode 100644 include/model/column_int16.h delete mode 100644 include/model/column_int32.h delete mode 100644 include/model/column_int64.h delete mode 100644 include/model/column_string.h delete mode 100644 include/model/column_timestamp.h delete mode 100644 include/model/fixed_column.h delete mode 100644 include/model/metadata.h delete mode 100644 include/model/schema.h delete mode 100644 include/model/variant_column.h delete mode 100644 include/sql_parser/tokenizer.h delete mode 100644 include/support/ascii.h delete mode 100644 include/support/error.h delete mode 100644 include/support/int128.h delete mode 100644 include/support/parsing.h delete mode 100644 include/testing/executor_test_utils.h delete mode 100644 include/testing/temp_file.h delete mode 100644 include/tl/expected.hpp delete mode 100755 script/build.sh delete mode 100755 script/convert.sh delete mode 100755 script/run_query.sh delete mode 100755 script/setup.sh delete mode 100644 scripts/Dockerfile delete mode 100644 scripts/action.py delete mode 100755 scripts/commands.sh delete mode 100644 scripts/env.sh delete mode 100755 scripts/install_dependencies.sh delete mode 100644 scripts/sample_input.columnar delete mode 100644 scripts/sample_input.csv delete mode 100644 scripts/sample_output.csv delete mode 100644 scripts/schema_input.csv delete mode 100644 src/CMakeLists.txt delete mode 100644 src/app/main.cpp delete mode 100644 src/convert/csv_columnar.cpp delete mode 100644 src/executor/aggregate_states.cpp delete mode 100644 src/executor/executor.cpp delete mode 100644 src/executor/operators.cpp delete mode 100644 src/executor/query_parser.cpp delete mode 100644 src/executor/query_planner.cpp delete mode 100644 src/io/batch_io.cpp delete mode 100644 src/io/csv.cpp delete mode 100644 src/io/fileio.cpp delete mode 100644 src/io/stream.cpp delete mode 100644 src/model/batch.cpp delete mode 100644 src/model/column.cpp delete mode 100644 src/model/column_boolean.cpp delete mode 100644 src/model/column_character.cpp delete mode 100644 src/model/column_date.cpp delete mode 100644 src/model/column_int128.cpp delete mode 100644 src/model/column_int16.cpp delete mode 100644 src/model/column_int32.cpp delete mode 100644 src/model/column_int64.cpp delete mode 100644 src/model/column_string.cpp delete mode 100644 src/model/column_timestamp.cpp delete mode 100644 src/model/metadata.cpp delete mode 100644 src/model/schema.cpp delete mode 100644 src/sql_parser/tokenizer.cpp delete mode 100644 src/sql_parser/tokenizer_factory.cpp delete mode 100644 src/sql_parser/tokenizer_tokens.cpp delete mode 100644 src/support/ascii.cpp delete mode 100644 src/support/error.cpp delete mode 100644 src/support/parsing.cpp delete mode 100644 tests/CMakeLists.txt delete mode 100644 tests/test_batch.cpp delete mode 100644 tests/test_columnar.cpp delete mode 100644 tests/test_columns.cpp delete mode 100644 tests/test_csv.cpp delete mode 100644 tests/test_executor.cpp delete mode 100644 tests/test_fileio.cpp delete mode 100644 tests/test_stream.cpp delete mode 100644 tests/test_tokenizer.cpp diff --git a/.clang-format b/.clang-format deleted file mode 100644 index 46280a9..0000000 --- a/.clang-format +++ /dev/null @@ -1,10 +0,0 @@ -BasedOnStyle: Google -IndentWidth: 4 -AccessModifierOffset: -1 -ColumnLimit: 120 -AllowShortFunctionsOnASingleLine: All -AllowShortIfStatementsOnASingleLine: false -AllowShortLoopsOnASingleLine: false -DerivePointerAlignment: true -KeepEmptyLinesAtTheStartOfBlocks: true -SortIncludes: true \ No newline at end of file diff --git a/.clang-tidy b/.clang-tidy deleted file mode 100644 index 3d705b4..0000000 --- a/.clang-tidy +++ /dev/null @@ -1,41 +0,0 @@ ---- -Checks: '-*,cppcoreguidelines-avoid-goto,cppcoreguidelines-pro-type-const-cast, google-readability-casting, modernize-replace-random-shuffle, readability-braces-around-statements, readability-container-size-empty, readability-redundant-control-flow, readability-redundant-string-init, modernize-use-nullptr, readability-identifier-naming, google-build-using-namespace' -HeaderFilterRegex: '\.h$' -WarningsAsErrors: '*' -CheckOptions: - - key: readability-identifier-naming.NamespaceCase - value: lower_case - - key: readability-identifier-naming.ClassCase - value: CamelCase - - key: readability-identifier-naming.TypedefCase - value: CamelCase - - key: readability-identifier-naming.TypedefIgnoredRegexp - value: (allocator_type|size_type|key_type|value_type|mapped_type|difference_type|pointer|const_pointer|reference|const_reference|iterator_category|const_iterator|iterator|reverse_iterator|const_reverse_iterator|key_compare) - - key: readability-identifier-naming.TypeAliasCase - value: CamelCase - - key: readability-identifier-naming.PrivateMemberSuffix - value: '_' - - key: readability-identifier-naming.StructCase - value: CamelCase - - key: readability-identifier-naming.FunctionCase - value: CamelCase - - key: readability-identifier-naming.VariableCase - value: lower_case - - key: readability-identifier-naming.PrivateMemberCase - value: lower_case - - key: readability-identifier-naming.ParameterCase - value: lower_case - - key: readability-identifier-naming.GlobalConstantPrefix - value: k - - key: readability-identifier-naming.GlobalConstantCase - value: CamelCase - - key: readability-identifier-naming.StaticConstantPrefix - value: k - - key: readability-identifier-naming.StaticConstantCase - value: CamelCase - - key: readability-identifier-naming.ConstexprVariableCase - value: CamelCase - - key: readability-identifier-naming.ConstexprVariablePrefix - value: k - - key: google-runtime-int.TypeSuffix - value: _t \ No newline at end of file diff --git a/.coderabbit.yaml b/.coderabbit.yaml deleted file mode 100644 index a541175..0000000 --- a/.coderabbit.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json -language: "ru-RU" -early_access: false - -reviews: - auto_review: - enabled: true - drafts: false - - path_filters: - - "src/**" - - "include/**" - - "CMakeLists.txt" - - "**/*.cmake" - - ".github/workflows/**" - - "!build/**" - - path_instructions: - - path: "**/*.{h,hpp,cpp,cc,cxx}" - instructions: | - Проверь: UB/границы/инициализация, RAII, обработку ошибок, - исключения, const-correctness, сложность, читаемость. - Если меняется логика — попроси тесты. - - path: "**/*CMakeLists.txt" - instructions: | - Проверь корректность target_* команд, include dirs, линковку, - и что сборка проходит на чистой машине. - -chat: - auto_reply: true diff --git a/.github/apt-packages.txt b/.github/apt-packages.txt deleted file mode 100644 index cad3f63..0000000 --- a/.github/apt-packages.txt +++ /dev/null @@ -1,7 +0,0 @@ -ninja-build -cmake -libgtest-dev -clang-18 -libclang-rt-18-dev -clang-format -clang-tidy diff --git a/.github/workflows/cmake-single-platform.yml b/.github/workflows/cmake-single-platform.yml deleted file mode 100644 index 5826612..0000000 --- a/.github/workflows/cmake-single-platform.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Build and test - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Fetch dependencies - shell: bash - run: scripts/install_dependencies.sh - - - name: Configure CMake - run: | - mkdir -p build - cd build - cmake -GNinja -DCMAKE_BUILD_TYPE=Debug .. - - - name: Build - run: | - cd build - ninja - - - name: Test - run: | - cd build - ./tests/test_batch - ./tests/test_columnar - ./tests/test_columns - ./tests/test_csv - ./tests/test_fileio - ./tests/test_stream - ./tests/test_tokenizer - - sanitizers: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Fetch dependencies - shell: bash - run: scripts/install_dependencies.sh - - - name: Configure CMake With Sanitizers - env: - CC: clang-18 - CXX: clang++-18 - run: | - mkdir -p build-sanitizers - cd build-sanitizers - cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DCOLUMNAR_SANITIZERS=address,undefined .. - - - name: Build With Sanitizers - run: | - cd build-sanitizers - ninja - - - name: Test With Sanitizers - env: - ASAN_OPTIONS: detect_leaks=1:strict_string_checks=1:check_initialization_order=1 - UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 - run: | - cd build-sanitizers - ctest --output-on-failure diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 3eef705..0000000 --- a/.gitignore +++ /dev/null @@ -1,111 +0,0 @@ -# Prerequisites -*.d - -# Compiled Object files -*.slo -*.lo -*.o -*.obj - -# Precompiled Headers -*.gch -*.pch - -# Linker files -*.ilk - -# Debugger Files -*.pdb - -# Compiled Dynamic libraries -*.so -*.dylib -*.dll - -# Fortran module files -*.mod -*.smod - -# Compiled Static libraries -*.lai -*.la -*.a -*.lib - -# Executables -*.exe -*.out -*.app - -# Debug information files -*.dwo - -.DS_Store - - -# CMake -.cmake/ -CMakeFiles/ -CMakeCache.txt -cmake_install.cmake -CTestTestfile.cmake -CPackConfig.cmake -CPackSourceConfig.cmake -Makefile -Testing/Temporary - -# Ninja -.ninja_deps -.ninja_log -build.ninja - -# Build -build -build-release -build-sanitizers -build-argparse -build-clean -build-tests -compile_commands.json -src/*.cmake -src/columnar_engine -_deps -.deps - -# Tests -scripts/__pycache__ -tests/*.cmake -tests/test_fileio -tests/test_foo -tests/test_columnar -tests/test_csv -tests/test_batch -tests/test_columns -tests/test_stream -tests/test_executor -tests/test_tokenizer - -# Examples -examples/*.cmake -examples/example_fileio -examples/example_columnar -examples/example_csv -examples/example_batch -examples/example_columns - -examples/out_batch -examples/out_columnar - -# IDE -.vscode -.idea -.venv - -# benchmarks -benchmarks/hits_sample.csv -benchmarks/hits_sample_new.csv -benchmarks/schema_sample.csv -benchmarks/schema_sample_new.csv -benchmarks/hits_sample.columnar -benchmarks/benchmark_csv_to_columnar -benchmarks/benchmark_queries diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index aeeaf52..0000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,84 +0,0 @@ -cmake_minimum_required(VERSION 3.25) - -project(columnar-engine) - -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -set(CMAKE_CXX_STANDARD 23) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set_property(GLOBAL PROPERTY USE_FOLDERS ON) - -option(ENABLE_TESTS "Build tests" ON) -option(ENABLE_BENCHMARKS "Build benchmarks" ON) - -include(cmake/dependencies_package.cmake) -include(cmake/sanitizers.cmake) - -set(COLUMNAR_INCLUDE_DIRS - ${PROJECT_SOURCE_DIR}/include/convert - ${PROJECT_SOURCE_DIR}/include/io - ${PROJECT_SOURCE_DIR}/include/model - ${PROJECT_SOURCE_DIR}/include/support - ${PROJECT_SOURCE_DIR}/include/testing - ${PROJECT_SOURCE_DIR}/include/executor - ${PROJECT_SOURCE_DIR}/include/sql_parser - ${PROJECT_SOURCE_DIR}/include/tl -) - -include_directories(${COLUMNAR_INCLUDE_DIRS}) - -add_compile_options(-Wall -Wextra -Werror -Wpedantic -O2) - -set(COLUMNAR_SANITIZERS "" CACHE STRING "Comma or semicolon separated list of sanitizers: address,undefined,leak,thread,memory") -columnar_enable_sanitizers("${COLUMNAR_SANITIZERS}") - -set(COLUMNAR_SANITIZER_TARGET_BUILD_DIR "${PROJECT_SOURCE_DIR}/build-sanitizers" CACHE PATH - "Build directory used by the test_sanitizers target") -set(COLUMNAR_SANITIZER_TARGET_SANITIZERS "address,undefined" CACHE STRING - "Sanitizers used by the test_sanitizers target") - -add_subdirectory(src) - -if (ENABLE_BENCHMARKS) - add_subdirectory(benchmarks) -endif () - -if (ENABLE_TESTS) - enable_testing() - add_subdirectory(tests) -endif () - -set(columnar_sanitizer_configure_args - -S ${PROJECT_SOURCE_DIR} - -B ${COLUMNAR_SANITIZER_TARGET_BUILD_DIR} - -G ${CMAKE_GENERATOR} - -DENABLE_TESTS=ON - -DCOLUMNAR_SANITIZERS=${COLUMNAR_SANITIZER_TARGET_SANITIZERS} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -) - -if (CMAKE_GENERATOR_PLATFORM) - list(APPEND columnar_sanitizer_configure_args -A ${CMAKE_GENERATOR_PLATFORM}) -endif () - -if (CMAKE_GENERATOR_TOOLSET) - list(APPEND columnar_sanitizer_configure_args -T ${CMAKE_GENERATOR_TOOLSET}) -endif () - -if (CMAKE_CONFIGURATION_TYPES) - add_custom_target(test_sanitizers - COMMAND ${CMAKE_COMMAND} ${columnar_sanitizer_configure_args} - COMMAND ${CMAKE_COMMAND} --build ${COLUMNAR_SANITIZER_TARGET_BUILD_DIR} --config Debug - COMMAND ${CMAKE_CTEST_COMMAND} --test-dir ${COLUMNAR_SANITIZER_TARGET_BUILD_DIR} -C Debug --output-on-failure - USES_TERMINAL - COMMENT "Configure, build and run tests with sanitizers" - ) -else () - add_custom_target(test_sanitizers - COMMAND ${CMAKE_COMMAND} ${columnar_sanitizer_configure_args} -DCMAKE_BUILD_TYPE=Debug - COMMAND ${CMAKE_COMMAND} --build ${COLUMNAR_SANITIZER_TARGET_BUILD_DIR} - COMMAND ${CMAKE_CTEST_COMMAND} --test-dir ${COLUMNAR_SANITIZER_TARGET_BUILD_DIR} --output-on-failure - USES_TERMINAL - COMMENT "Configure, build and run tests with sanitizers" - ) -endif () diff --git a/README.md b/README.md deleted file mode 100644 index 94cd7d0..0000000 --- a/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# columnar-engine - -Колононый движок с нуля (in progress) \ No newline at end of file diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt deleted file mode 100644 index 15f0bdc..0000000 --- a/benchmarks/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -add_executable(benchmark_csv_to_columnar - benchmark_csv_to_columnar.cpp -) - -target_link_libraries(benchmark_csv_to_columnar - PRIVATE - columnar_engine_lib -) - -add_executable(benchmark_queries - benchmark_queries.cpp -) - -target_link_libraries(benchmark_queries - PRIVATE - columnar_engine_lib -) - -target_compile_definitions(benchmark_csv_to_columnar - PRIVATE - COLUMNAR_BENCHMARK_DEFAULT_DATA="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample.csv" - COLUMNAR_BENCHMARK_DEFAULT_SCHEMA="${PROJECT_SOURCE_DIR}/benchmarks/schema_sample.csv" - COLUMNAR_BENCHMARK_DEFAULT_OUTPUT="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample.columnar" - COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_DATA="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample_new.csv" - COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA="${PROJECT_SOURCE_DIR}/benchmarks/schema_sample_new.csv" -) - -target_compile_definitions(benchmark_queries - PRIVATE - COLUMNAR_BENCHMARK_QUERIES_DIR="${PROJECT_SOURCE_DIR}/benchmarks/queries" -) - -set_target_properties(benchmark_csv_to_columnar PROPERTIES FOLDER "benchmarks") -set_target_properties(benchmark_queries PROPERTIES FOLDER "benchmarks") diff --git a/benchmarks/benchmark_csv_to_columnar.cpp b/benchmarks/benchmark_csv_to_columnar.cpp deleted file mode 100644 index 77632ba..0000000 --- a/benchmarks/benchmark_csv_to_columnar.cpp +++ /dev/null @@ -1,121 +0,0 @@ -#include -#include -#include -#include -#include - -#include "csv.h" -#include "csv_columnar.h" -#include "error.h" - -#ifndef COLUMNAR_BENCHMARK_DEFAULT_DATA -#define COLUMNAR_BENCHMARK_DEFAULT_DATA "benchmarks/hits_sample.csv" -#endif - -#ifndef COLUMNAR_BENCHMARK_DEFAULT_SCHEMA -#define COLUMNAR_BENCHMARK_DEFAULT_SCHEMA "benchmarks/schema_sample.csv" -#endif - -#ifndef COLUMNAR_BENCHMARK_DEFAULT_OUTPUT -#define COLUMNAR_BENCHMARK_DEFAULT_OUTPUT "benchmarks/hits_sample.columnar" -#endif - -#ifndef COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_DATA -#define COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_DATA "benchmarks/hits_sample_new.csv" -#endif - -#ifndef COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA -#define COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA "benchmarks/schema_sample_new.csv" -#endif - -static constexpr size_t kRowsPerGroup = 50000; - -struct CsvCompareResult { - bool equal = true; - size_t row = 0; -}; - -static CsvCompareResult CompareCsvFiles(const std::filesystem::path& lhs_path, const std::filesystem::path& rhs_path) { - CsvReader lhs(lhs_path); - CsvReader rhs(rhs_path); - - std::vector lhs_row; - std::vector rhs_row; - - size_t row = 0; - - while (true) { - const bool lhs_has_row = lhs.ReadRow(lhs_row); - const bool rhs_has_row = rhs.ReadRow(rhs_row); - - if (lhs_has_row != rhs_has_row) { - return CsvCompareResult{false, row}; - } - if (!lhs_has_row) { - return CsvCompareResult{true, row}; - } - if (lhs_row != rhs_row) { - return CsvCompareResult{false, row}; - } - - ++row; - } -} - -int main(const int argc, char** argv) { - try { - size_t rows_per_group = kRowsPerGroup; - if (argc == 2) { - rows_per_group = std::stoull(argv[1]); - } - - const std::filesystem::path schema_path = COLUMNAR_BENCHMARK_DEFAULT_SCHEMA; - const std::filesystem::path data_path = COLUMNAR_BENCHMARK_DEFAULT_DATA; - const std::filesystem::path output_path = COLUMNAR_BENCHMARK_DEFAULT_OUTPUT; - const std::filesystem::path roundtrip_schema_path = COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA; - const std::filesystem::path roundtrip_data_path = COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_DATA; - - const auto csv_to_columnar_started_at = std::chrono::steady_clock::now(); - ConvertCsvToColumnar(schema_path, data_path, output_path, rows_per_group); - const auto csv_to_columnar_finished_at = std::chrono::steady_clock::now(); - - const auto columnar_to_csv_started_at = std::chrono::steady_clock::now(); - ConvertColumnarToCsv(output_path, roundtrip_schema_path, roundtrip_data_path); - const auto columnar_to_csv_finished_at = std::chrono::steady_clock::now(); - - const auto compare_started_at = std::chrono::steady_clock::now(); - const CsvCompareResult schema_compare = CompareCsvFiles(schema_path, roundtrip_schema_path); - const CsvCompareResult data_compare = CompareCsvFiles(data_path, roundtrip_data_path); - const auto compare_finished_at = std::chrono::steady_clock::now(); - - const double csv_to_columnar_seconds = - std::chrono::duration(csv_to_columnar_finished_at - csv_to_columnar_started_at).count(); - const double columnar_to_csv_seconds = - std::chrono::duration(columnar_to_csv_finished_at - columnar_to_csv_started_at).count(); - const double compare_seconds = std::chrono::duration(compare_finished_at - compare_started_at).count(); - - std::cout << "schema: " << schema_path << std::endl - << "data: " << data_path << std::endl - << "columnar_output: " << output_path << std::endl - << "roundtrip_schema: " << roundtrip_schema_path << std::endl - << "roundtrip_data: " << roundtrip_data_path << std::endl - << "rows_per_group: " << rows_per_group << std::endl - << "csv_to_columnar_elapsed_seconds: " << csv_to_columnar_seconds << std::endl - << "columnar_to_csv_elapsed_seconds: " << columnar_to_csv_seconds << std::endl - << "csv_compare_elapsed_seconds: " << compare_seconds << std::endl - << "schema_matches_source: " << (schema_compare.equal ? "true" : "false") << std::endl - << "data_matches_source: " << (data_compare.equal ? "true" : "false") << std::endl; - - if (!schema_compare.equal) { - std::cout << "schema_first_mismatch_row: " << schema_compare.row << std::endl; - } - if (!data_compare.equal) { - std::cout << "data_first_mismatch_row: " << data_compare.row << std::endl; - } - - return schema_compare.equal && data_compare.equal ? 0 : 2; - } catch (const std::exception& ex) { - std::cerr << ex.what() << std::endl; - return 1; - } -} diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp deleted file mode 100644 index 90e8a79..0000000 --- a/benchmarks/benchmark_queries.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include -#include -#include - -#include "fileio.h" - -#ifndef COLUMNAR_BENCHMARK_QUERIES_DIR -#define COLUMNAR_BENCHMARK_QUERIES_DIR "benchmarks/queries" -#endif - -static constexpr int kFirstQueryId = 0; -static constexpr int kLastQueryId = 42; - -void RunQuerySkeleton(const std::filesystem::path& path, const int query_id) { - const std::string sql = ReadTextFile(path); - std::cout << "query_" << query_id << ": found, bytes=" << sql.size() << ", status=skeleton_only" << std::endl; -} - -int main() { - try { - const std::filesystem::path queries_dir = COLUMNAR_BENCHMARK_QUERIES_DIR; - - size_t found_queries = 0; - size_t missing_queries = 0; - - std::cout << "queries_dir=" << queries_dir << std::endl; - - for (int query_id = kFirstQueryId; query_id <= kLastQueryId; ++query_id) { - const std::filesystem::path query_path = queries_dir / ("query_" + std::to_string(query_id) + ".sql"); - - if (!std::filesystem::exists(query_path)) { - ++missing_queries; - std::cout << "query_" << query_id << ": missing, status=skipped" << std::endl; - continue; - } - - ++found_queries; - - RunQuerySkeleton(query_path, query_id); - } - - std::cout << "found_queries=" << found_queries << std::endl; - std::cout << "missing_queries=" << missing_queries << std::endl; - } catch (const std::exception& ex) { - std::cerr << ex.what() << std::endl; - return 1; - } - - return 0; -} diff --git a/benchmarks/queries/README.md b/benchmarks/queries/README.md deleted file mode 100644 index bc42131..0000000 --- a/benchmarks/queries/README.md +++ /dev/null @@ -1 +0,0 @@ -`columnar_engine run-query --input --query-file --output ` diff --git a/benchmarks/queries/query_0.sql b/benchmarks/queries/query_0.sql deleted file mode 100644 index c70aa7a..0000000 --- a/benchmarks/queries/query_0.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT COUNT(*) FROM hits; diff --git a/benchmarks/queries/query_1.sql b/benchmarks/queries/query_1.sql deleted file mode 100644 index ad8031a..0000000 --- a/benchmarks/queries/query_1.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; diff --git a/benchmarks/queries/query_2.sql b/benchmarks/queries/query_2.sql deleted file mode 100644 index 7db4dc2..0000000 --- a/benchmarks/queries/query_2.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; diff --git a/benchmarks/queries/query_3.sql b/benchmarks/queries/query_3.sql deleted file mode 100644 index e953498..0000000 --- a/benchmarks/queries/query_3.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT AVG(UserID) FROM hits; diff --git a/benchmarks/queries/query_4.sql b/benchmarks/queries/query_4.sql deleted file mode 100644 index 2f9baf8..0000000 --- a/benchmarks/queries/query_4.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT COUNT(DISTINCT UserID) FROM hits; diff --git a/cmake/dependencies_package.cmake b/cmake/dependencies_package.cmake deleted file mode 100644 index 3a0c86d..0000000 --- a/cmake/dependencies_package.cmake +++ /dev/null @@ -1,14 +0,0 @@ -if (ENABLE_TESTS) - find_package(GTest QUIET) - if (NOT GTest_FOUND) - message(WARNING "GTest not found; tests will be disabled for this configure run.") - set(ENABLE_TESTS OFF CACHE BOOL "Build tests" FORCE) - endif () -endif () - -include(FetchContent) -FetchContent_Declare( - argparse - GIT_REPOSITORY https://github.com/p-ranav/argparse.git -) -FetchContent_MakeAvailable(argparse) diff --git a/cmake/sanitizers.cmake b/cmake/sanitizers.cmake deleted file mode 100644 index 76fd40d..0000000 --- a/cmake/sanitizers.cmake +++ /dev/null @@ -1,59 +0,0 @@ -function(columnar_enable_sanitizers sanitizers_raw) - if (NOT sanitizers_raw) - return() - endif () - - string(REPLACE "," ";" sanitizers "${sanitizers_raw}") - list(REMOVE_DUPLICATES sanitizers) - - set(supported_sanitizers address undefined leak thread memory) - foreach (sanitizer IN LISTS sanitizers) - if (NOT sanitizer IN_LIST supported_sanitizers) - message(FATAL_ERROR "Unsupported sanitizer: ${sanitizer}") - endif () - endforeach () - - if ("thread" IN_LIST sanitizers AND "address" IN_LIST sanitizers) - message(FATAL_ERROR "ThreadSanitizer cannot be combined with AddressSanitizer") - endif () - - if ("thread" IN_LIST sanitizers AND "leak" IN_LIST sanitizers) - message(FATAL_ERROR "ThreadSanitizer cannot be combined with LeakSanitizer") - endif () - - if ("memory" IN_LIST sanitizers AND "address" IN_LIST sanitizers) - message(FATAL_ERROR "MemorySanitizer cannot be combined with AddressSanitizer") - endif () - - if ("memory" IN_LIST sanitizers AND "thread" IN_LIST sanitizers) - message(FATAL_ERROR "MemorySanitizer cannot be combined with ThreadSanitizer") - endif () - - if ("memory" IN_LIST sanitizers AND "leak" IN_LIST sanitizers) - message(FATAL_ERROR "MemorySanitizer cannot be combined with LeakSanitizer") - endif () - - if ("memory" IN_LIST sanitizers) - if (NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") - message(FATAL_ERROR "MemorySanitizer requires Clang") - endif () - if (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") - message(FATAL_ERROR "MemorySanitizer is not supported by AppleClang") - endif () - if (NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") - message(FATAL_ERROR "MemorySanitizer is only supported in this project on Linux") - endif () - endif () - - string(JOIN "," sanitizer_flags ${sanitizers}) - - add_compile_options( - -fno-omit-frame-pointer - -fno-optimize-sibling-calls - -fsanitize=${sanitizer_flags} - ) - - add_link_options(-fsanitize=${sanitizer_flags}) - - message(STATUS "Enabled sanitizers: ${sanitizer_flags}") -endfunction() diff --git a/include/convert/csv_columnar.h b/include/convert/csv_columnar.h deleted file mode 100644 index d58a734..0000000 --- a/include/convert/csv_columnar.h +++ /dev/null @@ -1,11 +0,0 @@ -/* (What is this? — CSV <-> колоночный формат) */ - -#pragma once - -#include -#include - -void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, - const std::filesystem::path& output_path, size_t max_rows_per_group = 4); -void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, - const std::filesystem::path& data_path); diff --git a/include/executor/aggregate_function.h b/include/executor/aggregate_function.h deleted file mode 100644 index 0494d4e..0000000 --- a/include/executor/aggregate_function.h +++ /dev/null @@ -1,45 +0,0 @@ -/* (What is this? — Объявление функций агрегации) */ - -#pragma once - -#include -#include -#include -#include - -#include "schema.h" - -class AggState; -struct PlannedAgg; - -using AggFactory = std::unique_ptr (*)(const PlannedAgg&); -using AggTypeSupport = bool (*)(ColumnType); - -struct AggFuncDefinition { - std::string_view canonical_name; - bool distinct = false; - bool star = false; - AggTypeSupport supports_type = nullptr; - AggFactory factory = nullptr; -}; - -class AggRegistry { - public: - using RegistryMap = std::unordered_map; - - static AggRegistry& Instance(); - - void Register(const AggFuncDefinition& def); - const AggFuncDefinition* Find(std::string_view name) const; - - private: - RegistryMap registry_; -}; - -struct AggRegistrar { - explicit AggRegistrar(const AggFuncDefinition& def); -}; - -const AggFuncDefinition& ResolveAggFunc(std::string_view name); -bool AggSupportsInputType(const AggFuncDefinition& definition, ColumnType type); -std::string FormatAgg(const AggFuncDefinition& definition, std::string_view column_name, bool distinct, bool star); diff --git a/include/executor/executor.h b/include/executor/executor.h deleted file mode 100644 index be10e0b..0000000 --- a/include/executor/executor.h +++ /dev/null @@ -1,120 +0,0 @@ -/* (What is this? — Объявление классов и функций исполнителя запросов) */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "aggregate_function.h" -#include "batch.h" -#include "error.h" -#include "expected.hpp" -#include "tokenizer.h" - -using ExecuteExpected = tl::expected; - -class Operator { - public: - Operator() = default; - Operator(const Operator&) = delete; - Operator(Operator&&) noexcept = default; - Operator& operator=(const Operator&) = delete; - Operator& operator=(Operator&&) noexcept = default; - virtual ~Operator() = default; - - public: - virtual std::optional Next() = 0; -}; - -class Executor { - public: - Executor() = default; - - public: - void RegisterTable(const std::string& name, std::filesystem::path path); - ExecuteExpected Execute(std::string_view query) const; - - private: - std::unordered_map tables_; -}; - -enum class ComparisonKind { - Equal, - NotEqual, - Less, - LessOrEqual, - Greater, - GreaterOrEqual, -}; - -struct ParsedLiteral { - std::string text; - Tokens token_type = Tokens::StringLiteral; -}; - -struct FilterSpec { - std::string column_name; - ComparisonKind comparison = ComparisonKind::Equal; - ParsedLiteral literal; -}; - -struct AggSpec { - const AggFuncDefinition* function = nullptr; - bool distinct = false; - bool star = false; - std::string column_name; - std::string output_name; -}; - -struct ParsedQuery { - std::string table_name; - std::vector aggregates; - std::optional filter; -}; - -struct PlannedAgg { - const AggFuncDefinition* function = nullptr; - bool distinct = false; - bool star = false; - std::optional column_index; - ColumnType input_type = ColumnType::String; - std::string output_name; -}; - -struct PlannedFilter { - size_t column_index = 0; - ColumnType column_type = ColumnType::String; - ComparisonKind comparison = ComparisonKind::Equal; - std::string literal_text; -}; - -struct PlannedQuery { - std::filesystem::path table_path; - std::vector projection_indexes; - std::optional filter; - std::vector aggregates; -}; - -class AggState { - public: - AggState() = default; - AggState(const AggState&) = delete; - AggState(AggState&&) noexcept = default; - AggState& operator=(const AggState&) = delete; - AggState& operator=(AggState&&) noexcept = default; - virtual ~AggState() = default; - - public: - virtual void Consume(std::optional value) = 0; - virtual std::string Finalize() const = 0; -}; - -ParsedQuery ParseQuery(std::string_view query); -PlannedQuery PlanQuery(const ParsedQuery& parsed, const std::unordered_map& tables); -std::unique_ptr CreateAggState(const PlannedAgg& aggregate); -std::unique_ptr BuildPlan(const PlannedQuery& planned); diff --git a/include/io/batch_io.h b/include/io/batch_io.h deleted file mode 100644 index 3fea3d1..0000000 --- a/include/io/batch_io.h +++ /dev/null @@ -1,42 +0,0 @@ -/* (What is this? — Потоковое чтение/запись батчей из CSV в колоночный формат) */ - -#pragma once - -#include - -#include "batch.h" - -struct BatchSizing { - std::optional max_rows; - std::optional max_values; - std::optional max_bytes; - - bool WouldExceed(size_t next_rows, size_t column_count, uint64_t next_bytes) const; -}; - -class BatchReader { - public: - BatchReader() = default; - BatchReader(const BatchReader&) = default; - BatchReader(BatchReader&&) noexcept = default; - BatchReader& operator=(const BatchReader&) = default; - BatchReader& operator=(BatchReader&&) noexcept = default; - virtual ~BatchReader() = default; - - public: - virtual std::optional ReadNext() = 0; -}; - -class BatchWriter { - public: - BatchWriter() = default; - BatchWriter(const BatchWriter&) = default; - BatchWriter(BatchWriter&&) noexcept = default; - BatchWriter& operator=(const BatchWriter&) = default; - BatchWriter& operator=(BatchWriter&&) noexcept = default; - virtual ~BatchWriter() = default; - - public: - virtual void Write(const Batch& batch) = 0; - virtual void Flush() = 0; -}; diff --git a/include/io/columnar_batch_io.h b/include/io/columnar_batch_io.h deleted file mode 100644 index d04f02b..0000000 --- a/include/io/columnar_batch_io.h +++ /dev/null @@ -1,62 +0,0 @@ -/* (What is this? — Чтение/запись батчей из CSV в колоночный формат) */ - -#pragma once - -#include -#include -#include - -#include "batch_io.h" -#include "fileio.h" -#include "metadata.h" - -class ColumnarBatchReader final : public BatchReader { - public: - explicit ColumnarBatchReader(const std::filesystem::path& path); - ColumnarBatchReader(const ColumnarBatchReader&) = delete; - ColumnarBatchReader(ColumnarBatchReader&&) noexcept = default; - ColumnarBatchReader& operator=(const ColumnarBatchReader&) = delete; - ColumnarBatchReader& operator=(ColumnarBatchReader&&) noexcept = default; - ~ColumnarBatchReader() override = default; - - public: - std::optional ReadNext() override; - - const Schema& GetSchema() const { return metadata_.schema; } - const ColumnarMetadata& GetMetadata() const { return metadata_; } - - private: - static ColumnarMetadata ReadFileMetadata(const std::filesystem::path& path); - - private: - std::filesystem::path path_; - std::ifstream in_; - - ColumnarMetadata metadata_; - size_t next_group_ = 0; -}; - -class ColumnarBatchWriter final : public BatchWriter { - public: - ColumnarBatchWriter(const std::filesystem::path& path, Schema schema); - ColumnarBatchWriter(const ColumnarBatchWriter&) = delete; - ColumnarBatchWriter(ColumnarBatchWriter&&) noexcept = default; - ColumnarBatchWriter& operator=(const ColumnarBatchWriter&) = delete; - ColumnarBatchWriter& operator=(ColumnarBatchWriter&&) noexcept = default; - ~ColumnarBatchWriter() override = default; - - public: - void Write(const Batch& batch) override; - void Flush() override; - - void Finalize(); - - const ColumnarMetadata& GetMetadata() const { return metadata_; } - - private: - std::filesystem::path path_; - std::ofstream out_; - - ColumnarMetadata metadata_; - bool finalized_ = false; -}; diff --git a/include/io/csv.h b/include/io/csv.h deleted file mode 100644 index e01e998..0000000 --- a/include/io/csv.h +++ /dev/null @@ -1,55 +0,0 @@ -/* (What is this? — Чтение/запись CSV) */ - -#pragma once - -#include -#include -#include -#include -#include - -class CsvReader { - public: - explicit CsvReader(std::istream& in); - explicit CsvReader(const std::filesystem::path& path); - CsvReader(const CsvReader&) = delete; - CsvReader(CsvReader&& other) noexcept; - CsvReader& operator=(const CsvReader&) = delete; - CsvReader& operator=(CsvReader&& other) noexcept; - ~CsvReader() = default; - - public: - bool ReadRow(std::vector& row) const; - - private: - void RebindAfterMove(bool uses_owned_stream, std::istream* source_stream) noexcept; - - private: - std::ifstream owned_in_; - std::istream* in_ = nullptr; -}; - -class CsvWriter { - public: - explicit CsvWriter(std::ostream& out); - explicit CsvWriter(const std::filesystem::path& path); - CsvWriter(const CsvWriter&) = delete; - CsvWriter(CsvWriter&& other) noexcept; - CsvWriter& operator=(const CsvWriter&) = delete; - CsvWriter& operator=(CsvWriter&& other) noexcept; - ~CsvWriter() = default; - - public: - void WriteRow(const std::vector& row) const; - void Flush() const; - - private: - void RebindAfterMove(bool uses_owned_stream, std::ostream* source_stream) noexcept; - - private: - std::ofstream owned_out_; - std::ostream* out_ = nullptr; -}; - -std::vector> ReadRows(const std::filesystem::path& path); -void WriteRows(const std::filesystem::path& path, const std::vector>& rows); diff --git a/include/io/csv_batch_io.h b/include/io/csv_batch_io.h deleted file mode 100644 index 86c108a..0000000 --- a/include/io/csv_batch_io.h +++ /dev/null @@ -1,54 +0,0 @@ -/* (What is this? — Чтение/запись CSV) */ - -#pragma once - -#include -#include -#include - -#include "batch_io.h" -#include "csv.h" - -class CsvBatchReader final : public BatchReader { - public: - CsvBatchReader(const std::filesystem::path& path, Schema schema, BatchSizing sizing); - CsvBatchReader(const CsvBatchReader&) = delete; - CsvBatchReader(CsvBatchReader&&) noexcept = default; - CsvBatchReader& operator=(const CsvBatchReader&) = delete; - CsvBatchReader& operator=(CsvBatchReader&&) noexcept = default; - ~CsvBatchReader() override = default; - - public: - std::optional ReadNext() override; - - const Schema& GetSchema() const { return schema_; } - - private: - CsvReader csv_reader_; - Schema schema_; - BatchSizing sizing_; - - std::optional> pending_row_; - bool reached_eof_ = false; -}; - -class CsvBatchWriter final : public BatchWriter { - public: - CsvBatchWriter(const std::filesystem::path& path, Schema schema); - CsvBatchWriter(const CsvBatchWriter&) = delete; - CsvBatchWriter(CsvBatchWriter&&) noexcept = default; - CsvBatchWriter& operator=(const CsvBatchWriter&) = delete; - CsvBatchWriter& operator=(CsvBatchWriter&&) noexcept = default; - ~CsvBatchWriter() override = default; - - public: - void Write(const Batch& batch) override; - void Flush() override; - - private: - CsvWriter csv_writer_; - Schema schema_; -}; - -void AppendBatchRows(const Batch& batch, std::vector>& rows); -void WriteBatchCsv(const std::filesystem::path& path, const Batch& batch); diff --git a/include/io/fileio.h b/include/io/fileio.h deleted file mode 100644 index c2d9b36..0000000 --- a/include/io/fileio.h +++ /dev/null @@ -1,35 +0,0 @@ -/* (What is this? — Обертки над потоками, чтение и запись) */ - -#pragma once - -#include -#include -#include -#include -#include - -struct FileMetadata { - uintmax_t size = 0; - bool is_regular = false; - bool is_directory = false; - std::filesystem::file_time_type last_write_time; -}; - -std::optional TryGetFileMetadata(const std::filesystem::path& path); - -bool FileExists(const std::filesystem::path& path); -void EnsureParentDirectory(const std::filesystem::path& path); - -std::ifstream OpenInputFile(const std::filesystem::path& path); - -enum class FileOpenMode { - Truncate, - Append, -}; - -std::ofstream OpenOutputFile(const std::filesystem::path& path, FileOpenMode mode = FileOpenMode::Truncate); - -std::string ReadTextFile(const std::filesystem::path& path); -std::vector ReadFileBytes(const std::filesystem::path& path); -void WriteFileBytes(const std::filesystem::path& path, std::span bytes); -void WriteFileBytes(const std::filesystem::path& path, const std::vector& bytes); diff --git a/include/io/stream.h b/include/io/stream.h deleted file mode 100644 index 676400a..0000000 --- a/include/io/stream.h +++ /dev/null @@ -1,31 +0,0 @@ -/* (What is this? — Обертки над потоками, чтение и запись) */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "error.h" - -void ReadBytes(std::istream& in, char* dst, size_t size); -void WriteBytes(std::ostream& out, std::string_view bytes); - -template -T ReadStream(std::istream& in) { - T value = 0; - ReadBytes(in, reinterpret_cast(&value), sizeof(value)); - return value; -} - -template -void WriteStream(std::ostream& out, T value) { - WriteBytes(out, {reinterpret_cast(&value), sizeof(value)}); -} diff --git a/include/model/batch.h b/include/model/batch.h deleted file mode 100644 index 08d5667..0000000 --- a/include/model/batch.h +++ /dev/null @@ -1,41 +0,0 @@ -/* (What is this? — Класс батча) */ - -#pragma once - -#include -#include - -#include "column.h" -#include "schema.h" - -class Batch { - public: - Batch() = default; - explicit Batch(Schema schema); - Batch(Schema schema, size_t reserve_rows); - Batch(const Batch& other); - Batch(Batch&& other) noexcept = default; - Batch& operator=(const Batch& other); - Batch& operator=(Batch&& other) noexcept = default; - ~Batch() = default; - - public: - const Schema& GetSchema() const { return schema_; } - - size_t ColumnsCount() const; - size_t RowsCount() const; - - void Reserve(size_t n) const; - - const Column& ColumnAt(size_t i) const; - Column& ColumnAt(size_t i); - - const std::vector>& GetColumns() const { return columns_; } - std::vector>& GetColumns() { return columns_; } - - void Validate() const; - - private: - Schema schema_; - std::vector> columns_; -}; diff --git a/include/model/column.h b/include/model/column.h deleted file mode 100644 index dc9c770..0000000 --- a/include/model/column.h +++ /dev/null @@ -1,44 +0,0 @@ -/* (What is this? — Абстракция колонок) */ - -#pragma once - -#include -#include -#include -#include -#include - -#include "schema.h" - -class Column { - public: - explicit Column(ColumnType type); - Column(const Column&) = default; - Column(Column&&) noexcept = default; - Column& operator=(const Column&) = default; - Column& operator=(Column&&) noexcept = default; - virtual ~Column() = default; - - public: - ColumnType Type() const { return type_; } - - virtual size_t Size() const = 0; - virtual void Reserve(size_t n) = 0; - virtual void Clear() = 0; - - virtual void AppendFromString(const std::string& value) = 0; - virtual uint64_t EstimateSizeFromString(std::string_view value) const = 0; - virtual std::string ValueAsString(size_t row) const = 0; - virtual std::unique_ptr Clone() const = 0; - - virtual void WriteTo(std::ostream& out) const = 0; - virtual void ReadFrom(std::istream& in, uint32_t row_count, uint64_t size) = 0; - - protected: - static void CheckRowIndex(const char* module, size_t row, size_t size); - - private: - ColumnType type_; -}; - -std::unique_ptr CreateColumn(ColumnType type); diff --git a/include/model/column_boolean.h b/include/model/column_boolean.h deleted file mode 100644 index 4194b4a..0000000 --- a/include/model/column_boolean.h +++ /dev/null @@ -1,15 +0,0 @@ -/* (What is this? — Колонка логического типа) */ - -#pragma once - -#include "fixed_column.h" - -class BooleanColumn final : public FixedColumn { - public: - using FixedColumn::FixedColumn; - - static const char* ModuleName() { return "column_boolean"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; -}; diff --git a/include/model/column_character.h b/include/model/column_character.h deleted file mode 100644 index 9ba8684..0000000 --- a/include/model/column_character.h +++ /dev/null @@ -1,15 +0,0 @@ -/* (What is this? — Колонка символьного типа) */ - -#pragma once - -#include "fixed_column.h" - -class CharacterColumn final : public FixedColumn { - public: - using FixedColumn::FixedColumn; - - static const char* ModuleName() { return "column_character"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; -}; diff --git a/include/model/column_date.h b/include/model/column_date.h deleted file mode 100644 index fa185a4..0000000 --- a/include/model/column_date.h +++ /dev/null @@ -1,15 +0,0 @@ -/* (What is this? — Колонка типа DATE) */ - -#pragma once - -#include "fixed_column.h" - -class DateColumn final : public FixedColumn { - public: - using FixedColumn::FixedColumn; - - static const char* ModuleName() { return "column_date"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; -}; diff --git a/include/model/column_int128.h b/include/model/column_int128.h deleted file mode 100644 index d24250d..0000000 --- a/include/model/column_int128.h +++ /dev/null @@ -1,16 +0,0 @@ -/* (What is this? — Колонка типа INT128) */ - -#pragma once - -#include "fixed_column.h" -#include "int128.h" - -class Int128Column final : public FixedColumn { - public: - using FixedColumn::FixedColumn; - - static const char* ModuleName() { return "column_int128"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; -}; diff --git a/include/model/column_int16.h b/include/model/column_int16.h deleted file mode 100644 index 14a0811..0000000 --- a/include/model/column_int16.h +++ /dev/null @@ -1,15 +0,0 @@ -/* (What is this? — Колонка типа INT16) */ - -#pragma once - -#include "fixed_column.h" - -class Int16Column final : public FixedColumn { - public: - using FixedColumn::FixedColumn; - - static const char* ModuleName() { return "column_int16"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; -}; diff --git a/include/model/column_int32.h b/include/model/column_int32.h deleted file mode 100644 index 458e60a..0000000 --- a/include/model/column_int32.h +++ /dev/null @@ -1,15 +0,0 @@ -/* (What is this? — Колонка типа INT32) */ - -#pragma once - -#include "fixed_column.h" - -class Int32Column final : public FixedColumn { - public: - using FixedColumn::FixedColumn; - - static const char* ModuleName() { return "column_int32"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; -}; diff --git a/include/model/column_int64.h b/include/model/column_int64.h deleted file mode 100644 index aa44865..0000000 --- a/include/model/column_int64.h +++ /dev/null @@ -1,15 +0,0 @@ -/* (What is this? — Колонка типа INT64) */ - -#pragma once - -#include "fixed_column.h" - -class Int64Column final : public FixedColumn { - public: - using FixedColumn::FixedColumn; - - static const char* ModuleName() { return "column_int64"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; -}; diff --git a/include/model/column_string.h b/include/model/column_string.h deleted file mode 100644 index 60589f8..0000000 --- a/include/model/column_string.h +++ /dev/null @@ -1,15 +0,0 @@ -/* (What is this? — Колонка типа STRING) */ - -#pragma once - -#include "variant_column.h" - -class StringColumn final : public VariantColumn { - public: - using VariantColumn::VariantColumn; - - static const char* ModuleName() { return "column_string"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; -}; diff --git a/include/model/column_timestamp.h b/include/model/column_timestamp.h deleted file mode 100644 index 6fb061b..0000000 --- a/include/model/column_timestamp.h +++ /dev/null @@ -1,15 +0,0 @@ -/* (What is this? — Колонка типа TIMESTAMP) */ - -#pragma once - -#include "fixed_column.h" - -class TimestampColumn final : public FixedColumn { - public: - using FixedColumn::FixedColumn; - - static const char* ModuleName() { return "column_timestamp"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; -}; diff --git a/include/model/fixed_column.h b/include/model/fixed_column.h deleted file mode 100644 index ce5b169..0000000 --- a/include/model/fixed_column.h +++ /dev/null @@ -1,58 +0,0 @@ -/* (What is this? — База для колонок фиксированного размера) */ - -#pragma once - -#include - -#include "column.h" -#include "error.h" -#include "stream.h" - -template -class FixedColumn : public Column { - public: - FixedColumn() : Column(TypeValue) {} - FixedColumn(const FixedColumn&) = default; - FixedColumn(FixedColumn&&) noexcept = default; - FixedColumn& operator=(const FixedColumn&) = default; - FixedColumn& operator=(FixedColumn&&) noexcept = default; - ~FixedColumn() override = default; - - public: - size_t Size() const override { return values_.size(); } - void Reserve(const size_t n) override { values_.reserve(n); } - void Clear() override { values_.clear(); } - - uint64_t EstimateSizeFromString(const std::string_view value) const override { - (void)value; - return sizeof(T); - } - - std::unique_ptr Clone() const override { - return std::make_unique(static_cast(*this)); - } - - void WriteTo(std::ostream& out) const override { - WriteBytes(out, {reinterpret_cast(values_.data()), values_.size() * sizeof(T)}); - } - - void ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) override { - const uint64_t expected = static_cast(row_count) * sizeof(T); - if (size != expected) { - throw Error::Mismatch(Derived::ModuleName(), "column chunk size mismatch"); - } - values_.resize(row_count); - ReadBytes(in, reinterpret_cast(values_.data()), values_.size() * sizeof(T)); - } - - protected: - void AppendValue(const T value) { values_.push_back(value); } - - T ValueAt(const size_t row) const { - CheckRowIndex(Derived::ModuleName(), row, values_.size()); - return values_[row]; - } - - protected: - std::vector values_; -}; diff --git a/include/model/metadata.h b/include/model/metadata.h deleted file mode 100644 index f442ac0..0000000 --- a/include/model/metadata.h +++ /dev/null @@ -1,27 +0,0 @@ -/* (What is this? — Сериализация метаданных) */ - -#pragma once - -#include -#include -#include - -#include "schema.h" - -struct ColumnChunkMetadata { - uint64_t offset = 0; - uint64_t size = 0; -}; - -struct RowGroupMetadata { - uint32_t row_count = 0; - std::vector columns; -}; - -struct ColumnarMetadata { - Schema schema; - std::vector row_groups; -}; - -ColumnarMetadata ReadMetadata(std::istream& in); -void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata); diff --git a/include/model/schema.h b/include/model/schema.h deleted file mode 100644 index 2e40fd7..0000000 --- a/include/model/schema.h +++ /dev/null @@ -1,37 +0,0 @@ -/* (What is this? — Чтение/запись схемы из CSV) */ - -#pragma once - -#include -#include -#include -#include - -enum class ColumnType : uint8_t { - Int64 = 1, - String = 2, - Boolean = 3, - Int16 = 4, - Int32 = 5, - Int128 = 6, - Date = 7, - Timestamp = 8, - Character = 9, -}; - -struct ColumnSchema { - std::string name; - ColumnType type = ColumnType::String; - - bool operator==(const ColumnSchema& other) const = default; -}; - -struct Schema { - std::vector columns; - - bool operator==(const Schema& other) const = default; -}; - -Schema ReadSchemaCsv(const std::filesystem::path& path); -Schema InferSchemaCsv(const std::filesystem::path& path); -void WriteSchemaCsv(const std::filesystem::path& path, const Schema& schema); diff --git a/include/model/variant_column.h b/include/model/variant_column.h deleted file mode 100644 index bbea74f..0000000 --- a/include/model/variant_column.h +++ /dev/null @@ -1,94 +0,0 @@ -/* (What is this? — База для колонок переменной длины) */ - -#pragma once - -#include -#include -#include - -#include "column.h" -#include "error.h" -#include "stream.h" - -template -class VariantColumn : public Column { - public: - VariantColumn() : Column(TypeValue) {} - VariantColumn(const VariantColumn&) = default; - VariantColumn(VariantColumn&&) noexcept = default; - VariantColumn& operator=(const VariantColumn&) = default; - VariantColumn& operator=(VariantColumn&&) noexcept = default; - ~VariantColumn() override = default; - - public: - size_t Size() const override { return values_.size(); } - void Reserve(const size_t n) override { values_.reserve(n); } - void Clear() override { values_.clear(); } - - uint64_t EstimateSizeFromString(const std::string_view value) const override { return value.size(); } - - std::unique_ptr Clone() const override { - return std::make_unique(static_cast(*this)); - } - - void WriteTo(std::ostream& out) const override { - size_t total_size = 0; - - for (const auto& value : values_) { - if (value.size() > std::numeric_limits::max()) { - throw Error::Overflow(Derived::ModuleName(), "value exceeds supported size"); - } - total_size += sizeof(uint32_t) + value.size(); - } - - std::string buffer(total_size, '\0'); - char* dst = buffer.data(); - - for (const auto& value : values_) { - const uint32_t length = value.size(); - - std::memcpy(dst, &length, sizeof(length)); - dst += sizeof(length); - - if (!value.empty()) { - std::memcpy(dst, value.data(), value.size()); - dst += value.size(); - } - } - - WriteBytes(out, buffer); - } - - void ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) override { - values_.clear(); - values_.reserve(row_count); - - uint64_t consumed = 0; - - for (uint32_t row_index = 0; row_index < row_count; ++row_index) { - const uint32_t length = ReadStream(in); - consumed += sizeof(uint32_t); - - std::string value(length, '\0'); - ReadBytes(in, value.data(), value.size()); - - consumed += length; - values_.push_back(std::move(value)); - } - - if (consumed != size) { - throw Error::Mismatch(Derived::ModuleName(), "column chunk size mismatch"); - } - } - - protected: - void AppendValue(std::string value) { values_.push_back(std::move(value)); } - - const std::string& ValueAt(const size_t row) const { - CheckRowIndex(Derived::ModuleName(), row, values_.size()); - return values_[row]; - } - - protected: - std::vector values_; -}; diff --git a/include/sql_parser/tokenizer.h b/include/sql_parser/tokenizer.h deleted file mode 100644 index 19bacb8..0000000 --- a/include/sql_parser/tokenizer.h +++ /dev/null @@ -1,322 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include "error.h" -#include "expected.hpp" - -enum class Tokens { - NameToken, - NumericLiteral, - StringLiteral, - - Select, - From, - Create, - As, - Sum, - Count, - Avg, - Max, - Min, - And, - Distinct, - Length, - Where, - By, - Group, - Order, - Limit, - - Plus, - Minus, - Asterisk, - OpenBracket, - CloseBracket, - Comma, - Dot, - Semicolon, - - Equal, - NotEqual, - Less, - LessOrEqual, - Greater, - GreaterOrEqual, - - EOI, -}; - -std::string_view TokenTypeToString(Tokens type) noexcept; - -template -using Expected = tl::expected; - -class Token { - public: - Token(std::string text, size_t offset); - virtual ~Token() = default; - - virtual Tokens GetType() const noexcept = 0; - - std::string_view GetText() const noexcept; - size_t GetOffset() const noexcept; - - private: - std::string text_; - size_t offset_ = 0; -}; - -using TokenPtr = std::shared_ptr; - -class Command : public Token { - public: - using Token::Token; - ~Command() override = default; - - void AddArg(TokenPtr arg); - const std::vector& GetArgs() const noexcept; - - protected: - std::vector args_; -}; - -class IOperatorToken : public Command { - public: - using Command::Command; - ~IOperatorToken() override = default; -}; - -class TSelectToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TCreateToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TFromToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TLimitToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TOrderToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TWhereToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TGroupToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TSumToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TMinToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TMaxToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TCountToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TDistinctToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TAvgToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TLengthToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TPlusToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TMinusToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TNameToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; - - std::string_view GetName() const noexcept; -}; - -class TNumericLiteralToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TStringLiteralToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TAsToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TAndToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TByToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TOpenBracketToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TCommaToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TCloseBracketToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TAsteriskToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TDotToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TSemicolonToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TNotEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TLessToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TLessOrEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TGreaterToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TGreaterOrEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TEndOfInputToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class Tokenizer { - public: - explicit Tokenizer(std::string data); - - Expected GetNext(); - - private: - std::string query_; - size_t pos_ = 0; -}; - -Expected> TokenizeSql(std::string_view input); - -Tokens ResolveIdentifierType(std::string_view text); -TokenPtr MakeToken(Tokens type, std::string text, size_t offset); -Expected MakeUnexpectedCharacter(char ch, size_t offset); -Expected MakeUnterminatedString(size_t offset); diff --git a/include/support/ascii.h b/include/support/ascii.h deleted file mode 100644 index 28eabe9..0000000 --- a/include/support/ascii.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -#include -#include - -std::string ToLowerAscii(std::string_view input); -std::string ToUpperAscii(std::string_view input); diff --git a/include/support/error.h b/include/support/error.h deleted file mode 100644 index 8cd7a87..0000000 --- a/include/support/error.h +++ /dev/null @@ -1,72 +0,0 @@ -/* (What is this? — Модуль ошибок) */ - -#pragma once - -#include -#include -#include -#include -#include - -class Error final : public std::exception { - public: - enum class Code : uint16_t { - InvalidArgument = 1, - InvalidData = 2, - NotFound = 3, - OutOfRange = 4, - Mismatch = 5, - Overflow = 6, - InvalidState = 7, - Unsupported = 8, - IoFailure = 9, - }; - - enum class Type : uint8_t { - Validation, - Data, - State, - Io, - }; - - public: - Error(Code code, std::string module, std::string module_instance, Type type, std::string message); - Error(const Error&) = default; - Error(Error&&) noexcept = default; - Error& operator=(const Error&) = default; - Error& operator=(Error&&) noexcept = default; - ~Error() override = default; - - public: - static Error InvalidArgument(std::string module, std::string message, std::string module_instance = {}); - static Error InvalidData(std::string module, std::string message, std::string module_instance = {}); - static Error NotFound(std::string module, std::string message, std::string module_instance = {}); - static Error OutOfRange(std::string module, std::string message, std::string module_instance = {}); - static Error Mismatch(std::string module, std::string message, std::string module_instance = {}); - static Error Overflow(std::string module, std::string message, std::string module_instance = {}); - static Error InvalidState(std::string module, std::string message, std::string module_instance = {}); - static Error Unsupported(std::string module, std::string message, std::string module_instance = {}); - static Error Io(std::string module, std::string message, std::string module_instance = {}); - static Error PathIo(std::string module, const std::filesystem::path& path, std::string_view action); - - const char* what() const noexcept override; - Code GetCode() const noexcept; - Type GetType() const noexcept; - const std::string& GetModule() const noexcept; - const std::string& GetModuleInstance() const noexcept; - const std::string& GetMessage() const noexcept; - - static std::string_view CodeToString(Code code) noexcept; - static std::string_view TypeToString(Type type) noexcept; - - private: - static std::string BuildWhat(std::string_view module, std::string_view message); - - private: - Code code_; - std::string module_; - std::string module_instance_; - Type type_; - std::string message_; - std::string what_; -}; diff --git a/include/support/int128.h b/include/support/int128.h deleted file mode 100644 index 7b9f28a..0000000 --- a/include/support/int128.h +++ /dev/null @@ -1,24 +0,0 @@ -/* (What is this? — Алиасы для 128-битных целых) */ - -#pragma once - -#if !defined(__SIZEOF_INT128__) -#error "This project requires compiler support for 128-bit integers" -#endif - -#if defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wpedantic" -#elif defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#endif - -using Int128 = __int128_t; -using UInt128 = unsigned __int128; - -#if defined(__clang__) -#pragma clang diagnostic pop -#elif defined(__GNUC__) -#pragma GCC diagnostic pop -#endif diff --git a/include/support/parsing.h b/include/support/parsing.h deleted file mode 100644 index 97ec50d..0000000 --- a/include/support/parsing.h +++ /dev/null @@ -1,27 +0,0 @@ -/* (What is this? — Парсеры) */ - -#pragma once - -#include -#include -#include - -#include "int128.h" -#include "schema.h" - -bool ParseBoolean(const std::string& value); -int16_t ParseInt16(const std::string& value); -int32_t ParseInt32(const std::string& value); -int64_t ParseInt64(const std::string& value); -Int128 ParseInt128(const std::string& value); -int32_t ParseDate(const std::string& value); -int64_t ParseTimestamp(const std::string& value); -char ParseCharacter(const std::string& value); - -std::string BooleanToString(bool value); -std::string Int128ToString(Int128 value); -std::string DateToString(int32_t value); -std::string TimestampToString(int64_t value); - -ColumnType ParseColumnType(std::string_view input); -std::string ColumnTypeToString(ColumnType type); diff --git a/include/testing/executor_test_utils.h b/include/testing/executor_test_utils.h deleted file mode 100644 index 55f3c9d..0000000 --- a/include/testing/executor_test_utils.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include -#include - -#include "csv_batch_io.h" -#include "gtest/gtest.h" - -inline std::vector SingleRowValues(const Batch& batch) { - EXPECT_EQ(batch.RowsCount(), 1u); - - std::vector> rows; - rows.reserve(1); - - AppendBatchRows(batch, rows); - - if (rows.empty()) { - return {}; - } - - return std::move(rows.front()); -} - -inline std::vector BatchColumnNames(const Batch& batch) { - std::vector names; - names.reserve(batch.ColumnsCount()); - - for (const auto& column : batch.GetSchema().columns) { - names.push_back(column.name); - } - - return names; -} diff --git a/include/testing/temp_file.h b/include/testing/temp_file.h deleted file mode 100644 index f4cea19..0000000 --- a/include/testing/temp_file.h +++ /dev/null @@ -1,52 +0,0 @@ -/* (What is this? — Модуль временного файла) */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -inline std::filesystem::path UniqueTempPath(const std::string& tag) { - const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); - - std::random_device rd; - std::mt19937_64 gen(rd()); - std::uniform_int_distribution dist; - - return std::filesystem::temp_directory_path() / (tag + "_" + std::to_string(now) + "_" + std::to_string(dist(gen))); -} - -class TempFile { - public: - explicit TempFile(const std::string& tag) : path_(UniqueTempPath(tag)) {} - TempFile(const TempFile&) = delete; - TempFile(TempFile&& other) noexcept : path_(std::move(other.path_)) { other.path_.clear(); } - TempFile& operator=(const TempFile&) = delete; - TempFile& operator=(TempFile&& other) noexcept { - if (this != &other) { - Remove(); - path_ = std::move(other.path_); - other.path_.clear(); - } - return *this; - } - ~TempFile() { Remove(); } - - public: - const std::filesystem::path& Path() const { return path_; } - - private: - void Remove() const { - if (path_.empty()) { - return; - } - std::error_code ec; - std::filesystem::remove(path_, ec); - } - - private: - std::filesystem::path path_; -}; diff --git a/include/tl/expected.hpp b/include/tl/expected.hpp deleted file mode 100644 index 59e59aa..0000000 --- a/include/tl/expected.hpp +++ /dev/null @@ -1,2475 +0,0 @@ -/// -// expected - An implementation of std::expected with extensions -// Written in 2017 by Sy Brand (tartanllama@gmail.com, @TartanLlama) -// -// Documentation available at http://tl.tartanllama.xyz/ -// -// To the extent possible under law, the author(s) have dedicated all -// copyright and related and neighboring rights to this software to the -// public domain worldwide. This software is distributed without any warranty. -// -// You should have received a copy of the CC0 Public Domain Dedication -// along with this software. If not, see -// . -/// - -#ifndef TL_EXPECTED_HPP -#define TL_EXPECTED_HPP - -#define TL_EXPECTED_VERSION_MAJOR 1 -#define TL_EXPECTED_VERSION_MINOR 3 -#define TL_EXPECTED_VERSION_PATCH 1 - -#include -#include -#include -#include - -#if defined(__EXCEPTIONS) || defined(_CPPUNWIND) -#define TL_EXPECTED_EXCEPTIONS_ENABLED -#endif - -#if (defined(_MSC_VER) && _MSC_VER == 1900) -#define TL_EXPECTED_MSVC2015 -#define TL_EXPECTED_MSVC2015_CONSTEXPR -#else -#define TL_EXPECTED_MSVC2015_CONSTEXPR constexpr -#endif - -#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ <= 9 && \ - !defined(__clang__)) -#define TL_EXPECTED_GCC49 -#endif - -#if (defined(__GNUC__) && __GNUC__ == 5 && __GNUC_MINOR__ <= 4 && \ - !defined(__clang__)) -#define TL_EXPECTED_GCC54 -#endif - -#if (defined(__GNUC__) && __GNUC__ == 5 && __GNUC_MINOR__ <= 5 && \ - !defined(__clang__)) -#define TL_EXPECTED_GCC55 -#endif - -#ifdef _MSVC_LANG -#define TL_CPLUSPLUS _MSVC_LANG -#else -#define TL_CPLUSPLUS __cplusplus -#endif - -#if !defined(TL_ASSERT) -//can't have assert in constexpr in C++11 and GCC 4.9 has a compiler bug -#if (TL_CPLUSPLUS > 201103L) && !defined(TL_EXPECTED_GCC49) -#include -#define TL_ASSERT(x) assert(x) -#else -#define TL_ASSERT(x) -#endif -#endif - -#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ <= 9 && \ - !defined(__clang__)) -// GCC < 5 doesn't support overloading on const&& for member functions - -#define TL_EXPECTED_NO_CONSTRR -// GCC < 5 doesn't support some standard C++11 type traits -#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \ - std::has_trivial_copy_constructor -#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \ - std::has_trivial_copy_assign - -// This one will be different for GCC 5.7 if it's ever supported -#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) \ - std::is_trivially_destructible - -// GCC 5 < v < 8 has a bug in is_trivially_copy_constructible which breaks -// std::vector for non-copyable types -#elif (defined(__GNUC__) && __GNUC__ < 8 && !defined(__clang__)) -#ifndef TL_GCC_LESS_8_TRIVIALLY_COPY_CONSTRUCTIBLE_MUTEX -#define TL_GCC_LESS_8_TRIVIALLY_COPY_CONSTRUCTIBLE_MUTEX -namespace tl { -namespace detail { -template -struct is_trivially_copy_constructible - : std::is_trivially_copy_constructible {}; -#ifdef _GLIBCXX_VECTOR -template -struct is_trivially_copy_constructible> : std::false_type {}; -#endif -} // namespace detail -} // namespace tl -#endif - -#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \ - tl::detail::is_trivially_copy_constructible -#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \ - std::is_trivially_copy_assignable -#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) \ - std::is_trivially_destructible -#else -#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T) \ - std::is_trivially_copy_constructible -#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T) \ - std::is_trivially_copy_assignable -#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) \ - std::is_trivially_destructible -#endif - -#if TL_CPLUSPLUS > 201103L -#define TL_EXPECTED_CXX14 -#endif - -#ifdef TL_EXPECTED_GCC49 -#define TL_EXPECTED_GCC49_CONSTEXPR -#else -#define TL_EXPECTED_GCC49_CONSTEXPR constexpr -#endif - -#if (TL_CPLUSPLUS == 201103L || defined(TL_EXPECTED_MSVC2015) || \ - defined(TL_EXPECTED_GCC49)) -#define TL_EXPECTED_11_CONSTEXPR -#else -#define TL_EXPECTED_11_CONSTEXPR constexpr -#endif - -#if TL_CPLUSPLUS >= 201703L -#define TL_EXPECTED_NODISCARD [[nodiscard]] -#else -#define TL_EXPECTED_NODISCARD -#endif - -namespace tl { -template class TL_EXPECTED_NODISCARD expected; - -#ifndef TL_MONOSTATE_INPLACE_MUTEX -#define TL_MONOSTATE_INPLACE_MUTEX -class monostate {}; - -struct in_place_t { - explicit in_place_t() = default; -}; -static constexpr in_place_t in_place{}; -#endif - -template class unexpected { -public: - static_assert(!std::is_same::value, "E must not be void"); - - unexpected() = delete; - constexpr explicit unexpected(const E &e) : m_val(e) {} - - constexpr explicit unexpected(E &&e) : m_val(std::move(e)) {} - - template ::value>::type * = nullptr> - constexpr explicit unexpected(Args &&...args) - : m_val(std::forward(args)...) {} - template < - class U, class... Args, - typename std::enable_if &, Args &&...>::value>::type * = nullptr> - constexpr explicit unexpected(std::initializer_list l, Args &&...args) - : m_val(l, std::forward(args)...) {} - - constexpr const E &value() const & { return m_val; } - TL_EXPECTED_11_CONSTEXPR E &value() & { return m_val; } - TL_EXPECTED_11_CONSTEXPR E &&value() && { return std::move(m_val); } - constexpr const E &&value() const && { return std::move(m_val); } - -private: - E m_val; -}; - -#ifdef __cpp_deduction_guides -template unexpected(E) -> unexpected; -#endif - -template -constexpr bool operator==(const unexpected &lhs, const unexpected &rhs) { - return lhs.value() == rhs.value(); -} -template -constexpr bool operator!=(const unexpected &lhs, const unexpected &rhs) { - return lhs.value() != rhs.value(); -} -template -constexpr bool operator<(const unexpected &lhs, const unexpected &rhs) { - return lhs.value() < rhs.value(); -} -template -constexpr bool operator<=(const unexpected &lhs, const unexpected &rhs) { - return lhs.value() <= rhs.value(); -} -template -constexpr bool operator>(const unexpected &lhs, const unexpected &rhs) { - return lhs.value() > rhs.value(); -} -template -constexpr bool operator>=(const unexpected &lhs, const unexpected &rhs) { - return lhs.value() >= rhs.value(); -} - -template -unexpected::type> make_unexpected(E &&e) { - return unexpected::type>(std::forward(e)); -} - -struct unexpect_t { - unexpect_t() = default; -}; -static constexpr unexpect_t unexpect{}; - -#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED -#define TL_EXPECTED_THROW_EXCEPTION(e) throw((e)); -#else -#define TL_EXPECTED_THROW_EXCEPTION(e) std::terminate(); -#endif - -namespace detail { -#ifndef TL_TRAITS_MUTEX -#define TL_TRAITS_MUTEX -// C++14-style aliases for brevity -template using remove_const_t = typename std::remove_const::type; -template -using remove_reference_t = typename std::remove_reference::type; -template using decay_t = typename std::decay::type; -template -using enable_if_t = typename std::enable_if::type; -template -using conditional_t = typename std::conditional::type; - -// std::conjunction from C++17 -template struct conjunction : std::true_type {}; -template struct conjunction : B {}; -template -struct conjunction - : std::conditional, B>::type {}; - -#if defined(_LIBCPP_VERSION) && __cplusplus == 201103L -#define TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND -#endif - -// In C++11 mode, there's an issue in libc++'s std::mem_fn -// which results in a hard-error when using it in a noexcept expression -// in some cases. This is a check to workaround the common failing case. -#ifdef TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND -template -struct is_pointer_to_non_const_member_func : std::false_type {}; -template -struct is_pointer_to_non_const_member_func - : std::true_type {}; -template -struct is_pointer_to_non_const_member_func - : std::true_type {}; -template -struct is_pointer_to_non_const_member_func - : std::true_type {}; -template -struct is_pointer_to_non_const_member_func - : std::true_type {}; -template -struct is_pointer_to_non_const_member_func - : std::true_type {}; -template -struct is_pointer_to_non_const_member_func - : std::true_type {}; - -template struct is_const_or_const_ref : std::false_type {}; -template struct is_const_or_const_ref : std::true_type {}; -template struct is_const_or_const_ref : std::true_type {}; -#endif - -// std::invoke from C++17 -// https://stackoverflow.com/questions/38288042/c11-14-invoke-workaround -template < - typename Fn, typename... Args, -#ifdef TL_TRAITS_LIBCXX_MEM_FN_WORKAROUND - typename = enable_if_t::value && - is_const_or_const_ref::value)>, -#endif - typename = enable_if_t>::value>, int = 0> -constexpr auto invoke(Fn &&f, Args &&...args) noexcept( - noexcept(std::mem_fn(f)(std::forward(args)...))) - -> decltype(std::mem_fn(f)(std::forward(args)...)) { - return std::mem_fn(f)(std::forward(args)...); -} - -template >::value>> -constexpr auto invoke(Fn &&f, Args &&...args) noexcept( - noexcept(std::forward(f)(std::forward(args)...))) - -> decltype(std::forward(f)(std::forward(args)...)) { - return std::forward(f)(std::forward(args)...); -} - -// std::invoke_result from C++17 -template struct invoke_result_impl; - -template -struct invoke_result_impl< - F, - decltype(detail::invoke(std::declval(), std::declval()...), void()), - Us...> { - using type = - decltype(detail::invoke(std::declval(), std::declval()...)); -}; - -template -using invoke_result = invoke_result_impl; - -template -using invoke_result_t = typename invoke_result::type; - -#if defined(_MSC_VER) && _MSC_VER <= 1900 -// TODO make a version which works with MSVC 2015 -template struct is_swappable : std::true_type {}; - -template struct is_nothrow_swappable : std::true_type {}; -#else -// https://stackoverflow.com/questions/26744589/what-is-a-proper-way-to-implement-is-swappable-to-test-for-the-swappable-concept -namespace swap_adl_tests { -// if swap ADL finds this then it would call std::swap otherwise (same -// signature) -struct tag {}; - -template tag swap(T &, T &); -template tag swap(T (&a)[N], T (&b)[N]); - -// helper functions to test if an unqualified swap is possible, and if it -// becomes std::swap -template std::false_type can_swap(...) noexcept(false); -template (), std::declval()))> -std::true_type can_swap(int) noexcept(noexcept(swap(std::declval(), - std::declval()))); - -template std::false_type uses_std(...); -template -std::is_same(), std::declval())), tag> -uses_std(int); - -template -struct is_std_swap_noexcept - : std::integral_constant::value && - std::is_nothrow_move_assignable::value> {}; - -template -struct is_std_swap_noexcept : is_std_swap_noexcept {}; - -template -struct is_adl_swap_noexcept - : std::integral_constant(0))> {}; -} // namespace swap_adl_tests - -template -struct is_swappable - : std::integral_constant< - bool, - decltype(detail::swap_adl_tests::can_swap(0))::value && - (!decltype(detail::swap_adl_tests::uses_std(0))::value || - (std::is_move_assignable::value && - std::is_move_constructible::value))> {}; - -template -struct is_swappable - : std::integral_constant< - bool, - decltype(detail::swap_adl_tests::can_swap(0))::value && - (!decltype(detail::swap_adl_tests::uses_std( - 0))::value || - is_swappable::value)> {}; - -template -struct is_nothrow_swappable - : std::integral_constant< - bool, - is_swappable::value && - ((decltype(detail::swap_adl_tests::uses_std(0))::value && - detail::swap_adl_tests::is_std_swap_noexcept::value) || - (!decltype(detail::swap_adl_tests::uses_std(0))::value && - detail::swap_adl_tests::is_adl_swap_noexcept::value))> {}; -#endif -#endif - -// Trait for checking if a type is a tl::expected -template struct is_expected_impl : std::false_type {}; -template -struct is_expected_impl> : std::true_type {}; -template using is_expected = is_expected_impl>; - -template -using expected_enable_forward_value = detail::enable_if_t< - std::is_constructible::value && - !std::is_same, in_place_t>::value && - !std::is_same, detail::decay_t>::value && - !std::is_same, detail::decay_t>::value>; - -template -using expected_enable_from_other = detail::enable_if_t< - std::is_constructible::value && - std::is_constructible::value && - !std::is_constructible &>::value && - !std::is_constructible &&>::value && - !std::is_constructible &>::value && - !std::is_constructible &&>::value && - !std::is_convertible &, T>::value && - !std::is_convertible &&, T>::value && - !std::is_convertible &, T>::value && - !std::is_convertible &&, T>::value>; - -template -using is_void_or = conditional_t::value, std::true_type, U>; - -template -using is_copy_constructible_or_void = - is_void_or>; - -template -using is_move_constructible_or_void = - is_void_or>; - -template -using is_copy_assignable_or_void = is_void_or>; - -template -using is_move_assignable_or_void = is_void_or>; - -} // namespace detail - -namespace detail { -struct no_init_t {}; -static constexpr no_init_t no_init{}; - -// Implements the storage of the values, and ensures that the destructor is -// trivial if it can be. -// -// This specialization is for where neither `T` or `E` is trivially -// destructible, so the destructors must be called on destruction of the -// `expected` -template ::value, - bool = std::is_trivially_destructible::value> -struct expected_storage_base { - constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {} - constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {} - - template ::value> * = - nullptr> - constexpr expected_storage_base(in_place_t, Args &&...args) - : m_val(std::forward(args)...), m_has_val(true) {} - - template &, Args &&...>::value> * = nullptr> - constexpr expected_storage_base(in_place_t, std::initializer_list il, - Args &&...args) - : m_val(il, std::forward(args)...), m_has_val(true) {} - template ::value> * = - nullptr> - constexpr explicit expected_storage_base(unexpect_t, Args &&...args) - : m_unexpect(std::forward(args)...), m_has_val(false) {} - - template &, Args &&...>::value> * = nullptr> - constexpr explicit expected_storage_base(unexpect_t, - std::initializer_list il, - Args &&...args) - : m_unexpect(il, std::forward(args)...), m_has_val(false) {} - - ~expected_storage_base() { - if (m_has_val) { - m_val.~T(); - } else { - m_unexpect.~unexpected(); - } - } - union { - T m_val; - unexpected m_unexpect; - char m_no_init; - }; - bool m_has_val; -}; - -// This specialization is for when both `T` and `E` are trivially-destructible, -// so the destructor of the `expected` can be trivial. -template struct expected_storage_base { - constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {} - constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {} - - template ::value> * = - nullptr> - constexpr expected_storage_base(in_place_t, Args &&...args) - : m_val(std::forward(args)...), m_has_val(true) {} - - template &, Args &&...>::value> * = nullptr> - constexpr expected_storage_base(in_place_t, std::initializer_list il, - Args &&...args) - : m_val(il, std::forward(args)...), m_has_val(true) {} - template ::value> * = - nullptr> - constexpr explicit expected_storage_base(unexpect_t, Args &&...args) - : m_unexpect(std::forward(args)...), m_has_val(false) {} - - template &, Args &&...>::value> * = nullptr> - constexpr explicit expected_storage_base(unexpect_t, - std::initializer_list il, - Args &&...args) - : m_unexpect(il, std::forward(args)...), m_has_val(false) {} - - expected_storage_base(const expected_storage_base &) = default; - expected_storage_base(expected_storage_base &&) = default; - expected_storage_base &operator=(const expected_storage_base &) = default; - expected_storage_base &operator=(expected_storage_base &&) = default; - ~expected_storage_base() = default; - union { - T m_val; - unexpected m_unexpect; - char m_no_init; - }; - bool m_has_val; -}; - -// T is trivial, E is not. -template struct expected_storage_base { - constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {} - TL_EXPECTED_MSVC2015_CONSTEXPR expected_storage_base(no_init_t) - : m_no_init(), m_has_val(false) {} - - template ::value> * = - nullptr> - constexpr expected_storage_base(in_place_t, Args &&...args) - : m_val(std::forward(args)...), m_has_val(true) {} - - template &, Args &&...>::value> * = nullptr> - constexpr expected_storage_base(in_place_t, std::initializer_list il, - Args &&...args) - : m_val(il, std::forward(args)...), m_has_val(true) {} - template ::value> * = - nullptr> - constexpr explicit expected_storage_base(unexpect_t, Args &&...args) - : m_unexpect(std::forward(args)...), m_has_val(false) {} - - template &, Args &&...>::value> * = nullptr> - constexpr explicit expected_storage_base(unexpect_t, - std::initializer_list il, - Args &&...args) - : m_unexpect(il, std::forward(args)...), m_has_val(false) {} - - expected_storage_base(const expected_storage_base &) = default; - expected_storage_base(expected_storage_base &&) = default; - expected_storage_base &operator=(const expected_storage_base &) = default; - expected_storage_base &operator=(expected_storage_base &&) = default; - ~expected_storage_base() { - if (!m_has_val) { - m_unexpect.~unexpected(); - } - } - - union { - T m_val; - unexpected m_unexpect; - char m_no_init; - }; - bool m_has_val; -}; - -// E is trivial, T is not. -template struct expected_storage_base { - constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {} - constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {} - - template ::value> * = - nullptr> - constexpr expected_storage_base(in_place_t, Args &&...args) - : m_val(std::forward(args)...), m_has_val(true) {} - - template &, Args &&...>::value> * = nullptr> - constexpr expected_storage_base(in_place_t, std::initializer_list il, - Args &&...args) - : m_val(il, std::forward(args)...), m_has_val(true) {} - template ::value> * = - nullptr> - constexpr explicit expected_storage_base(unexpect_t, Args &&...args) - : m_unexpect(std::forward(args)...), m_has_val(false) {} - - template &, Args &&...>::value> * = nullptr> - constexpr explicit expected_storage_base(unexpect_t, - std::initializer_list il, - Args &&...args) - : m_unexpect(il, std::forward(args)...), m_has_val(false) {} - - expected_storage_base(const expected_storage_base &) = default; - expected_storage_base(expected_storage_base &&) = default; - expected_storage_base &operator=(const expected_storage_base &) = default; - expected_storage_base &operator=(expected_storage_base &&) = default; - ~expected_storage_base() { - if (m_has_val) { - m_val.~T(); - } - } - union { - T m_val; - unexpected m_unexpect; - char m_no_init; - }; - bool m_has_val; -}; - -// `T` is `void`, `E` is trivially-destructible -template struct expected_storage_base { - #if __GNUC__ <= 5 - //no constexpr for GCC 4/5 bug - #else - TL_EXPECTED_MSVC2015_CONSTEXPR - #endif - expected_storage_base() : m_has_val(true) {} - - constexpr expected_storage_base(no_init_t) : m_val(), m_has_val(false) {} - - constexpr expected_storage_base(in_place_t) : m_has_val(true) {} - - template ::value> * = - nullptr> - constexpr explicit expected_storage_base(unexpect_t, Args &&...args) - : m_unexpect(std::forward(args)...), m_has_val(false) {} - - template &, Args &&...>::value> * = nullptr> - constexpr explicit expected_storage_base(unexpect_t, - std::initializer_list il, - Args &&...args) - : m_unexpect(il, std::forward(args)...), m_has_val(false) {} - - expected_storage_base(const expected_storage_base &) = default; - expected_storage_base(expected_storage_base &&) = default; - expected_storage_base &operator=(const expected_storage_base &) = default; - expected_storage_base &operator=(expected_storage_base &&) = default; - ~expected_storage_base() = default; - struct dummy {}; - union { - unexpected m_unexpect; - dummy m_val; - }; - bool m_has_val; -}; - -// `T` is `void`, `E` is not trivially-destructible -template struct expected_storage_base { - constexpr expected_storage_base() : m_dummy(), m_has_val(true) {} - constexpr expected_storage_base(no_init_t) : m_dummy(), m_has_val(false) {} - - constexpr expected_storage_base(in_place_t) : m_dummy(), m_has_val(true) {} - - template ::value> * = - nullptr> - constexpr explicit expected_storage_base(unexpect_t, Args &&...args) - : m_unexpect(std::forward(args)...), m_has_val(false) {} - - template &, Args &&...>::value> * = nullptr> - constexpr explicit expected_storage_base(unexpect_t, - std::initializer_list il, - Args &&...args) - : m_unexpect(il, std::forward(args)...), m_has_val(false) {} - - expected_storage_base(const expected_storage_base &) = default; - expected_storage_base(expected_storage_base &&) = default; - expected_storage_base &operator=(const expected_storage_base &) = default; - expected_storage_base &operator=(expected_storage_base &&) = default; - ~expected_storage_base() { - if (!m_has_val) { - m_unexpect.~unexpected(); - } - } - - union { - unexpected m_unexpect; - char m_dummy; - }; - bool m_has_val; -}; - -// This base class provides some handy member functions which can be used in -// further derived classes -template -struct expected_operations_base : expected_storage_base { - using expected_storage_base::expected_storage_base; - - template void construct(Args &&...args) noexcept { - new (std::addressof(this->m_val)) T(std::forward(args)...); - this->m_has_val = true; - } - - template void construct_with(Rhs &&rhs) noexcept { - new (std::addressof(this->m_val)) T(std::forward(rhs).get()); - this->m_has_val = true; - } - - template void construct_error(Args &&...args) noexcept { - new (std::addressof(this->m_unexpect)) - unexpected(std::forward(args)...); - this->m_has_val = false; - } - -#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED - - // These assign overloads ensure that the most efficient assignment - // implementation is used while maintaining the strong exception guarantee. - // The problematic case is where rhs has a value, but *this does not. - // - // This overload handles the case where we can just copy-construct `T` - // directly into place without throwing. - template ::value> - * = nullptr> - void assign(const expected_operations_base &rhs) noexcept { - if (!this->m_has_val && rhs.m_has_val) { - geterr().~unexpected(); - construct(rhs.get()); - } else { - assign_common(rhs); - } - } - - // This overload handles the case where we can attempt to create a copy of - // `T`, then no-throw move it into place if the copy was successful. - template ::value && - std::is_nothrow_move_constructible::value> - * = nullptr> - void assign(const expected_operations_base &rhs) noexcept { - if (!this->m_has_val && rhs.m_has_val) { - T tmp = rhs.get(); - geterr().~unexpected(); - construct(std::move(tmp)); - } else { - assign_common(rhs); - } - } - - // This overload is the worst-case, where we have to move-construct the - // unexpected value into temporary storage, then try to copy the T into place. - // If the construction succeeds, then everything is fine, but if it throws, - // then we move the old unexpected value back into place before rethrowing the - // exception. - template ::value && - !std::is_nothrow_move_constructible::value> - * = nullptr> - void assign(const expected_operations_base &rhs) { - if (!this->m_has_val && rhs.m_has_val) { - auto tmp = std::move(geterr()); - geterr().~unexpected(); - -#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED - try { - construct(rhs.get()); - } catch (...) { - geterr() = std::move(tmp); - throw; - } -#else - construct(rhs.get()); -#endif - } else { - assign_common(rhs); - } - } - - // These overloads do the same as above, but for rvalues - template ::value> - * = nullptr> - void assign(expected_operations_base &&rhs) noexcept { - if (!this->m_has_val && rhs.m_has_val) { - geterr().~unexpected(); - construct(std::move(rhs).get()); - } else { - assign_common(std::move(rhs)); - } - } - - template ::value> - * = nullptr> - void assign(expected_operations_base &&rhs) { - if (!this->m_has_val && rhs.m_has_val) { - auto tmp = std::move(geterr()); - geterr().~unexpected(); -#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED - try { - construct(std::move(rhs).get()); - } catch (...) { - geterr() = std::move(tmp); - throw; - } -#else - construct(std::move(rhs).get()); -#endif - } else { - assign_common(std::move(rhs)); - } - } - -#else - - // If exceptions are disabled then we can just copy-construct - void assign(const expected_operations_base &rhs) noexcept { - if (!this->m_has_val && rhs.m_has_val) { - geterr().~unexpected(); - construct(rhs.get()); - } else { - assign_common(rhs); - } - } - - void assign(expected_operations_base &&rhs) noexcept { - if (!this->m_has_val && rhs.m_has_val) { - geterr().~unexpected(); - construct(std::move(rhs).get()); - } else { - assign_common(std::move(rhs)); - } - } - -#endif - - // The common part of move/copy assigning - template void assign_common(Rhs &&rhs) { - if (this->m_has_val) { - if (rhs.m_has_val) { - get() = std::forward(rhs).get(); - } else { - destroy_val(); - construct_error(std::forward(rhs).geterr()); - } - } else { - if (!rhs.m_has_val) { - geterr() = std::forward(rhs).geterr(); - } - } - } - - bool has_value() const { return this->m_has_val; } - - TL_EXPECTED_11_CONSTEXPR T &get() & { return this->m_val; } - constexpr const T &get() const & { return this->m_val; } - TL_EXPECTED_11_CONSTEXPR T &&get() && { return std::move(this->m_val); } -#ifndef TL_EXPECTED_NO_CONSTRR - constexpr const T &&get() const && { return std::move(this->m_val); } -#endif - - TL_EXPECTED_11_CONSTEXPR unexpected &geterr() & { - return this->m_unexpect; - } - constexpr const unexpected &geterr() const & { return this->m_unexpect; } - TL_EXPECTED_11_CONSTEXPR unexpected &&geterr() && { - return std::move(this->m_unexpect); - } -#ifndef TL_EXPECTED_NO_CONSTRR - constexpr const unexpected &&geterr() const && { - return std::move(this->m_unexpect); - } -#endif - - TL_EXPECTED_11_CONSTEXPR void destroy_val() { get().~T(); } -}; - -// This base class provides some handy member functions which can be used in -// further derived classes -template -struct expected_operations_base : expected_storage_base { - using expected_storage_base::expected_storage_base; - - template void construct() noexcept { this->m_has_val = true; } - - // This function doesn't use its argument, but needs it so that code in - // levels above this can work independently of whether T is void - template void construct_with(Rhs &&) noexcept { - this->m_has_val = true; - } - - template void construct_error(Args &&...args) noexcept { - new (std::addressof(this->m_unexpect)) - unexpected(std::forward(args)...); - this->m_has_val = false; - } - - template void assign(Rhs &&rhs) noexcept { - if (!this->m_has_val) { - if (rhs.m_has_val) { - geterr().~unexpected(); - construct(); - } else { - geterr() = std::forward(rhs).geterr(); - } - } else { - if (!rhs.m_has_val) { - construct_error(std::forward(rhs).geterr()); - } - } - } - - bool has_value() const { return this->m_has_val; } - - TL_EXPECTED_11_CONSTEXPR unexpected &geterr() & { - return this->m_unexpect; - } - constexpr const unexpected &geterr() const & { return this->m_unexpect; } - TL_EXPECTED_11_CONSTEXPR unexpected &&geterr() && { - return std::move(this->m_unexpect); - } -#ifndef TL_EXPECTED_NO_CONSTRR - constexpr const unexpected &&geterr() const && { - return std::move(this->m_unexpect); - } -#endif - - TL_EXPECTED_11_CONSTEXPR void destroy_val() { - // no-op - } -}; - -// This class manages conditionally having a trivial copy constructor -// This specialization is for when T and E are trivially copy constructible -template :: - value &&TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(E)::value, - bool = (is_copy_constructible_or_void::value && - std::is_copy_constructible::value)> -struct expected_copy_base : expected_operations_base { - using expected_operations_base::expected_operations_base; -}; - -// This specialization is for when T or E are non-trivially copy constructible -template -struct expected_copy_base : expected_operations_base { - using expected_operations_base::expected_operations_base; - - expected_copy_base() = default; - expected_copy_base(const expected_copy_base &rhs) - : expected_operations_base(no_init) { - if (rhs.has_value()) { - this->construct_with(rhs); - } else { - this->construct_error(rhs.geterr()); - } - } - - expected_copy_base(expected_copy_base &&rhs) = default; - expected_copy_base &operator=(const expected_copy_base &rhs) = default; - expected_copy_base &operator=(expected_copy_base &&rhs) = default; -}; - -// This class manages conditionally having a trivial move constructor -// Unfortunately there's no way to achieve this in GCC < 5 AFAIK, since it -// doesn't implement an analogue to std::is_trivially_move_constructible. We -// have to make do with a non-trivial move constructor even if T is trivially -// move constructible -#ifndef TL_EXPECTED_GCC49 -template >::value - &&std::is_trivially_move_constructible::value> -struct expected_move_base : expected_copy_base { - using expected_copy_base::expected_copy_base; -}; -#else -template struct expected_move_base; -#endif -template -struct expected_move_base : expected_copy_base { - using expected_copy_base::expected_copy_base; - - expected_move_base() = default; - expected_move_base(const expected_move_base &rhs) = default; - - expected_move_base(expected_move_base &&rhs) noexcept( - std::is_nothrow_move_constructible::value) - : expected_copy_base(no_init) { - if (rhs.has_value()) { - this->construct_with(std::move(rhs)); - } else { - this->construct_error(std::move(rhs.geterr())); - } - } - expected_move_base &operator=(const expected_move_base &rhs) = default; - expected_move_base &operator=(expected_move_base &&rhs) = default; -}; - -// This class manages conditionally having a trivial copy assignment operator -template >::value - &&TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(E)::value - &&TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(E)::value - &&TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(E)::value, - bool = (is_copy_constructible_or_void::value && - std::is_copy_constructible::value && - is_copy_assignable_or_void::value && - std::is_copy_assignable::value)> -struct expected_copy_assign_base : expected_move_base { - using expected_move_base::expected_move_base; -}; - -template -struct expected_copy_assign_base : expected_move_base { - using expected_move_base::expected_move_base; - - expected_copy_assign_base() = default; - expected_copy_assign_base(const expected_copy_assign_base &rhs) = default; - - expected_copy_assign_base(expected_copy_assign_base &&rhs) = default; - expected_copy_assign_base &operator=(const expected_copy_assign_base &rhs) { - this->assign(rhs); - return *this; - } - expected_copy_assign_base & - operator=(expected_copy_assign_base &&rhs) = default; -}; - -// This class manages conditionally having a trivial move assignment operator -// Unfortunately there's no way to achieve this in GCC < 5 AFAIK, since it -// doesn't implement an analogue to std::is_trivially_move_assignable. We have -// to make do with a non-trivial move assignment operator even if T is trivially -// move assignable -#ifndef TL_EXPECTED_GCC49 -template , - std::is_trivially_move_constructible, - std::is_trivially_move_assignable>>:: - value &&std::is_trivially_destructible::value - &&std::is_trivially_move_constructible::value - &&std::is_trivially_move_assignable::value> -struct expected_move_assign_base : expected_copy_assign_base { - using expected_copy_assign_base::expected_copy_assign_base; -}; -#else -template struct expected_move_assign_base; -#endif - -template -struct expected_move_assign_base - : expected_copy_assign_base { - using expected_copy_assign_base::expected_copy_assign_base; - - expected_move_assign_base() = default; - expected_move_assign_base(const expected_move_assign_base &rhs) = default; - - expected_move_assign_base(expected_move_assign_base &&rhs) = default; - - expected_move_assign_base & - operator=(const expected_move_assign_base &rhs) = default; - - expected_move_assign_base & - operator=(expected_move_assign_base &&rhs) noexcept( - std::is_nothrow_move_constructible::value - &&std::is_nothrow_move_assignable::value) { - this->assign(std::move(rhs)); - return *this; - } -}; - -// expected_delete_ctor_base will conditionally delete copy and move -// constructors depending on whether T is copy/move constructible -template ::value && - std::is_copy_constructible::value), - bool EnableMove = (is_move_constructible_or_void::value && - std::is_move_constructible::value)> -struct expected_delete_ctor_base { - expected_delete_ctor_base() = default; - expected_delete_ctor_base(const expected_delete_ctor_base &) = default; - expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = default; - expected_delete_ctor_base & - operator=(const expected_delete_ctor_base &) = default; - expected_delete_ctor_base & - operator=(expected_delete_ctor_base &&) noexcept = default; -}; - -template -struct expected_delete_ctor_base { - expected_delete_ctor_base() = default; - expected_delete_ctor_base(const expected_delete_ctor_base &) = default; - expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = delete; - expected_delete_ctor_base & - operator=(const expected_delete_ctor_base &) = default; - expected_delete_ctor_base & - operator=(expected_delete_ctor_base &&) noexcept = default; -}; - -template -struct expected_delete_ctor_base { - expected_delete_ctor_base() = default; - expected_delete_ctor_base(const expected_delete_ctor_base &) = delete; - expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = default; - expected_delete_ctor_base & - operator=(const expected_delete_ctor_base &) = default; - expected_delete_ctor_base & - operator=(expected_delete_ctor_base &&) noexcept = default; -}; - -template -struct expected_delete_ctor_base { - expected_delete_ctor_base() = default; - expected_delete_ctor_base(const expected_delete_ctor_base &) = delete; - expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = delete; - expected_delete_ctor_base & - operator=(const expected_delete_ctor_base &) = default; - expected_delete_ctor_base & - operator=(expected_delete_ctor_base &&) noexcept = default; -}; - -// expected_delete_assign_base will conditionally delete copy and move -// constructors depending on whether T and E are copy/move constructible + -// assignable -template ::value && - std::is_copy_constructible::value && - is_copy_assignable_or_void::value && - std::is_copy_assignable::value), - bool EnableMove = (is_move_constructible_or_void::value && - std::is_move_constructible::value && - is_move_assignable_or_void::value && - std::is_move_assignable::value)> -struct expected_delete_assign_base { - expected_delete_assign_base() = default; - expected_delete_assign_base(const expected_delete_assign_base &) = default; - expected_delete_assign_base(expected_delete_assign_base &&) noexcept = - default; - expected_delete_assign_base & - operator=(const expected_delete_assign_base &) = default; - expected_delete_assign_base & - operator=(expected_delete_assign_base &&) noexcept = default; -}; - -template -struct expected_delete_assign_base { - expected_delete_assign_base() = default; - expected_delete_assign_base(const expected_delete_assign_base &) = default; - expected_delete_assign_base(expected_delete_assign_base &&) noexcept = - default; - expected_delete_assign_base & - operator=(const expected_delete_assign_base &) = default; - expected_delete_assign_base & - operator=(expected_delete_assign_base &&) noexcept = delete; -}; - -template -struct expected_delete_assign_base { - expected_delete_assign_base() = default; - expected_delete_assign_base(const expected_delete_assign_base &) = default; - expected_delete_assign_base(expected_delete_assign_base &&) noexcept = - default; - expected_delete_assign_base & - operator=(const expected_delete_assign_base &) = delete; - expected_delete_assign_base & - operator=(expected_delete_assign_base &&) noexcept = default; -}; - -template -struct expected_delete_assign_base { - expected_delete_assign_base() = default; - expected_delete_assign_base(const expected_delete_assign_base &) = default; - expected_delete_assign_base(expected_delete_assign_base &&) noexcept = - default; - expected_delete_assign_base & - operator=(const expected_delete_assign_base &) = delete; - expected_delete_assign_base & - operator=(expected_delete_assign_base &&) noexcept = delete; -}; - -// This is needed to be able to construct the expected_default_ctor_base which -// follows, while still conditionally deleting the default constructor. -struct default_constructor_tag { - explicit constexpr default_constructor_tag() = default; -}; - -// expected_default_ctor_base will ensure that expected has a deleted default -// constructor if T is not default constructible. -// This specialization is for when T is default constructible -template ::value || std::is_void::value> -struct expected_default_ctor_base { - constexpr expected_default_ctor_base() noexcept = default; - constexpr expected_default_ctor_base( - expected_default_ctor_base const &) noexcept = default; - constexpr expected_default_ctor_base(expected_default_ctor_base &&) noexcept = - default; - expected_default_ctor_base & - operator=(expected_default_ctor_base const &) noexcept = default; - expected_default_ctor_base & - operator=(expected_default_ctor_base &&) noexcept = default; - - constexpr explicit expected_default_ctor_base(default_constructor_tag) {} -}; - -// This specialization is for when T is not default constructible -template struct expected_default_ctor_base { - constexpr expected_default_ctor_base() noexcept = delete; - constexpr expected_default_ctor_base( - expected_default_ctor_base const &) noexcept = default; - constexpr expected_default_ctor_base(expected_default_ctor_base &&) noexcept = - default; - expected_default_ctor_base & - operator=(expected_default_ctor_base const &) noexcept = default; - expected_default_ctor_base & - operator=(expected_default_ctor_base &&) noexcept = default; - - constexpr explicit expected_default_ctor_base(default_constructor_tag) {} -}; -} // namespace detail - -template class bad_expected_access : public std::exception { -public: - explicit bad_expected_access(E e) : m_val(std::move(e)) {} - - virtual const char *what() const noexcept override { - return "Bad expected access"; - } - - const E &error() const & { return m_val; } - E &error() & { return m_val; } - const E &&error() const && { return std::move(m_val); } - E &&error() && { return std::move(m_val); } - -private: - E m_val; -}; - -/// An `expected` object is an object that contains the storage for -/// another object and manages the lifetime of this contained object `T`. -/// Alternatively it could contain the storage for another unexpected object -/// `E`. The contained object may not be initialized after the expected object -/// has been initialized, and may not be destroyed before the expected object -/// has been destroyed. The initialization state of the contained object is -/// tracked by the expected object. -template -class TL_EXPECTED_NODISCARD expected : - private detail::expected_move_assign_base, - private detail::expected_delete_ctor_base, - private detail::expected_delete_assign_base, - private detail::expected_default_ctor_base { - static_assert(!std::is_reference::value, "T must not be a reference"); - static_assert(!std::is_same::type>::value, - "T must not be in_place_t"); - static_assert(!std::is_same::type>::value, - "T must not be unexpect_t"); - static_assert( - !std::is_same>::type>::value, - "T must not be unexpected"); - static_assert(!std::is_reference::value, "E must not be a reference"); - - T *valptr() { return std::addressof(this->m_val); } - const T *valptr() const { return std::addressof(this->m_val); } - unexpected *errptr() { return std::addressof(this->m_unexpect); } - const unexpected *errptr() const { - return std::addressof(this->m_unexpect); - } - - template ::value> * = nullptr> - TL_EXPECTED_11_CONSTEXPR U &val() { - return this->m_val; - } - TL_EXPECTED_11_CONSTEXPR unexpected &err() { return this->m_unexpect; } - - template ::value> * = nullptr> - constexpr const U &val() const { - return this->m_val; - } - constexpr const unexpected &err() const { return this->m_unexpect; } - - using impl_base = detail::expected_move_assign_base; - using ctor_base = detail::expected_default_ctor_base; - -public: - typedef T value_type; - typedef E error_type; - typedef unexpected unexpected_type; - -#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ - !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) - template TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) & { - return and_then_impl(*this, std::forward(f)); - } - template TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) && { - return and_then_impl(std::move(*this), std::forward(f)); - } - template constexpr auto and_then(F &&f) const & { - return and_then_impl(*this, std::forward(f)); - } - -#ifndef TL_EXPECTED_NO_CONSTRR - template constexpr auto and_then(F &&f) const && { - return and_then_impl(std::move(*this), std::forward(f)); - } -#endif - -#else - template - TL_EXPECTED_11_CONSTEXPR auto - and_then(F &&f) & -> decltype(and_then_impl(std::declval(), - std::forward(f))) { - return and_then_impl(*this, std::forward(f)); - } - template - TL_EXPECTED_11_CONSTEXPR auto - and_then(F &&f) && -> decltype(and_then_impl(std::declval(), - std::forward(f))) { - return and_then_impl(std::move(*this), std::forward(f)); - } - template - constexpr auto and_then(F &&f) const & -> decltype(and_then_impl( - std::declval(), std::forward(f))) { - return and_then_impl(*this, std::forward(f)); - } - -#ifndef TL_EXPECTED_NO_CONSTRR - template - constexpr auto and_then(F &&f) const && -> decltype(and_then_impl( - std::declval(), std::forward(f))) { - return and_then_impl(std::move(*this), std::forward(f)); - } -#endif -#endif - -#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ - !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) - template TL_EXPECTED_11_CONSTEXPR auto map(F &&f) & { - return expected_map_impl(*this, std::forward(f)); - } - template TL_EXPECTED_11_CONSTEXPR auto map(F &&f) && { - return expected_map_impl(std::move(*this), std::forward(f)); - } - template constexpr auto map(F &&f) const & { - return expected_map_impl(*this, std::forward(f)); - } - template constexpr auto map(F &&f) const && { - return expected_map_impl(std::move(*this), std::forward(f)); - } -#else - template - TL_EXPECTED_11_CONSTEXPR decltype(expected_map_impl( - std::declval(), std::declval())) - map(F &&f) & { - return expected_map_impl(*this, std::forward(f)); - } - template - TL_EXPECTED_11_CONSTEXPR decltype(expected_map_impl(std::declval(), - std::declval())) - map(F &&f) && { - return expected_map_impl(std::move(*this), std::forward(f)); - } - template - constexpr decltype(expected_map_impl(std::declval(), - std::declval())) - map(F &&f) const & { - return expected_map_impl(*this, std::forward(f)); - } - -#ifndef TL_EXPECTED_NO_CONSTRR - template - constexpr decltype(expected_map_impl(std::declval(), - std::declval())) - map(F &&f) const && { - return expected_map_impl(std::move(*this), std::forward(f)); - } -#endif -#endif - -#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ - !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) - template TL_EXPECTED_11_CONSTEXPR auto transform(F &&f) & { - return expected_map_impl(*this, std::forward(f)); - } - template TL_EXPECTED_11_CONSTEXPR auto transform(F &&f) && { - return expected_map_impl(std::move(*this), std::forward(f)); - } - template constexpr auto transform(F &&f) const & { - return expected_map_impl(*this, std::forward(f)); - } - template constexpr auto transform(F &&f) const && { - return expected_map_impl(std::move(*this), std::forward(f)); - } -#else - template - TL_EXPECTED_11_CONSTEXPR decltype(expected_map_impl( - std::declval(), std::declval())) - transform(F &&f) & { - return expected_map_impl(*this, std::forward(f)); - } - template - TL_EXPECTED_11_CONSTEXPR decltype(expected_map_impl(std::declval(), - std::declval())) - transform(F &&f) && { - return expected_map_impl(std::move(*this), std::forward(f)); - } - template - constexpr decltype(expected_map_impl(std::declval(), - std::declval())) - transform(F &&f) const & { - return expected_map_impl(*this, std::forward(f)); - } - -#ifndef TL_EXPECTED_NO_CONSTRR - template - constexpr decltype(expected_map_impl(std::declval(), - std::declval())) - transform(F &&f) const && { - return expected_map_impl(std::move(*this), std::forward(f)); - } -#endif -#endif - -#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ - !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) - template TL_EXPECTED_11_CONSTEXPR auto map_error(F &&f) & { - return map_error_impl(*this, std::forward(f)); - } - template TL_EXPECTED_11_CONSTEXPR auto map_error(F &&f) && { - return map_error_impl(std::move(*this), std::forward(f)); - } - template constexpr auto map_error(F &&f) const & { - return map_error_impl(*this, std::forward(f)); - } - template constexpr auto map_error(F &&f) const && { - return map_error_impl(std::move(*this), std::forward(f)); - } -#else - template - TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval(), - std::declval())) - map_error(F &&f) & { - return map_error_impl(*this, std::forward(f)); - } - template - TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval(), - std::declval())) - map_error(F &&f) && { - return map_error_impl(std::move(*this), std::forward(f)); - } - template - constexpr decltype(map_error_impl(std::declval(), - std::declval())) - map_error(F &&f) const & { - return map_error_impl(*this, std::forward(f)); - } - -#ifndef TL_EXPECTED_NO_CONSTRR - template - constexpr decltype(map_error_impl(std::declval(), - std::declval())) - map_error(F &&f) const && { - return map_error_impl(std::move(*this), std::forward(f)); - } -#endif -#endif -#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ - !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) - template TL_EXPECTED_11_CONSTEXPR auto transform_error(F &&f) & { - return map_error_impl(*this, std::forward(f)); - } - template TL_EXPECTED_11_CONSTEXPR auto transform_error(F &&f) && { - return map_error_impl(std::move(*this), std::forward(f)); - } - template constexpr auto transform_error(F &&f) const & { - return map_error_impl(*this, std::forward(f)); - } - template constexpr auto transform_error(F &&f) const && { - return map_error_impl(std::move(*this), std::forward(f)); - } -#else - template - TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval(), - std::declval())) - transform_error(F &&f) & { - return map_error_impl(*this, std::forward(f)); - } - template - TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval(), - std::declval())) - transform_error(F &&f) && { - return map_error_impl(std::move(*this), std::forward(f)); - } - template - constexpr decltype(map_error_impl(std::declval(), - std::declval())) - transform_error(F &&f) const & { - return map_error_impl(*this, std::forward(f)); - } - -#ifndef TL_EXPECTED_NO_CONSTRR - template - constexpr decltype(map_error_impl(std::declval(), - std::declval())) - transform_error(F &&f) const && { - return map_error_impl(std::move(*this), std::forward(f)); - } -#endif -#endif - template expected TL_EXPECTED_11_CONSTEXPR or_else(F &&f) & { - return or_else_impl(*this, std::forward(f)); - } - - template expected TL_EXPECTED_11_CONSTEXPR or_else(F &&f) && { - return or_else_impl(std::move(*this), std::forward(f)); - } - - template expected constexpr or_else(F &&f) const & { - return or_else_impl(*this, std::forward(f)); - } - -#ifndef TL_EXPECTED_NO_CONSTRR - template expected constexpr or_else(F &&f) const && { - return or_else_impl(std::move(*this), std::forward(f)); - } -#endif - constexpr expected() = default; - constexpr expected(const expected &rhs) = default; - constexpr expected(expected &&rhs) = default; - expected &operator=(const expected &rhs) = default; - expected &operator=(expected &&rhs) = default; - - template ::value> * = - nullptr> - constexpr expected(in_place_t, Args &&...args) - : impl_base(in_place, std::forward(args)...), - ctor_base(detail::default_constructor_tag{}) {} - - template &, Args &&...>::value> * = nullptr> - constexpr expected(in_place_t, std::initializer_list il, Args &&...args) - : impl_base(in_place, il, std::forward(args)...), - ctor_base(detail::default_constructor_tag{}) {} - - template ::value> * = - nullptr, - detail::enable_if_t::value> * = - nullptr> - explicit constexpr expected(const unexpected &e) - : impl_base(unexpect, e.value()), - ctor_base(detail::default_constructor_tag{}) {} - - template < - class G = E, - detail::enable_if_t::value> * = - nullptr, - detail::enable_if_t::value> * = nullptr> - constexpr expected(unexpected const &e) - : impl_base(unexpect, e.value()), - ctor_base(detail::default_constructor_tag{}) {} - - template < - class G = E, - detail::enable_if_t::value> * = nullptr, - detail::enable_if_t::value> * = nullptr> - explicit constexpr expected(unexpected &&e) noexcept( - std::is_nothrow_constructible::value) - : impl_base(unexpect, std::move(e.value())), - ctor_base(detail::default_constructor_tag{}) {} - - template < - class G = E, - detail::enable_if_t::value> * = nullptr, - detail::enable_if_t::value> * = nullptr> - constexpr expected(unexpected &&e) noexcept( - std::is_nothrow_constructible::value) - : impl_base(unexpect, std::move(e.value())), - ctor_base(detail::default_constructor_tag{}) {} - - template ::value> * = - nullptr> - constexpr explicit expected(unexpect_t, Args &&...args) - : impl_base(unexpect, std::forward(args)...), - ctor_base(detail::default_constructor_tag{}) {} - - template &, Args &&...>::value> * = nullptr> - constexpr explicit expected(unexpect_t, std::initializer_list il, - Args &&...args) - : impl_base(unexpect, il, std::forward(args)...), - ctor_base(detail::default_constructor_tag{}) {} - - template ::value && - std::is_convertible::value)> * = - nullptr, - detail::expected_enable_from_other - * = nullptr> - explicit TL_EXPECTED_11_CONSTEXPR expected(const expected &rhs) - : ctor_base(detail::default_constructor_tag{}) { - if (rhs.has_value()) { - this->construct(*rhs); - } else { - this->construct_error(rhs.error()); - } - } - - template ::value && - std::is_convertible::value)> * = - nullptr, - detail::expected_enable_from_other - * = nullptr> - TL_EXPECTED_11_CONSTEXPR expected(const expected &rhs) - : ctor_base(detail::default_constructor_tag{}) { - if (rhs.has_value()) { - this->construct(*rhs); - } else { - this->construct_error(rhs.error()); - } - } - - template < - class U, class G, - detail::enable_if_t::value && - std::is_convertible::value)> * = nullptr, - detail::expected_enable_from_other * = nullptr> - explicit TL_EXPECTED_11_CONSTEXPR expected(expected &&rhs) - : ctor_base(detail::default_constructor_tag{}) { - if (rhs.has_value()) { - this->construct(std::move(*rhs)); - } else { - this->construct_error(std::move(rhs.error())); - } - } - - template < - class U, class G, - detail::enable_if_t<(std::is_convertible::value && - std::is_convertible::value)> * = nullptr, - detail::expected_enable_from_other * = nullptr> - TL_EXPECTED_11_CONSTEXPR expected(expected &&rhs) - : ctor_base(detail::default_constructor_tag{}) { - if (rhs.has_value()) { - this->construct(std::move(*rhs)); - } else { - this->construct_error(std::move(rhs.error())); - } - } - - template < - class U = T, - detail::enable_if_t::value> * = nullptr, - detail::expected_enable_forward_value * = nullptr> - explicit TL_EXPECTED_MSVC2015_CONSTEXPR expected(U &&v) - : expected(in_place, std::forward(v)) {} - - template < - class U = T, - detail::enable_if_t::value> * = nullptr, - detail::expected_enable_forward_value * = nullptr> - TL_EXPECTED_MSVC2015_CONSTEXPR expected(U &&v) - : expected(in_place, std::forward(v)) {} - - template < - class U = T, class G = T, - detail::enable_if_t::value> * = - nullptr, - detail::enable_if_t::value> * = nullptr, - detail::enable_if_t< - (!std::is_same, detail::decay_t>::value && - !detail::conjunction, - std::is_same>>::value && - std::is_constructible::value && - std::is_assignable::value && - std::is_nothrow_move_constructible::value)> * = nullptr> - expected &operator=(U &&v) { - if (has_value()) { - val() = std::forward(v); - } else { - err().~unexpected(); - ::new (valptr()) T(std::forward(v)); - this->m_has_val = true; - } - - return *this; - } - - template < - class U = T, class G = T, - detail::enable_if_t::value> * = - nullptr, - detail::enable_if_t::value> * = nullptr, - detail::enable_if_t< - (!std::is_same, detail::decay_t>::value && - !detail::conjunction, - std::is_same>>::value && - std::is_constructible::value && - std::is_assignable::value && - std::is_nothrow_move_constructible::value)> * = nullptr> - expected &operator=(U &&v) { - if (has_value()) { - val() = std::forward(v); - } else { - auto tmp = std::move(err()); - err().~unexpected(); - -#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED - try { - ::new (valptr()) T(std::forward(v)); - this->m_has_val = true; - } catch (...) { - err() = std::move(tmp); - throw; - } -#else - ::new (valptr()) T(std::forward(v)); - this->m_has_val = true; -#endif - } - - return *this; - } - - template ::value && - std::is_assignable::value> * = nullptr> - expected &operator=(const unexpected &rhs) { - if (!has_value()) { - err() = rhs; - } else { - this->destroy_val(); - ::new (errptr()) unexpected(rhs); - this->m_has_val = false; - } - - return *this; - } - - template ::value && - std::is_move_assignable::value> * = nullptr> - expected &operator=(unexpected &&rhs) noexcept { - if (!has_value()) { - err() = std::move(rhs); - } else { - this->destroy_val(); - ::new (errptr()) unexpected(std::move(rhs)); - this->m_has_val = false; - } - - return *this; - } - - template ::value> * = nullptr> - void emplace(Args &&...args) { - if (has_value()) { - val().~T(); - } else { - err().~unexpected(); - this->m_has_val = true; - } - ::new (valptr()) T(std::forward(args)...); - } - - template ::value> * = nullptr> - void emplace(Args &&...args) { - if (has_value()) { - val().~T(); - ::new (valptr()) T(std::forward(args)...); - } else { - auto tmp = std::move(err()); - err().~unexpected(); - -#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED - try { - ::new (valptr()) T(std::forward(args)...); - this->m_has_val = true; - } catch (...) { - err() = std::move(tmp); - throw; - } -#else - ::new (valptr()) T(std::forward(args)...); - this->m_has_val = true; -#endif - } - } - - template &, Args &&...>::value> * = nullptr> - void emplace(std::initializer_list il, Args &&...args) { - if (has_value()) { - T t(il, std::forward(args)...); - val() = std::move(t); - } else { - err().~unexpected(); - ::new (valptr()) T(il, std::forward(args)...); - this->m_has_val = true; - } - } - - template &, Args &&...>::value> * = nullptr> - void emplace(std::initializer_list il, Args &&...args) { - if (has_value()) { - T t(il, std::forward(args)...); - val() = std::move(t); - } else { - auto tmp = std::move(err()); - err().~unexpected(); - -#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED - try { - ::new (valptr()) T(il, std::forward(args)...); - this->m_has_val = true; - } catch (...) { - err() = std::move(tmp); - throw; - } -#else - ::new (valptr()) T(il, std::forward(args)...); - this->m_has_val = true; -#endif - } - } - -private: - using t_is_void = std::true_type; - using t_is_not_void = std::false_type; - using t_is_nothrow_move_constructible = std::true_type; - using move_constructing_t_can_throw = std::false_type; - using e_is_nothrow_move_constructible = std::true_type; - using move_constructing_e_can_throw = std::false_type; - - void swap_where_both_have_value(expected & /*rhs*/, t_is_void) noexcept { - // swapping void is a no-op - } - - void swap_where_both_have_value(expected &rhs, t_is_not_void) { - using std::swap; - swap(val(), rhs.val()); - } - - void swap_where_only_one_has_value(expected &rhs, t_is_void) noexcept( - std::is_nothrow_move_constructible::value) { - ::new (errptr()) unexpected_type(std::move(rhs.err())); - rhs.err().~unexpected_type(); - std::swap(this->m_has_val, rhs.m_has_val); - } - - void swap_where_only_one_has_value(expected &rhs, t_is_not_void) { - swap_where_only_one_has_value_and_t_is_not_void( - rhs, typename std::is_nothrow_move_constructible::type{}, - typename std::is_nothrow_move_constructible::type{}); - } - - void swap_where_only_one_has_value_and_t_is_not_void( - expected &rhs, t_is_nothrow_move_constructible, - e_is_nothrow_move_constructible) noexcept { - auto temp = std::move(val()); - val().~T(); - ::new (errptr()) unexpected_type(std::move(rhs.err())); - rhs.err().~unexpected_type(); - ::new (rhs.valptr()) T(std::move(temp)); - std::swap(this->m_has_val, rhs.m_has_val); - } - - void swap_where_only_one_has_value_and_t_is_not_void( - expected &rhs, t_is_nothrow_move_constructible, - move_constructing_e_can_throw) { - auto temp = std::move(val()); - val().~T(); -#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED - try { - ::new (errptr()) unexpected_type(std::move(rhs.err())); - rhs.err().~unexpected_type(); - ::new (rhs.valptr()) T(std::move(temp)); - std::swap(this->m_has_val, rhs.m_has_val); - } catch (...) { - val() = std::move(temp); - throw; - } -#else - ::new (errptr()) unexpected_type(std::move(rhs.err())); - rhs.err().~unexpected_type(); - ::new (rhs.valptr()) T(std::move(temp)); - std::swap(this->m_has_val, rhs.m_has_val); -#endif - } - - void swap_where_only_one_has_value_and_t_is_not_void( - expected &rhs, move_constructing_t_can_throw, - e_is_nothrow_move_constructible) { - auto temp = std::move(rhs.err()); - rhs.err().~unexpected_type(); -#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED - try { - ::new (rhs.valptr()) T(std::move(val())); - val().~T(); - ::new (errptr()) unexpected_type(std::move(temp)); - std::swap(this->m_has_val, rhs.m_has_val); - } catch (...) { - rhs.err() = std::move(temp); - throw; - } -#else - ::new (rhs.valptr()) T(std::move(val())); - val().~T(); - ::new (errptr()) unexpected_type(std::move(temp)); - std::swap(this->m_has_val, rhs.m_has_val); -#endif - } - -public: - template - detail::enable_if_t::value && - detail::is_swappable::value && - (std::is_nothrow_move_constructible::value || - std::is_nothrow_move_constructible::value)> - swap(expected &rhs) noexcept( - std::is_nothrow_move_constructible::value - &&detail::is_nothrow_swappable::value - &&std::is_nothrow_move_constructible::value - &&detail::is_nothrow_swappable::value) { - if (has_value() && rhs.has_value()) { - swap_where_both_have_value(rhs, typename std::is_void::type{}); - } else if (!has_value() && rhs.has_value()) { - rhs.swap(*this); - } else if (has_value()) { - swap_where_only_one_has_value(rhs, typename std::is_void::type{}); - } else { - using std::swap; - swap(err(), rhs.err()); - } - } - - constexpr const T *operator->() const { - TL_ASSERT(has_value()); - return valptr(); - } - TL_EXPECTED_11_CONSTEXPR T *operator->() { - TL_ASSERT(has_value()); - return valptr(); - } - - template ::value> * = nullptr> - constexpr const U &operator*() const & { - TL_ASSERT(has_value()); - return val(); - } - template ::value> * = nullptr> - TL_EXPECTED_11_CONSTEXPR U &operator*() & { - TL_ASSERT(has_value()); - return val(); - } - template ::value> * = nullptr> - constexpr const U &&operator*() const && { - TL_ASSERT(has_value()); - return std::move(val()); - } - template ::value> * = nullptr> - TL_EXPECTED_11_CONSTEXPR U &&operator*() && { - TL_ASSERT(has_value()); - return std::move(val()); - } - - constexpr bool has_value() const noexcept { return this->m_has_val; } - constexpr explicit operator bool() const noexcept { return this->m_has_val; } - - template ::value> * = nullptr> - TL_EXPECTED_11_CONSTEXPR const U &value() const & { - if (!has_value()) - TL_EXPECTED_THROW_EXCEPTION(bad_expected_access(err().value())); - return val(); - } - template ::value> * = nullptr> - TL_EXPECTED_11_CONSTEXPR U &value() & { - if (!has_value()) - TL_EXPECTED_THROW_EXCEPTION(bad_expected_access(err().value())); - return val(); - } - template ::value> * = nullptr> - TL_EXPECTED_11_CONSTEXPR const U &&value() const && { - if (!has_value()) - TL_EXPECTED_THROW_EXCEPTION(bad_expected_access(std::move(err()).value())); - return std::move(val()); - } - template ::value> * = nullptr> - TL_EXPECTED_11_CONSTEXPR U &&value() && { - if (!has_value()) - TL_EXPECTED_THROW_EXCEPTION(bad_expected_access(std::move(err()).value())); - return std::move(val()); - } - - constexpr const E &error() const & { - TL_ASSERT(!has_value()); - return err().value(); - } - TL_EXPECTED_11_CONSTEXPR E &error() & { - TL_ASSERT(!has_value()); - return err().value(); - } - constexpr const E &&error() const && { - TL_ASSERT(!has_value()); - return std::move(err().value()); - } - TL_EXPECTED_11_CONSTEXPR E &&error() && { - TL_ASSERT(!has_value()); - return std::move(err().value()); - } - - template constexpr T value_or(U &&v) const & { - static_assert(std::is_copy_constructible::value && - std::is_convertible::value, - "T must be copy-constructible and convertible to from U&&"); - return bool(*this) ? **this : static_cast(std::forward(v)); - } - template TL_EXPECTED_11_CONSTEXPR T value_or(U &&v) && { - static_assert(std::is_move_constructible::value && - std::is_convertible::value, - "T must be move-constructible and convertible to from U&&"); - return bool(*this) ? std::move(**this) : static_cast(std::forward(v)); - } -}; - -namespace detail { -template using exp_t = typename detail::decay_t::value_type; -template using err_t = typename detail::decay_t::error_type; -template using ret_t = expected>; - -#ifdef TL_EXPECTED_CXX14 -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - *std::declval()))> -constexpr auto and_then_impl(Exp &&exp, F &&f) { - static_assert(detail::is_expected::value, "F must return an expected"); - - return exp.has_value() - ? detail::invoke(std::forward(f), *std::forward(exp)) - : Ret(unexpect, std::forward(exp).error()); -} - -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval()))> -constexpr auto and_then_impl(Exp &&exp, F &&f) { - static_assert(detail::is_expected::value, "F must return an expected"); - - return exp.has_value() ? detail::invoke(std::forward(f)) - : Ret(unexpect, std::forward(exp).error()); -} -#else -template struct TC; -template (), - *std::declval())), - detail::enable_if_t>::value> * = nullptr> -auto and_then_impl(Exp &&exp, F &&f) -> Ret { - static_assert(detail::is_expected::value, "F must return an expected"); - - return exp.has_value() - ? detail::invoke(std::forward(f), *std::forward(exp)) - : Ret(unexpect, std::forward(exp).error()); -} - -template ())), - detail::enable_if_t>::value> * = nullptr> -constexpr auto and_then_impl(Exp &&exp, F &&f) -> Ret { - static_assert(detail::is_expected::value, "F must return an expected"); - - return exp.has_value() ? detail::invoke(std::forward(f)) - : Ret(unexpect, std::forward(exp).error()); -} -#endif - -#ifdef TL_EXPECTED_CXX14 -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - *std::declval())), - detail::enable_if_t::value> * = nullptr> -constexpr auto expected_map_impl(Exp &&exp, F &&f) { - using result = ret_t>; - return exp.has_value() ? result(detail::invoke(std::forward(f), - *std::forward(exp))) - : result(unexpect, std::forward(exp).error()); -} - -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - *std::declval())), - detail::enable_if_t::value> * = nullptr> -auto expected_map_impl(Exp &&exp, F &&f) { - using result = expected>; - if (exp.has_value()) { - detail::invoke(std::forward(f), *std::forward(exp)); - return result(); - } - - return result(unexpect, std::forward(exp).error()); -} - -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval())), - detail::enable_if_t::value> * = nullptr> -constexpr auto expected_map_impl(Exp &&exp, F &&f) { - using result = ret_t>; - return exp.has_value() ? result(detail::invoke(std::forward(f))) - : result(unexpect, std::forward(exp).error()); -} - -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval())), - detail::enable_if_t::value> * = nullptr> -auto expected_map_impl(Exp &&exp, F &&f) { - using result = expected>; - if (exp.has_value()) { - detail::invoke(std::forward(f)); - return result(); - } - - return result(unexpect, std::forward(exp).error()); -} -#else -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - *std::declval())), - detail::enable_if_t::value> * = nullptr> - -constexpr auto expected_map_impl(Exp &&exp, F &&f) - -> ret_t> { - using result = ret_t>; - - return exp.has_value() ? result(detail::invoke(std::forward(f), - *std::forward(exp))) - : result(unexpect, std::forward(exp).error()); -} - -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - *std::declval())), - detail::enable_if_t::value> * = nullptr> - -auto expected_map_impl(Exp &&exp, F &&f) -> expected> { - if (exp.has_value()) { - detail::invoke(std::forward(f), *std::forward(exp)); - return {}; - } - - return unexpected>(std::forward(exp).error()); -} - -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval())), - detail::enable_if_t::value> * = nullptr> - -constexpr auto expected_map_impl(Exp &&exp, F &&f) - -> ret_t> { - using result = ret_t>; - - return exp.has_value() ? result(detail::invoke(std::forward(f))) - : result(unexpect, std::forward(exp).error()); -} - -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval())), - detail::enable_if_t::value> * = nullptr> - -auto expected_map_impl(Exp &&exp, F &&f) -> expected> { - if (exp.has_value()) { - detail::invoke(std::forward(f)); - return {}; - } - - return unexpected>(std::forward(exp).error()); -} -#endif - -#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) && \ - !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55) -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -constexpr auto map_error_impl(Exp &&exp, F &&f) { - using result = expected, detail::decay_t>; - return exp.has_value() - ? result(*std::forward(exp)) - : result(unexpect, detail::invoke(std::forward(f), - std::forward(exp).error())); -} -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -auto map_error_impl(Exp &&exp, F &&f) { - using result = expected, monostate>; - if (exp.has_value()) { - return result(*std::forward(exp)); - } - - detail::invoke(std::forward(f), std::forward(exp).error()); - return result(unexpect, monostate{}); -} -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -constexpr auto map_error_impl(Exp &&exp, F &&f) { - using result = expected, detail::decay_t>; - return exp.has_value() - ? result() - : result(unexpect, detail::invoke(std::forward(f), - std::forward(exp).error())); -} -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -auto map_error_impl(Exp &&exp, F &&f) { - using result = expected, monostate>; - if (exp.has_value()) { - return result(); - } - - detail::invoke(std::forward(f), std::forward(exp).error()); - return result(unexpect, monostate{}); -} -#else -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -constexpr auto map_error_impl(Exp &&exp, F &&f) - -> expected, detail::decay_t> { - using result = expected, detail::decay_t>; - - return exp.has_value() - ? result(*std::forward(exp)) - : result(unexpect, detail::invoke(std::forward(f), - std::forward(exp).error())); -} - -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -auto map_error_impl(Exp &&exp, F &&f) -> expected, monostate> { - using result = expected, monostate>; - if (exp.has_value()) { - return result(*std::forward(exp)); - } - - detail::invoke(std::forward(f), std::forward(exp).error()); - return result(unexpect, monostate{}); -} - -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -constexpr auto map_error_impl(Exp &&exp, F &&f) - -> expected, detail::decay_t> { - using result = expected, detail::decay_t>; - - return exp.has_value() - ? result() - : result(unexpect, detail::invoke(std::forward(f), - std::forward(exp).error())); -} - -template >::value> * = nullptr, - class Ret = decltype(detail::invoke(std::declval(), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -auto map_error_impl(Exp &&exp, F &&f) -> expected, monostate> { - using result = expected, monostate>; - if (exp.has_value()) { - return result(); - } - - detail::invoke(std::forward(f), std::forward(exp).error()); - return result(unexpect, monostate{}); -} -#endif - -#ifdef TL_EXPECTED_CXX14 -template (), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -constexpr auto or_else_impl(Exp &&exp, F &&f) { - static_assert(detail::is_expected::value, "F must return an expected"); - return exp.has_value() ? std::forward(exp) - : detail::invoke(std::forward(f), - std::forward(exp).error()); -} - -template (), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -detail::decay_t or_else_impl(Exp &&exp, F &&f) { - return exp.has_value() ? std::forward(exp) - : (detail::invoke(std::forward(f), - std::forward(exp).error()), - std::forward(exp)); -} -#else -template (), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -auto or_else_impl(Exp &&exp, F &&f) -> Ret { - static_assert(detail::is_expected::value, "F must return an expected"); - return exp.has_value() ? std::forward(exp) - : detail::invoke(std::forward(f), - std::forward(exp).error()); -} - -template (), - std::declval().error())), - detail::enable_if_t::value> * = nullptr> -detail::decay_t or_else_impl(Exp &&exp, F &&f) { - return exp.has_value() ? std::forward(exp) - : (detail::invoke(std::forward(f), - std::forward(exp).error()), - std::forward(exp)); -} -#endif -} // namespace detail - -template -constexpr bool operator==(const expected &lhs, - const expected &rhs) { - return (lhs.has_value() != rhs.has_value()) - ? false - : (!lhs.has_value() ? lhs.error() == rhs.error() : *lhs == *rhs); -} -template -constexpr bool operator!=(const expected &lhs, - const expected &rhs) { - return (lhs.has_value() != rhs.has_value()) - ? true - : (!lhs.has_value() ? lhs.error() != rhs.error() : *lhs != *rhs); -} -template -constexpr bool operator==(const expected &lhs, - const expected &rhs) { - return (lhs.has_value() != rhs.has_value()) - ? false - : (!lhs.has_value() ? lhs.error() == rhs.error() : true); -} -template -constexpr bool operator!=(const expected &lhs, - const expected &rhs) { - return (lhs.has_value() != rhs.has_value()) - ? true - : (!lhs.has_value() ? lhs.error() != rhs.error() : false); -} - -template -constexpr bool operator==(const expected &x, const U &v) { - return x.has_value() ? *x == v : false; -} -template -constexpr bool operator==(const U &v, const expected &x) { - return x.has_value() ? *x == v : false; -} -template -constexpr bool operator!=(const expected &x, const U &v) { - return x.has_value() ? *x != v : true; -} -template -constexpr bool operator!=(const U &v, const expected &x) { - return x.has_value() ? *x != v : true; -} - -template -constexpr bool operator==(const expected &x, const unexpected &e) { - return x.has_value() ? false : x.error() == e.value(); -} -template -constexpr bool operator==(const unexpected &e, const expected &x) { - return x.has_value() ? false : x.error() == e.value(); -} -template -constexpr bool operator!=(const expected &x, const unexpected &e) { - return x.has_value() ? true : x.error() != e.value(); -} -template -constexpr bool operator!=(const unexpected &e, const expected &x) { - return x.has_value() ? true : x.error() != e.value(); -} - -template ::value || - std::is_move_constructible::value) && - detail::is_swappable::value && - std::is_move_constructible::value && - detail::is_swappable::value> * = nullptr> -void swap(expected &lhs, - expected &rhs) noexcept(noexcept(lhs.swap(rhs))) { - lhs.swap(rhs); -} -} // namespace tl - -#endif diff --git a/script/build.sh b/script/build.sh deleted file mode 100755 index 7b3e976..0000000 --- a/script/build.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -build_dir="${BUILD_DIR:-build}" -build_type="${BUILD_TYPE:-Release}" - -cmake_args=( - -S . - -B "${build_dir}" - -DCMAKE_BUILD_TYPE="${build_type}" - -DENABLE_TESTS=OFF - -DENABLE_BENCHMARKS=OFF -) - -if command -v ninja >/dev/null 2>&1; then - cmake_args+=(-G Ninja) -fi - -cmake "${cmake_args[@]}" - -cmake --build "${build_dir}" --target columnar_engine - -if [[ -x "${build_dir}/src/columnar_engine" ]]; then - ln -sf "src/columnar_engine" "${build_dir}/columnar_engine" -fi diff --git a/script/convert.sh b/script/convert.sh deleted file mode 100755 index 9cd90cc..0000000 --- a/script/convert.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ $# -ne 2 ]]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -input_csv="$1" -columnar_path="$2" -build_dir="${BUILD_DIR:-build}" -row_group_size="${ROW_GROUP_SIZE:-50000}" -schema_path="${COLUMNAR_SCHEMA:-${columnar_path}.schema.csv}" -binary="./${build_dir}/columnar_engine" - -if [[ ! -x "${binary}" ]]; then - binary="./${build_dir}/src/columnar_engine" -fi - -if [[ ! -x "${binary}" ]]; then - echo "columnar_engine binary not found; run ./script/build.sh first." >&2 - exit 1 -fi - -mkdir -p "$(dirname "${columnar_path}")" -mkdir -p "$(dirname "${schema_path}")" - -"${binary}" infer-schema \ - --input "${input_csv}" \ - --output "${schema_path}" - -"${binary}" convert \ - --schema "${schema_path}" \ - --input "${input_csv}" \ - --output "${columnar_path}" \ - --row-group-size "${row_group_size}" diff --git a/script/run_query.sh b/script/run_query.sh deleted file mode 100755 index 2996f56..0000000 --- a/script/run_query.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ $# -ne 4 ]]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -query_num="$1" -columnar_path="$2" -output_csv="$3" -log_path="$4" -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -repo_dir="$(cd "${script_dir}/.." && pwd)" -build_dir="${BUILD_DIR:-build}" -binary="${repo_dir}/${build_dir}/columnar_engine" -query_file="${repo_dir}/benchmarks/queries/query_${query_num}.sql" - -mkdir -p "$(dirname "${output_csv}")" -mkdir -p "$(dirname "${log_path}")" - -: > "${output_csv}" - -if [[ ! -f "${query_file}" ]]; then - cat > "${log_path}" < "${log_path}" < "${log_path}" 2>&1; then - printf 'query_num=%s\nstatus=ok\ncolumnar_path=%s\nquery_file=%s\n' \ - "${query_num}" "${columnar_path}" "${query_file}" >> "${log_path}" -else - status=$? - printf 'query_num=%s\nstatus=failed\ncolumnar_path=%s\nquery_file=%s\nexit_code=%s\n' \ - "${query_num}" "${columnar_path}" "${query_file}" "${status}" >> "${log_path}" - exit "${status}" -fi diff --git a/script/setup.sh b/script/setup.sh deleted file mode 100755 index 3e6f8f2..0000000 --- a/script/setup.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -packages=( - build-essential - cmake - ninja-build - git - ca-certificates -) - -if ! command -v apt-get >/dev/null 2>&1; then - echo "apt-get not found; assuming build dependencies are already installed." - exit 0 -fi - -missing=() -for package in "${packages[@]}"; do - if ! dpkg -s "${package}" >/dev/null 2>&1; then - missing+=("${package}") - fi -done - -if ((${#missing[@]} == 0)); then - echo "Build dependencies already installed." - exit 0 -fi - -if [[ "$(id -u)" -eq 0 ]]; then - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${missing[@]}" - rm -rf /var/lib/apt/lists/* - exit 0 -fi - -sudo apt-get update -sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${missing[@]}" diff --git a/scripts/Dockerfile b/scripts/Dockerfile deleted file mode 100644 index e4d0f97..0000000 --- a/scripts/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:24.04 - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -COPY commands.sh env.sh sample_input.csv /bench/ -RUN chmod +x /bench/commands.sh - -WORKDIR /work - -ENTRYPOINT ["/bench/commands.sh"] diff --git a/scripts/action.py b/scripts/action.py deleted file mode 100644 index a6b61a5..0000000 --- a/scripts/action.py +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env python3 -""" -action.py — starts Docker (if not running) and then runs act -for a GitHub Actions workflow. - -Examples: - python3 scripts/action.py - python3 scripts/action.py --workflow .github/workflows/cmake-single-platform.yml - python3 scripts/action.py -- --verbose -""" - -from __future__ import annotations - -import argparse -import os -import platform -import shutil -import subprocess -import sys -import time -from pathlib import Path -from typing import List, Optional - - -DEFAULT_WORKFLOW = ".github/workflows/cmake-single-platform.yml" -DEFAULT_PLATFORM_MAP = "ubuntu-latest=ghcr.io/catthehacker/ubuntu:act-latest" -DEFAULT_ARCH = "linux/amd64" -DEFAULT_DOCKER_TIMEOUT_SECONDS = 180 - -DEFAULT_REPO_ROOT_SENTINEL = "" - -DOCKER_DESKTOP_APP_NAME_MAC = "Docker" - -ACT_EXE = "act" -DOCKER_EXE = "docker" -GIT_EXE = "git" -SYSTEMCTL_EXE = "systemctl" -SERVICE_EXE = "service" -SUDO_EXE = "sudo" -OPEN_EXE_MAC = "open" - -APT_PACKAGES_FILE = ".github/apt-packages.txt" - - -ERR_DOCKER_NOT_FOUND = ( - "Docker executable not found in PATH. Install Docker Desktop (macOS/Windows) " - "or Docker Engine (Linux)." -) -ERR_ACT_NOT_FOUND = ( - "'act' executable not found in PATH. Install nektos/act (e.g. 'brew install act') " - "or follow the official installation docs." -) -ERR_WORKFLOW_NOT_FOUND = "Workflow file not found: {path}" -ERR_DOCKER_NOT_READY_TIMEOUT = ( - "Docker did not become ready in time. Ensure Docker Desktop is running and does not " - "require manual confirmation." -) -ERR_APT_PACKAGES_NOT_FOUND = ( - "Required file not found: {path}\n" - "Create it (repo-root/.github/apt-packages.txt) or update the workflow to not depend on it." -) - -INFO_DOCKER_STARTING_MAC = "Docker daemon is not responding — starting Docker Desktop…" -INFO_DOCKER_STARTING_LINUX = ( - "Docker daemon is not responding — trying to start docker service…" -) -INFO_DOCKER_UNSUPPORTED_OS = ( - "Docker daemon is not responding. OS '{os_name}'. Start Docker manually and retry." -) -INFO_RUNNING_ACT = "Running:" -INFO_USING_REPO_ROOT = "Using repo root:" -INFO_AUTO_DETECT_REPO_ROOT_FAILED = ( - "Auto-detecting repo root via git failed; falling back to script parent directory." -) - - -def run( - cmd: List[str], check: bool = False, capture: bool = False -) -> subprocess.CompletedProcess: - return subprocess.run( - cmd, - check=check, - text=True, - stdout=subprocess.PIPE if capture else None, - stderr=subprocess.STDOUT if capture else None, - ) - - -def have_exe(name: str) -> bool: - return shutil.which(name) is not None - - -def resolve_path(base: Path, maybe_path: str) -> Path: - p = Path(maybe_path) - return p if p.is_absolute() else (base / p) - - -def detect_repo_root() -> Path: - if have_exe(GIT_EXE): - p = run([GIT_EXE, "rev-parse", "--show-toplevel"], check=False, capture=True) - if p.returncode == 0 and p.stdout: - return Path(p.stdout.strip()).resolve() - - print(INFO_AUTO_DETECT_REPO_ROOT_FAILED, flush=True) - return Path(__file__).resolve().parent.parent - - -def docker_daemon_ready() -> bool: - if not have_exe(DOCKER_EXE): - return False - p = run([DOCKER_EXE, "info"], check=False, capture=True) - return p.returncode == 0 - - -def start_docker_desktop_mac() -> None: - run( - [OPEN_EXE_MAC, "-g", "-a", DOCKER_DESKTOP_APP_NAME_MAC], - check=False, - capture=False, - ) - - -def start_docker_linux() -> None: - if have_exe(SYSTEMCTL_EXE): - p = run([SYSTEMCTL_EXE, "start", DOCKER_EXE], check=False, capture=True) - if p.returncode != 0 and have_exe(SUDO_EXE): - run( - [SUDO_EXE, SYSTEMCTL_EXE, "start", DOCKER_EXE], - check=False, - capture=False, - ) - elif have_exe(SERVICE_EXE): - p = run([SERVICE_EXE, DOCKER_EXE, "start"], check=False, capture=True) - if p.returncode != 0 and have_exe(SUDO_EXE): - run( - [SUDO_EXE, SERVICE_EXE, DOCKER_EXE, "start"], check=False, capture=False - ) - - -def ensure_docker_running( - timeout_seconds: int = DEFAULT_DOCKER_TIMEOUT_SECONDS, -) -> None: - if not have_exe(DOCKER_EXE): - raise RuntimeError(ERR_DOCKER_NOT_FOUND) - - if docker_daemon_ready(): - return - - system = platform.system().lower() - if system == "darwin": - print(INFO_DOCKER_STARTING_MAC, flush=True) - start_docker_desktop_mac() - elif system == "linux": - print(INFO_DOCKER_STARTING_LINUX, flush=True) - start_docker_linux() - else: - print(INFO_DOCKER_UNSUPPORTED_OS.format(os_name=platform.system()), flush=True) - - start = time.time() - while time.time() - start < timeout_seconds: - if docker_daemon_ready(): - return - time.sleep(2) - - raise RuntimeError(ERR_DOCKER_NOT_READY_TIMEOUT) - - -def ensure_act_installed() -> None: - if have_exe(ACT_EXE): - return - raise RuntimeError(ERR_ACT_NOT_FOUND) - - -def run_act(workflow: str, arch: str, platform_map: str, extra_args: List[str]) -> int: - cmd = [ - ACT_EXE, - "-W", - workflow, - "--container-architecture", - arch, - "-P", - platform_map, - ] - cmd.extend(extra_args) - - print(f"\n{INFO_RUNNING_ACT} {' '.join(cmd)}\n", flush=True) - p = subprocess.run(cmd) - return p.returncode - - -def build_arg_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description="Start Docker (if needed) and run act workflow." - ) - parser.add_argument( - "--workflow", - default=DEFAULT_WORKFLOW, - help="Path to workflow yml (relative to repo root)", - ) - parser.add_argument( - "--arch", default=DEFAULT_ARCH, help="act --container-architecture value" - ) - parser.add_argument( - "--platform-map", - default=DEFAULT_PLATFORM_MAP, - help='act -P mapping, e.g. "ubuntu-latest=ghcr.io/catthehacker/ubuntu:act-latest"', - ) - parser.add_argument( - "--repo-root", - default=DEFAULT_REPO_ROOT_SENTINEL, - help="Repo root. If empty, auto-detect using git (default).", - ) - parser.add_argument( - "--docker-timeout", - type=int, - default=DEFAULT_DOCKER_TIMEOUT_SECONDS, - help="Seconds to wait for Docker to become ready", - ) - parser.add_argument( - "act_args", - nargs=argparse.REMAINDER, - help="Extra args passed to act (prefix with --), e.g. -- -v", - ) - return parser - - -def main() -> int: - parser = build_arg_parser() - args = parser.parse_args() - - extra = args.act_args - if extra and extra[0] == "--": - extra = extra[1:] - - repo_root = Path(args.repo_root).resolve() if args.repo_root else detect_repo_root() - print(f"{INFO_USING_REPO_ROOT} {repo_root}", flush=True) - os.chdir(repo_root) - - apt_file = resolve_path(repo_root, APT_PACKAGES_FILE) - if not apt_file.exists(): - raise RuntimeError(ERR_APT_PACKAGES_NOT_FOUND.format(path=apt_file)) - - workflow_path = resolve_path(repo_root, args.workflow) - if not workflow_path.exists(): - raise RuntimeError(ERR_WORKFLOW_NOT_FOUND.format(path=workflow_path.resolve())) - - ensure_act_installed() - ensure_docker_running(timeout_seconds=args.docker_timeout) - - return run_act(str(workflow_path), args.arch, args.platform_map, extra) - - -if __name__ == "__main__": - try: - raise SystemExit(main()) - except Exception as e: - print(f"\nERROR: {e}\n", file=sys.stderr) - raise SystemExit(1) diff --git a/scripts/commands.sh b/scripts/commands.sh deleted file mode 100755 index 8309487..0000000 --- a/scripts/commands.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${script_dir}/env.sh" - -log() { - printf '[%s] %s\n' "$(date -Iseconds)" "$*" -} - -required_vars=( - REPO_URL - BRANCH - INPUT_CSV - COLUMNAR - RESULTS -) - -for var_name in "${required_vars[@]}"; do - if [[ -z "${!var_name:-}" ]]; then - echo "Missing required environment variable: ${var_name}" >&2 - exit 1 - fi -done - -log "Cloning ${REPO_URL} (branch: ${BRANCH})" -git clone --branch "${BRANCH}" "${REPO_URL}" repo -cd repo - -mkdir -p "$(dirname "${COLUMNAR}")" "${RESULTS}" - -log "Installing dependencies" -./script/setup.sh - -log "Building project" -./script/build.sh - -log "Converting CSV ${INPUT_CSV} -> ${COLUMNAR}" -./script/convert.sh "${INPUT_CSV}" "${COLUMNAR}" - -for query_num in {0..42}; do - output="${RESULTS}/query_${query_num}.csv" - logs="${RESULTS}/query_${query_num}.log" - log "Generating placeholder output for query ${query_num}" - ./script/run_query.sh "${query_num}" "${COLUMNAR}" "${output}" "${logs}" -done - -log "Completed. Columnar file: ${COLUMNAR}; results dir: ${RESULTS}" diff --git a/scripts/env.sh b/scripts/env.sh deleted file mode 100644 index 29dd087..0000000 --- a/scripts/env.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -: "${REPO_URL:=https://github.com/ik4rd/columnar-engine}" -: "${BRANCH:=main}" -: "${INPUT_CSV:=/bench/sample_input.csv}" -: "${COLUMNAR:=/work/results/output.columnar}" -: "${RESULTS:=/work/results}" - -export REPO_URL -export BRANCH -export INPUT_CSV -export COLUMNAR -export RESULTS diff --git a/scripts/install_dependencies.sh b/scripts/install_dependencies.sh deleted file mode 100755 index 7ecacf8..0000000 --- a/scripts/install_dependencies.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -APT_PACKAGES_FILE=".github/apt-packages.txt" -APT_CACHE_DIR="${HOME}/.cache/apt" -APT_ARCHIVES_DIR="${APT_CACHE_DIR}/archives" -APT_PARTIAL_DIR="${APT_ARCHIVES_DIR}/partial" - -missing=0 -while read -r pkg; do - [[ -z "${pkg}" ]] && continue - dpkg -s "${pkg}" >/dev/null 2>&1 || missing=1 -done < "${APT_PACKAGES_FILE}" - -if [[ "${missing}" -eq 0 ]]; then - echo "All dependencies already installed — skipping apt install." - exit 0 -fi - -sudo apt-get update -mkdir -p "${APT_PARTIAL_DIR}" - -sudo DEBIAN_FRONTEND=noninteractive apt-get \ - -o Dir::Cache::archives="${APT_ARCHIVES_DIR}" \ - -o Dir::Cache::archives::partial="${APT_PARTIAL_DIR}" \ - install -y --no-install-recommends $(tr '\n' ' ' < "${APT_PACKAGES_FILE}") diff --git a/scripts/sample_input.columnar b/scripts/sample_input.columnar deleted file mode 100644 index a6151e99661714b367b33b3802c0ca908cda4dee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 154 zcmZQ%U}9iqU}a!naLma}PGtr%obr>{fs}JbVo^?JDi?zYgB(x^0|N&TC+Fvs=H|s4 mvLJDcn2 -#include -#include -#include - -#include "csv_batch_io.h" -#include "csv_columnar.h" -#include "error.h" -#include "executor.h" -#include "fileio.h" -#include "schema.h" - -void ConfigureInferSchemaCommand(argparse::ArgumentParser& command) { - command.add_description("Infer a schema from a CSV file."); - command.add_argument("--input").required(); - command.add_argument("--output").required(); -} - -void ConfigureConvertCommand(argparse::ArgumentParser& command) { - command.add_description("Convert CSV to the internal columnar format."); - command.add_argument("--schema").required(); - command.add_argument("--input").required(); - command.add_argument("--output").required(); - command.add_argument("--row-group-size").scan<'u', size_t>().default_value(50000); -} - -void ConfigureRoundtripCommand(argparse::ArgumentParser& command) { - command.add_description("Convert a columnar file back to schema CSV and data CSV."); - command.add_argument("--input").required(); - command.add_argument("--schema-output").required(); - command.add_argument("--csv-output").required(); -} - -void ConfigureRunQueryCommand(argparse::ArgumentParser& command) { - command.add_description("Execute a SQL query against a columnar file and write the result to CSV."); - command.add_argument("--input").required(); - command.add_argument("--output").required(); - command.add_argument("--table-name").default_value(std::string("hits")); - command.add_argument("--query"); - command.add_argument("--query-file"); -} - -int RunInferSchema(const argparse::ArgumentParser& command) { - const auto input_path = std::filesystem::path(command.get("--input")); - const auto output_path = std::filesystem::path(command.get("--output")); - EnsureParentDirectory(output_path); - WriteSchemaCsv(output_path, InferSchemaCsv(input_path)); - return 0; -} - -int RunConvert(const argparse::ArgumentParser& command) { - const auto output_path = std::filesystem::path(command.get("--output")); - EnsureParentDirectory(output_path); - ConvertCsvToColumnar(std::filesystem::path(command.get("--schema")), - std::filesystem::path(command.get("--input")), output_path, - command.get("--row-group-size")); - return 0; -} - -int RunRoundtrip(const argparse::ArgumentParser& command) { - const auto schema_output_path = std::filesystem::path(command.get("--schema-output")); - const auto csv_output_path = std::filesystem::path(command.get("--csv-output")); - EnsureParentDirectory(schema_output_path); - EnsureParentDirectory(csv_output_path); - ConvertColumnarToCsv(std::filesystem::path(command.get("--input")), schema_output_path, - csv_output_path); - return 0; -} - -int RunQuery(const argparse::ArgumentParser& command) { - const bool has_query = command.is_used("--query"); - const bool has_query_file = command.is_used("--query-file"); - - if (has_query == has_query_file) { - throw Error::InvalidArgument("app", "exactly one of --query or --query-file must be specified"); - } - - const std::string query = has_query ? command.get("--query") - : ReadTextFile(std::filesystem::path(command.get("--query-file"))); - - const auto output_path = std::filesystem::path(command.get("--output")); - EnsureParentDirectory(output_path); - - Executor executor; - executor.RegisterTable(command.get("--table-name"), - std::filesystem::path(command.get("--input"))); - - const ExecuteExpected result = executor.Execute(query); - if (!result.has_value()) { - throw result.error(); - } - - WriteBatchCsv(output_path, result.value()); - - return 0; -} - -int main(const int argc, char** argv) { - try { - argparse::ArgumentParser program("columnar_engine"); - argparse::ArgumentParser infer_schema_command("infer-schema"); - argparse::ArgumentParser convert_command("convert"); - argparse::ArgumentParser roundtrip_command("roundtrip"); - argparse::ArgumentParser run_query_command("run-query"); - - ConfigureInferSchemaCommand(infer_schema_command); - ConfigureConvertCommand(convert_command); - ConfigureRoundtripCommand(roundtrip_command); - ConfigureRunQueryCommand(run_query_command); - - program.add_subparser(infer_schema_command); - program.add_subparser(convert_command); - program.add_subparser(roundtrip_command); - program.add_subparser(run_query_command); - program.parse_args(argc, argv); - - if (program.is_subcommand_used("infer-schema")) { - return RunInferSchema(infer_schema_command); - } - if (program.is_subcommand_used("convert")) { - return RunConvert(convert_command); - } - if (program.is_subcommand_used("roundtrip")) { - return RunRoundtrip(roundtrip_command); - } - if (program.is_subcommand_used("run-query")) { - return RunQuery(run_query_command); - } - - throw Error::InvalidArgument("app", "a subcommand is required"); - } catch (const Error& ex) { - std::cerr << ex.what() << std::endl; - } catch (const std::exception& ex) { - std::cerr << ex.what() << std::endl; - } - - return 1; -} diff --git a/src/convert/csv_columnar.cpp b/src/convert/csv_columnar.cpp deleted file mode 100644 index 3f9d600..0000000 --- a/src/convert/csv_columnar.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#include "csv_columnar.h" - -#include "columnar_batch_io.h" -#include "csv_batch_io.h" -#include "error.h" -#include "schema.h" - -void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, - const std::filesystem::path& output_path, size_t max_rows_per_group) { - if (max_rows_per_group == 0) { - throw Error::InvalidArgument("columnar", "row group size must be > 0"); - } - - const Schema schema = ReadSchemaCsv(schema_path); - - BatchSizing sizing; - sizing.max_rows = max_rows_per_group; - - CsvBatchReader batch_reader(data_path, schema, sizing); - ColumnarBatchWriter batch_writer(output_path, schema); - - while (auto batch = batch_reader.ReadNext()) { - batch_writer.Write(*batch); - } - - batch_writer.Finalize(); -} - -void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, - const std::filesystem::path& data_path) { - ColumnarBatchReader batch_reader(columnar_path); - - WriteSchemaCsv(schema_path, batch_reader.GetSchema()); - - CsvBatchWriter batch_writer(data_path, batch_reader.GetSchema()); - - while (auto batch = batch_reader.ReadNext()) { - batch_writer.Write(*batch); - } - - batch_writer.Flush(); -} diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp deleted file mode 100644 index e9d00ef..0000000 --- a/src/executor/aggregate_states.cpp +++ /dev/null @@ -1,406 +0,0 @@ -#include -#include -#include -#include -#include - -#include "aggregate_function.h" -#include "ascii.h" -#include "executor.h" -#include "int128.h" -#include "parsing.h" - -static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view candidate, - const std::string_view current, const bool is_min) { - switch (type) { - case ColumnType::Boolean: { - const bool left = ParseBoolean(std::string(candidate)); - const bool right = ParseBoolean(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Int16: { - const int16_t left = ParseInt16(std::string(candidate)); - const int16_t right = ParseInt16(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Int32: { - const int32_t left = ParseInt32(std::string(candidate)); - const int32_t right = ParseInt32(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Int64: { - const int64_t left = ParseInt64(std::string(candidate)); - const int64_t right = ParseInt64(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Int128: { - const Int128 left = ParseInt128(std::string(candidate)); - const Int128 right = ParseInt128(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::String: - return is_min ? candidate < current : candidate > current; - case ColumnType::Date: { - const int32_t left = ParseDate(std::string(candidate)); - const int32_t right = ParseDate(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Timestamp: { - const int64_t left = ParseTimestamp(std::string(candidate)); - const int64_t right = ParseTimestamp(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Character: { - const char left = ParseCharacter(std::string(candidate)); - const char right = ParseCharacter(std::string(current)); - return is_min ? left < right : left > right; - } - } - throw Error::Unsupported("executor", "unsupported column type in comparison"); -} - -static Int128 ParseToInt128(const ColumnType type, const std::string_view value) { - switch (type) { - case ColumnType::Boolean: - return ParseBoolean(std::string(value)) ? 1 : 0; - case ColumnType::Int16: - return ParseInt16(std::string(value)); - case ColumnType::Int32: - return ParseInt32(std::string(value)); - case ColumnType::Int64: - return ParseInt64(std::string(value)); - case ColumnType::Int128: - return ParseInt128(std::string(value)); - case ColumnType::Character: - return ParseCharacter(std::string(value)); - case ColumnType::String: - case ColumnType::Date: - case ColumnType::Timestamp: - break; - } - throw Error::Unsupported("executor", "unsupported numeric conversion"); -} - -static UInt128 ToMagnitude(const Int128 value) { - if (value >= 0) { - return static_cast(value); - } - - UInt128 magnitude = static_cast(-(value + 1)); - ++magnitude; - - return magnitude; -} - -static std::string UInt128ToString(UInt128 value) { - if (value == 0) { - return "0"; - } - - std::string out; - - while (value > 0) { - out.push_back(static_cast('0' + value % 10)); - value /= 10; - } - - std::ranges::reverse(out); - - return out; -} - -static std::string FormatAverage(const Int128 sum, const uint64_t count) { - if (count == 0) { - return "0"; - } - - const bool negative = sum < 0; - const UInt128 magnitude = ToMagnitude(sum); - const UInt128 divisor = count; - const UInt128 integer = magnitude / divisor; - UInt128 remainder = magnitude % divisor; - - std::string output = UInt128ToString(integer); - - if (remainder != 0) { - output.push_back('.'); - - std::string fractional; - fractional.reserve(6); - - for (size_t i = 0; i < 6 && remainder != 0; ++i) { - remainder *= 10; - fractional.push_back(static_cast('0' + remainder / divisor)); - remainder %= divisor; - } - - while (!fractional.empty() && fractional.back() == '0') { - fractional.pop_back(); - } - - if (!fractional.empty()) { - output += fractional; - } else { - output.pop_back(); - } - } - - if (negative && output != "0") { - output.insert(output.begin(), '-'); - } - - return output; -} - -class CountAS final : public AggState { - public: - void Consume(const std::optional value) override { - (void)value; - ++count_; - } - - std::string Finalize() const override { return std::to_string(count_); } - - private: - uint64_t count_ = 0; -}; - -class SumAS final : public AggState { - public: - explicit SumAS(const ColumnType type) : type_(type) {} - - public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "SUM requires a value"); - } - sum_ += ParseToInt128(type_, *value); - } - - std::string Finalize() const override { return Int128ToString(sum_); } - - private: - ColumnType type_; - Int128 sum_ = 0; -}; - -class AvgAS final : public AggState { - public: - explicit AvgAS(const ColumnType type) : type_(type) {} - - public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "AVG requires a value"); - } - sum_ += ParseToInt128(type_, *value); - ++count_; - } - - std::string Finalize() const override { return FormatAverage(sum_, count_); } - - private: - ColumnType type_; - Int128 sum_ = 0; - uint64_t count_ = 0; -}; - -class ExtremumAS final : public AggState { - public: - ExtremumAS(const ColumnType type, const bool is_min) : type_(type), is_min_(is_min) {} - - public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "MIN/MAX requires a value"); - } - - if (!extremum_.has_value()) { - extremum_ = std::string(*value); - return; - } - - if (ShouldReplaceExtremum(type_, *value, *extremum_, is_min_)) { - extremum_ = std::string(*value); - } - } - - std::string Finalize() const override { return extremum_.value_or(""); } - - private: - ColumnType type_; - bool is_min_ = true; - std::optional extremum_; -}; - -class DistinctAS final : public AggState { - public: - explicit DistinctAS(std::unique_ptr nested) : nested_(std::move(nested)) {} - - public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "DISTINCT requires a value"); - } - - if (seen_.insert(std::string(*value)).second) { - nested_->Consume(value); - } - } - - std::string Finalize() const override { return nested_->Finalize(); } - - private: - std::unique_ptr nested_; - std::unordered_set seen_; -}; - -static bool SupportsAnyType(const ColumnType type) { - (void)type; - return true; -} - -static bool SupportsNumericType(const ColumnType type) { - switch (type) { - case ColumnType::Boolean: - case ColumnType::Int16: - case ColumnType::Int32: - case ColumnType::Int64: - case ColumnType::Int128: - case ColumnType::Character: - return true; - case ColumnType::String: - case ColumnType::Date: - case ColumnType::Timestamp: - return false; - } - return false; -} - -static std::unique_ptr CreateCountState(const PlannedAgg& aggregate) { - (void)aggregate; - return std::make_unique(); -} - -static std::unique_ptr CreateSumState(const PlannedAgg& aggregate) { - return std::make_unique(aggregate.input_type); -} - -static std::unique_ptr CreateAvgState(const PlannedAgg& aggregate) { - return std::make_unique(aggregate.input_type); -} - -static std::unique_ptr CreateMinState(const PlannedAgg& aggregate) { - return std::make_unique(aggregate.input_type, true); -} - -static std::unique_ptr CreateMaxState(const PlannedAgg& aggregate) { - return std::make_unique(aggregate.input_type, false); -} - -AggRegistry& AggRegistry::Instance() { - static AggRegistry instance; - return instance; -} - -void AggRegistry::Register(const AggFuncDefinition& def) { registry_[ToUpperAscii(def.canonical_name)] = def; } - -const AggFuncDefinition* AggRegistry::Find(const std::string_view name) const { - const auto it = registry_.find(ToUpperAscii(name)); - if (it == registry_.end()) { - return nullptr; - } - return &it->second; -} - -AggRegistrar::AggRegistrar(const AggFuncDefinition& def) { AggRegistry::Instance().Register(def); } - -[[maybe_unused]] static AggRegistrar register_count({ - .canonical_name = "COUNT", - .distinct = true, - .star = true, - .supports_type = &SupportsAnyType, - .factory = &CreateCountState, -}); - -[[maybe_unused]] static AggRegistrar register_sum({ - .canonical_name = "SUM", - .distinct = false, - .star = false, - .supports_type = &SupportsNumericType, - .factory = &CreateSumState, -}); - -[[maybe_unused]] static AggRegistrar register_avg({ - .canonical_name = "AVG", - .distinct = false, - .star = false, - .supports_type = &SupportsNumericType, - .factory = &CreateAvgState, -}); - -[[maybe_unused]] static AggRegistrar register_min({ - .canonical_name = "MIN", - .distinct = false, - .star = false, - .supports_type = &SupportsAnyType, - .factory = &CreateMinState, -}); - -[[maybe_unused]] static AggRegistrar register_max({ - .canonical_name = "MAX", - .distinct = false, - .star = false, - .supports_type = &SupportsAnyType, - .factory = &CreateMaxState, -}); - -const AggFuncDefinition& ResolveAggFunc(const std::string_view name) { - const AggFuncDefinition* definition = AggRegistry::Instance().Find(name); - if (definition == nullptr) { - throw Error::Unsupported("executor", "aggregate '" + std::string(name) + "' is not registered"); - } - return *definition; -} - -bool AggSupportsInputType(const AggFuncDefinition& definition, const ColumnType type) { - if (definition.supports_type == nullptr) { - return false; - } - return definition.supports_type(type); -} - -std::string FormatAgg(const AggFuncDefinition& definition, const std::string_view column_name, const bool distinct, - const bool star) { - std::string output(definition.canonical_name); - output += '('; - if (star) { - output += '*'; - } else { - if (distinct) { - output += "DISTINCT "; - } - output += column_name; - } - output += ')'; - return output; -} - -std::unique_ptr CreateAggState(const PlannedAgg& aggregate) { - if (aggregate.function == nullptr || aggregate.function->factory == nullptr) { - throw Error::Unsupported("executor", "aggregate is not registered"); - } - if (aggregate.distinct && !aggregate.function->distinct) { - throw Error::Unsupported("executor", "DISTINCT is not supported for aggregate '" + - std::string(aggregate.function->canonical_name) + "'"); - } - if (aggregate.star && !aggregate.function->star) { - throw Error::Unsupported( - "executor", "aggregate '" + std::string(aggregate.function->canonical_name) + "' does not support '*'"); - } - std::unique_ptr state = aggregate.function->factory(aggregate); - if (aggregate.distinct) { - state = std::make_unique(std::move(state)); - } - return state; -} diff --git a/src/executor/executor.cpp b/src/executor/executor.cpp deleted file mode 100644 index 5f8bc82..0000000 --- a/src/executor/executor.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "executor.h" - -#include - -#include "ascii.h" - -void Executor::RegisterTable(const std::string& name, std::filesystem::path path) { - tables_[ToLowerAscii(name)] = std::move(path); -} - -ExecuteExpected Executor::Execute(const std::string_view query) const { - try { - const ParsedQuery parsed = ParseQuery(query); - const PlannedQuery planned = PlanQuery(parsed, tables_); - - std::unique_ptr root = BuildPlan(planned); - - auto batch = root->Next(); - if (!batch.has_value()) { - return Batch{}; - } - - return std::move(*batch); - } catch (const Error& error) { - return tl::unexpected(error); - } -} diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp deleted file mode 100644 index d12602b..0000000 --- a/src/executor/operators.cpp +++ /dev/null @@ -1,217 +0,0 @@ -#include -#include -#include - -#include "columnar_batch_io.h" -#include "executor.h" -#include "fileio.h" -#include "parsing.h" - -static void SeekRead(const std::filesystem::path& path, std::ifstream& in, const uint64_t offset) { - in.seekg(static_cast(offset)); - if (!in) { - throw Error::PathIo("operators", path, "seek file"); - } -} - -template -static bool EvaluateComparison(const T left, const T right, const ComparisonKind kind) { - if (kind == ComparisonKind::Equal) { - return left == right; - } - if (kind == ComparisonKind::NotEqual) { - return left != right; - } - if (kind == ComparisonKind::Less) { - return left < right; - } - if (kind == ComparisonKind::LessOrEqual) { - return left <= right; - } - if (kind == ComparisonKind::Greater) { - return left > right; - } - if (kind == ComparisonKind::GreaterOrEqual) { - return left >= right; - } - throw Error::Unsupported("operators", "unsupported comparison kind"); -} - -static bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, - const ComparisonKind kind) { - switch (type) { - case ColumnType::Boolean: - return EvaluateComparison(ParseBoolean(std::string(lhs)), ParseBoolean(std::string(rhs)), kind); - case ColumnType::Int16: - return EvaluateComparison(ParseInt16(std::string(lhs)), ParseInt16(std::string(rhs)), kind); - case ColumnType::Int32: - return EvaluateComparison(ParseInt32(std::string(lhs)), ParseInt32(std::string(rhs)), kind); - case ColumnType::Int64: - return EvaluateComparison(ParseInt64(std::string(lhs)), ParseInt64(std::string(rhs)), kind); - case ColumnType::Int128: - return EvaluateComparison(ParseInt128(std::string(lhs)), ParseInt128(std::string(rhs)), kind); - case ColumnType::String: - return EvaluateComparison(std::string(lhs), std::string(rhs), kind); - case ColumnType::Date: - return EvaluateComparison(ParseDate(std::string(lhs)), ParseDate(std::string(rhs)), kind); - case ColumnType::Timestamp: - return EvaluateComparison(ParseTimestamp(std::string(lhs)), ParseTimestamp(std::string(rhs)), kind); - case ColumnType::Character: - return EvaluateComparison(ParseCharacter(std::string(lhs)), ParseCharacter(std::string(rhs)), kind); - } - throw Error::Unsupported("operators", "unsupported column type in comparison"); -} - -class ScanOperator final : public Operator { - public: - ScanOperator(std::filesystem::path path, std::vector projection_indexes) - : path_(std::move(path)), in_(OpenInputFile(path_)), metadata_(ColumnarBatchReader(path_).GetMetadata()) { - if (metadata_.schema.columns.empty()) { - throw Error::InvalidData("operators", "columnar schema is empty", path_.string()); - } - - projection_indexes_ = std::move(projection_indexes); - projected_schema_.columns.reserve(projection_indexes_.size()); - - for (const size_t source_index : projection_indexes_) { - if (source_index >= metadata_.schema.columns.size()) { - throw Error::OutOfRange("operators", "projection index out of range", path_.string()); - } - projected_schema_.columns.push_back(metadata_.schema.columns[source_index]); - } - } - - public: - std::optional Next() override { - if (next_group_ >= metadata_.row_groups.size()) { - return std::nullopt; - } - - const auto& row_group = metadata_.row_groups[next_group_++]; - Batch batch(projected_schema_, row_group.row_count); - - for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { - const size_t source_index = projection_indexes_[projected_index]; - const auto& chunk = row_group.columns[source_index]; - SeekRead(path_, in_, chunk.offset); - batch.ColumnAt(projected_index).ReadFrom(in_, row_group.row_count, chunk.size); - } - - return batch; - } - - private: - std::filesystem::path path_; - std::ifstream in_; - - ColumnarMetadata metadata_; - std::vector projection_indexes_; - Schema projected_schema_; - size_t next_group_ = 0; -}; - -class FilterOperator final : public Operator { - public: - FilterOperator(std::unique_ptr child, PlannedFilter filter) - : child_(std::move(child)), filter_(std::move(filter)) {} - - public: - std::optional Next() override { - while (auto batch = child_->Next()) { - Batch filtered(batch->GetSchema()); - - for (size_t row = 0; row < batch->RowsCount(); ++row) { - const std::string value = batch->ColumnAt(filter_.column_index).ValueAsString(row); - - if (!MatchesComparison(filter_.column_type, value, filter_.literal_text, filter_.comparison)) { - continue; - } - - for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { - filtered.ColumnAt(column_index).AppendFromString(batch->ColumnAt(column_index).ValueAsString(row)); - } - } - - if (filtered.RowsCount() > 0) { - return filtered; - } - } - - return std::nullopt; - } - - private: - std::unique_ptr child_; - PlannedFilter filter_; -}; - -struct AggBinding { - PlannedAgg aggregate; - std::unique_ptr state; -}; - -class AggOperator final : public Operator { - public: - AggOperator(std::unique_ptr child, const std::vector& aggregates) : child_(std::move(child)) { - bindings_.reserve(aggregates.size()); - for (const auto& aggregate : aggregates) { - bindings_.push_back(AggBinding{ - .aggregate = aggregate, - .state = CreateAggState(aggregate), - }); - } - } - - public: - std::optional Next() override { - if (returned_) { - return std::nullopt; - } - returned_ = true; - - while (auto batch = child_->Next()) { - for (size_t row = 0; row < batch->RowsCount(); ++row) { - for (auto& binding : bindings_) { - if (!binding.aggregate.star) { - std::string materialized = batch->ColumnAt(*binding.aggregate.column_index).ValueAsString(row); - binding.state->Consume(materialized); - } else { - binding.state->Consume(std::nullopt); - } - } - } - } - - Schema schema; - schema.columns.reserve(bindings_.size()); - - for (const auto& binding : bindings_) { - schema.columns.push_back(ColumnSchema{ - .name = binding.aggregate.output_name, - .type = ColumnType::String, - }); - } - - Batch result(schema); - - for (size_t i = 0; i < bindings_.size(); ++i) { - result.ColumnAt(i).AppendFromString(bindings_[i].state->Finalize()); - } - - return result; - } - - private: - std::unique_ptr child_; - std::vector bindings_; - bool returned_ = false; -}; - -std::unique_ptr BuildPlan(const PlannedQuery& planned) { - std::unique_ptr root = std::make_unique(planned.table_path, planned.projection_indexes); - if (planned.filter.has_value()) { - root = std::make_unique(std::move(root), *planned.filter); - } - root = std::make_unique(std::move(root), planned.aggregates); - return root; -} diff --git a/src/executor/query_parser.cpp b/src/executor/query_parser.cpp deleted file mode 100644 index 396b093..0000000 --- a/src/executor/query_parser.cpp +++ /dev/null @@ -1,175 +0,0 @@ -#include -#include - -#include "executor.h" - -class TokenCursor { - public: - explicit TokenCursor(std::vector tokens) : tokens_(std::move(tokens)) {} - - public: - const Token& Peek() const { - if (pos_ >= tokens_.size()) { - throw Error::InvalidArgument("query_parser", "unexpected end of query"); - } - return *tokens_[pos_]; - } - - bool Match(const Tokens type) const { return pos_ < tokens_.size() && tokens_[pos_]->GetType() == type; } - - const Token& Consume(const Tokens type, const std::string_view message) { - if (!Match(type)) { - throw Error::InvalidArgument("query_parser", std::string(message)); - } - return *tokens_[pos_++]; - } - - bool TryConsume(const Tokens type) { - if (!Match(type)) { - return false; - } - ++pos_; - return true; - } - - const Token& ConsumeFunctionToken(const std::string_view message) { - if (pos_ >= tokens_.size()) { - throw Error::InvalidArgument("query_parser", std::string(message)); - } - - const Tokens type = tokens_[pos_]->GetType(); - switch (type) { - case Tokens::NumericLiteral: - case Tokens::StringLiteral: - case Tokens::OpenBracket: - case Tokens::CloseBracket: - case Tokens::Comma: - case Tokens::Dot: - case Tokens::Semicolon: - case Tokens::Equal: - case Tokens::NotEqual: - case Tokens::Less: - case Tokens::LessOrEqual: - case Tokens::Greater: - case Tokens::GreaterOrEqual: - case Tokens::EOI: - throw Error::InvalidArgument("query_parser", std::string(message)); - default: - break; - } - - return *tokens_[pos_++]; - } - - private: - std::vector tokens_; - size_t pos_ = 0; -}; - -static ComparisonKind ComparisonKindFromToken(const Tokens type) { - switch (type) { - case Tokens::Equal: - return ComparisonKind::Equal; - case Tokens::NotEqual: - return ComparisonKind::NotEqual; - case Tokens::Less: - return ComparisonKind::Less; - case Tokens::LessOrEqual: - return ComparisonKind::LessOrEqual; - case Tokens::Greater: - return ComparisonKind::Greater; - case Tokens::GreaterOrEqual: - return ComparisonKind::GreaterOrEqual; - default: - throw Error::InvalidArgument("query_parser", "expected comparison operator in WHERE"); - } -} - -static AggSpec ParseAggregate(TokenCursor& cursor) { - const Token& function_token = cursor.ConsumeFunctionToken("expected aggregate function"); - - AggSpec spec; - spec.function = &ResolveAggFunc(function_token.GetText()); - - cursor.Consume(Tokens::OpenBracket, "expected '(' after aggregate function"); - - if (cursor.TryConsume(Tokens::Asterisk)) { - if (!spec.function->star) { - throw Error::InvalidArgument("query_parser", - std::string(spec.function->canonical_name) + " does not support '*'"); - } - spec.star = true; - } else { - spec.distinct = cursor.TryConsume(Tokens::Distinct); - if (spec.distinct && !spec.function->distinct) { - throw Error::InvalidArgument("query_parser", - std::string(spec.function->canonical_name) + " does not support DISTINCT"); - } - const auto& token = cursor.Consume(Tokens::NameToken, "expected column name in aggregate"); - spec.column_name = std::string(token.GetText()); - } - - cursor.Consume(Tokens::CloseBracket, "expected ')' after aggregate arguments"); - spec.output_name = FormatAgg(*spec.function, spec.column_name, spec.distinct, spec.star); - - return spec; -} - -ParsedQuery ParseQuery(const std::string_view query) { - auto tokenized = TokenizeSql(query); - if (!tokenized.has_value()) { - throw tokenized.error(); - } - - TokenCursor cursor(std::move(tokenized.value())); - - ParsedQuery parsed; - cursor.Consume(Tokens::Select, "expected SELECT"); - - parsed.aggregates.push_back(ParseAggregate(cursor)); - while (cursor.TryConsume(Tokens::Comma)) { - parsed.aggregates.push_back(ParseAggregate(cursor)); - } - - cursor.Consume(Tokens::From, "expected FROM"); - parsed.table_name = std::string(cursor.Consume(Tokens::NameToken, "expected table name").GetText()); - - if (cursor.TryConsume(Tokens::Where)) { - FilterSpec filter; - filter.column_name = - std::string(cursor.Consume(Tokens::NameToken, "expected column name after WHERE").GetText()); - - const Tokens comparison_token = cursor.Peek().GetType(); - filter.comparison = ComparisonKindFromToken(comparison_token); - - cursor.Consume(comparison_token, "expected comparison operator in WHERE"); - - if (cursor.TryConsume(Tokens::Minus)) { - const Token& literal = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after '-'"); - filter.literal = - ParsedLiteral{.text = "-" + std::string(literal.GetText()), .token_type = Tokens::NumericLiteral}; - } else { - const Token& literal = cursor.Peek(); - switch (literal.GetType()) { - case Tokens::NumericLiteral: - case Tokens::StringLiteral: - case Tokens::NameToken: - filter.literal = ParsedLiteral{ - .text = std::string(literal.GetText()), - .token_type = literal.GetType(), - }; - cursor.Consume(literal.GetType(), "expected literal after comparison operator"); - break; - default: - throw Error::InvalidArgument("query_parser", "unsupported literal in WHERE"); - } - } - - parsed.filter = std::move(filter); - } - - cursor.TryConsume(Tokens::Semicolon); - cursor.Consume(Tokens::EOI, "unexpected trailing tokens"); - - return parsed; -} diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp deleted file mode 100644 index 80a245e..0000000 --- a/src/executor/query_planner.cpp +++ /dev/null @@ -1,118 +0,0 @@ -#include - -#include "ascii.h" -#include "columnar_batch_io.h" -#include "executor.h" - -static size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { - const std::string needle = ToLowerAscii(column_name); - for (size_t i = 0; i < schema.columns.size(); ++i) { - if (ToLowerAscii(schema.columns[i].name) == needle) { - return i; - } - } - throw Error::InvalidArgument("query_planner", "unknown column '" + std::string(column_name) + "'"); -} - -static std::string UnescapeSqlString(const std::string_view text) { - if (text.size() < 2 || text.front() != '\'' || text.back() != '\'') { - throw Error::InvalidArgument("query_planner", "invalid string literal"); - } - - std::string result; - result.reserve(text.size() - 2); - - for (size_t i = 1; i + 1 < text.size(); ++i) { - if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { - result.push_back('\''); - ++i; - continue; - } - result.push_back(text[i]); - } - - return result; -} - -static std::string NormalizeLiteral(const ParsedLiteral& literal, const ColumnType type) { - switch (literal.token_type) { - case Tokens::StringLiteral: - return UnescapeSqlString(literal.text); - case Tokens::NumericLiteral: - case Tokens::NameToken: - if (type == ColumnType::String && literal.token_type == Tokens::NameToken) { - return literal.text; - } - return literal.text; - default: - throw Error::InvalidArgument("query_planner", "unsupported literal type"); - } -} - -PlannedQuery PlanQuery(const ParsedQuery& parsed, - const std::unordered_map& tables) { - PlannedQuery planned; - - const auto table_it = tables.find(ToLowerAscii(parsed.table_name)); - if (table_it == tables.end()) { - throw Error::NotFound("query_planner", "unknown table", parsed.table_name); - } - - planned.table_path = table_it->second; - - ColumnarBatchReader reader(planned.table_path); - const Schema& schema = reader.GetSchema(); - - const auto register_projection = [&](const size_t source_index) -> size_t { - const auto it = std::ranges::find(planned.projection_indexes, source_index); - if (it != planned.projection_indexes.end()) { - return static_cast(std::distance(planned.projection_indexes.begin(), it)); - } - planned.projection_indexes.push_back(source_index); - return planned.projection_indexes.size() - 1; - }; - - if (parsed.filter.has_value()) { - const size_t source_index = FindColumnIndex(schema, parsed.filter->column_name); - const ColumnType column_type = schema.columns[source_index].type; - - planned.filter = PlannedFilter{ - .column_index = register_projection(source_index), - .column_type = column_type, - .comparison = parsed.filter->comparison, - .literal_text = NormalizeLiteral(parsed.filter->literal, column_type), - }; - } - - planned.aggregates.reserve(parsed.aggregates.size()); - - for (const auto& aggregate : parsed.aggregates) { - PlannedAgg planned_aggregate{ - .function = aggregate.function, - .distinct = aggregate.distinct, - .star = aggregate.star, - .column_index = std::nullopt, - .input_type = ColumnType::String, - .output_name = aggregate.output_name, - }; - - if (!aggregate.star) { - const size_t source_index = FindColumnIndex(schema, aggregate.column_name); - planned_aggregate.column_index = register_projection(source_index); - planned_aggregate.input_type = schema.columns[source_index].type; - if (!AggSupportsInputType(*aggregate.function, planned_aggregate.input_type)) { - throw Error::Unsupported("query_planner", - "aggregate " + std::string(aggregate.function->canonical_name) + - " does not support column '" + aggregate.column_name + "'"); - } - } - - planned.aggregates.push_back(std::move(planned_aggregate)); - } - - if (planned.projection_indexes.empty() && !schema.columns.empty()) { - planned.projection_indexes.push_back(0); - } - - return planned; -} diff --git a/src/io/batch_io.cpp b/src/io/batch_io.cpp deleted file mode 100644 index f66042a..0000000 --- a/src/io/batch_io.cpp +++ /dev/null @@ -1,328 +0,0 @@ -#include -#include -#include - -#include "columnar_batch_io.h" -#include "csv.h" -#include "csv_batch_io.h" -#include "error.h" -#include "metadata.h" -#include "stream.h" - -static constexpr std::string_view kColumnarMagic = "CLMN"; - -static void SeekRead(const std::filesystem::path& path, std::ifstream& in, const uint64_t pos) { - in.seekg(pos); - if (!in) { - throw Error::PathIo("batch_io", path, "seek file"); - } -} - -static uint64_t TellWrite(const std::filesystem::path& path, std::ofstream& out) { - const auto pos = out.tellp(); - if (pos == -1) { - throw Error::PathIo("batch_io", path, "tell file"); - } - return pos; -} - -static void FlushWrite(const std::filesystem::path& path, std::ofstream& out) { - out.flush(); - if (!out) { - throw Error::PathIo("batch_io", path, "write file"); - } -} - -static uint64_t AddChecked(const uint64_t current, const uint64_t add) { - if (add > std::numeric_limits::max() - current) { - throw Error::Overflow("batch_io", "batch byte size overflow"); - } - return current + add; -} - -static uint64_t EstimateRowBytes(const std::vector>& columns, - const std::vector& row) { - uint64_t bytes = 0; - for (size_t i = 0; i < columns.size(); ++i) { - bytes = AddChecked(bytes, columns[i]->EstimateSizeFromString(row[i])); - } - return bytes; -} - -static void ValidateSizing(const BatchSizing& sizing) { - if (sizing.max_rows && *sizing.max_rows == 0) { - throw Error::InvalidArgument("batch_io", "max rows must be > 0"); - } - if (sizing.max_values && *sizing.max_values == 0) { - throw Error::InvalidArgument("batch_io", "max values must be > 0"); - } - if (sizing.max_bytes && *sizing.max_bytes == 0) { - throw Error::InvalidArgument("batch_io", "max bytes must be > 0"); - } -} - -static void ReserveForSizing(const Batch& batch, const BatchSizing& sizing, const size_t column_count) { - if (sizing.max_rows) { - batch.Reserve(*sizing.max_rows); - return; - } - if (sizing.max_values && column_count > 0) { - batch.Reserve(std::max(1, *sizing.max_values / column_count)); - } -} - -bool BatchSizing::WouldExceed(const size_t next_rows, const size_t column_count, const uint64_t next_bytes) const { - if (max_rows && next_rows > *max_rows) { - return true; - } - if (max_values && column_count > 0) { - if (next_rows > *max_values / column_count) { - return true; - } - } - if (max_bytes && next_bytes > *max_bytes) { - return true; - } - return false; -} - -CsvBatchReader::CsvBatchReader(const std::filesystem::path& path, Schema schema, BatchSizing sizing) - : csv_reader_(path), schema_(std::move(schema)), sizing_(std::move(sizing)) { - if (schema_.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns"); - } - ValidateSizing(sizing_); -} - -std::optional CsvBatchReader::ReadNext() { - if (reached_eof_ && !pending_row_) { - return std::nullopt; - } - - Batch batch(schema_); - - const size_t column_count = schema_.columns.size(); - ReserveForSizing(batch, sizing_, column_count); - const auto& columns = batch.GetColumns(); - - std::vector row; - row.reserve(column_count); - - size_t rows = 0; - uint64_t bytes = 0; - - while (true) { - if (pending_row_) { - row = std::move(*pending_row_); - pending_row_.reset(); - } else { - if (!csv_reader_.ReadRow(row)) { - reached_eof_ = true; - break; - } - } - - if (row.size() != column_count) { - throw Error::Mismatch("batch_io", "data csv column count mismatch"); - } - - const size_t next_rows = rows + 1; - uint64_t next_bytes = bytes; - - if (sizing_.max_bytes) { - next_bytes = AddChecked(bytes, EstimateRowBytes(columns, row)); - } - - if (rows > 0 && sizing_.WouldExceed(next_rows, column_count, next_bytes)) { - pending_row_ = std::move(row); - break; - } - - for (size_t col = 0; col < column_count; ++col) { - columns[col]->AppendFromString(row[col]); - } - - rows = next_rows; - bytes = next_bytes; - } - - if (rows == 0 && reached_eof_ && !pending_row_) { - return std::nullopt; - } - - return batch; -} - -CsvBatchWriter::CsvBatchWriter(const std::filesystem::path& path, Schema schema) - : csv_writer_(path), schema_(std::move(schema)) { - if (schema_.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns"); - } -} - -void CsvBatchWriter::Write(const Batch& batch) { - batch.Validate(); - if (batch.GetSchema() != schema_) { - throw Error::Mismatch("batch_io", "batch schema mismatch"); - } - - const size_t column_count = schema_.columns.size(); - const size_t row_count = batch.RowsCount(); - - std::vector row(column_count); - - for (size_t row_index = 0; row_index < row_count; ++row_index) { - for (size_t col = 0; col < column_count; ++col) { - row[col] = batch.ColumnAt(col).ValueAsString(row_index); - } - csv_writer_.WriteRow(row); - } -} - -void CsvBatchWriter::Flush() { csv_writer_.Flush(); } - -void AppendBatchRows(const Batch& batch, std::vector>& rows) { - const size_t row_count = batch.RowsCount(); - const size_t column_count = batch.ColumnsCount(); - - for (size_t row = 0; row < row_count; ++row) { - std::vector values; - values.reserve(column_count); - - for (size_t col = 0; col < column_count; ++col) { - values.push_back(batch.ColumnAt(col).ValueAsString(row)); - } - - rows.push_back(std::move(values)); - } -} - -void WriteBatchCsv(const std::filesystem::path& path, const Batch& batch) { - CsvBatchWriter writer(path, batch.GetSchema()); - writer.Write(batch); - writer.Flush(); -} - -ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::path& path) { - const auto file_metadata = TryGetFileMetadata(path); - - if (!file_metadata || !file_metadata->is_regular) { - throw Error::NotFound("batch_io", "columnar file not found", path.string()); - } - - const uint64_t file_size = file_metadata->size; - constexpr uint64_t kFooterSize = sizeof(uint64_t) + kColumnarMagic.size(); - - if (file_size < kFooterSize) { - throw Error::InvalidData("batch_io", "columnar file is too small", path.string()); - } - - std::ifstream in = OpenInputFile(path); - SeekRead(path, in, file_size - kFooterSize); - - const uint64_t metadata_size = ReadStream(in); - std::string magic_read(kColumnarMagic.size(), '\0'); - ReadBytes(in, magic_read.data(), magic_read.size()); - - if (magic_read != kColumnarMagic) { - throw Error::InvalidData("batch_io", "invalid columnar magic", path.string()); - } - - if (metadata_size > file_size - kFooterSize) { - throw Error::InvalidData("batch_io", "metadata size exceeds file size", path.string()); - } - - SeekRead(path, in, file_size - kFooterSize - metadata_size); - - return ReadMetadata(in); -} - -ColumnarBatchReader::ColumnarBatchReader(const std::filesystem::path& path) - : path_(path), in_(OpenInputFile(path)), metadata_(ReadFileMetadata(path)) { - if (metadata_.schema.columns.empty()) { - throw Error::InvalidData("batch_io", "columnar schema is empty", path.string()); - } -} - -std::optional ColumnarBatchReader::ReadNext() { - if (next_group_ >= metadata_.row_groups.size()) { - return std::nullopt; - } - - const auto& [row_count, row_group_columns] = metadata_.row_groups[next_group_++]; - Batch batch(metadata_.schema, row_count); - - const auto& columns = batch.GetColumns(); - - if (row_group_columns.size() != columns.size()) { - throw Error::Mismatch("batch_io", "row group column count mismatch", path_.string()); - } - - for (size_t col = 0; col < columns.size(); ++col) { - const auto& [offset, size] = row_group_columns[col]; - SeekRead(path_, in_, offset); - columns[col]->ReadFrom(in_, row_count, size); - } - - return batch; -} - -ColumnarBatchWriter::ColumnarBatchWriter(const std::filesystem::path& path, Schema schema) - : path_(path), out_(OpenOutputFile(path)) { - if (schema.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns", path.string()); - } - metadata_.schema = std::move(schema); -} - -void ColumnarBatchWriter::Write(const Batch& batch) { - if (finalized_) { - throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); - } - - batch.Validate(); - if (batch.GetSchema() != metadata_.schema) { - throw Error::Mismatch("batch_io", "batch schema mismatch", path_.string()); - } - - const size_t row_count = batch.RowsCount(); - - if (row_count == 0) { - return; - } - if (row_count > std::numeric_limits::max()) { - throw Error::Overflow("batch_io", "row group exceeds supported size", path_.string()); - } - - RowGroupMetadata group; - group.row_count = row_count; - group.columns.reserve(batch.ColumnsCount()); - - for (const auto& column : batch.GetColumns()) { - const uint64_t offset = TellWrite(path_, out_); - column->WriteTo(out_); - const uint64_t end = TellWrite(path_, out_); - group.columns.push_back(ColumnChunkMetadata{offset, end - offset}); - } - - metadata_.row_groups.push_back(std::move(group)); -} - -void ColumnarBatchWriter::Flush() { FlushWrite(path_, out_); } - -void ColumnarBatchWriter::Finalize() { - if (finalized_) { - throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); - } - - const uint64_t metadata_start = TellWrite(path_, out_); - WriteMetadata(out_, metadata_); - const uint64_t metadata_end = TellWrite(path_, out_); - const uint64_t metadata_size = metadata_end - metadata_start; - - WriteStream(out_, metadata_size); - WriteBytes(out_, kColumnarMagic); - - FlushWrite(path_, out_); - finalized_ = true; -} diff --git a/src/io/csv.cpp b/src/io/csv.cpp deleted file mode 100644 index 408d572..0000000 --- a/src/io/csv.cpp +++ /dev/null @@ -1,196 +0,0 @@ -#include "csv.h" - -#include - -#include "error.h" -#include "fileio.h" - -static bool NeedsCsvQuotes(const std::string_view value) { - return value.find_first_of(",\"\n\r") != std::string_view::npos; -} - -CsvReader::CsvReader(std::istream& in) : in_(&in) {} - -CsvReader::CsvReader(const std::filesystem::path& path) : owned_in_(OpenInputFile(path)), in_(&owned_in_) {} - -CsvReader::CsvReader(CsvReader&& other) noexcept : owned_in_(std::move(other.owned_in_)) { - RebindAfterMove(other.in_ == &other.owned_in_, other.in_); - other.in_ = nullptr; -} - -CsvReader& CsvReader::operator=(CsvReader&& other) noexcept { - if (this != &other) { - owned_in_ = std::move(other.owned_in_); - RebindAfterMove(other.in_ == &other.owned_in_, other.in_); - other.in_ = nullptr; - } - return *this; -} - -void CsvReader::RebindAfterMove(const bool uses_owned_stream, std::istream* source_stream) noexcept { - if (uses_owned_stream) { - in_ = &owned_in_; - return; - } - in_ = source_stream; -} - -bool CsvReader::ReadRow(std::vector& row) const { - row.clear(); - row.emplace_back(); - - auto* buffer = in_->rdbuf(); - using Traits = std::streambuf::traits_type; - /* До этого чтение происходило через istream::get() и peek() - * Traits требуется только как вспомогательный тип для: - * - eof() - * - to_chat_type() - * - eq_int_type() - */ - - bool in_quotes = false; - bool saw_data = false; - - while (true) { - const Traits::int_type next = buffer->sbumpc(); - - if (Traits::eq_int_type(next, Traits::eof())) { - if (!saw_data) { - return false; - } - if (in_quotes) { - throw Error::InvalidData("csv", "unexpected EOF inside quoted field"); - } - return true; - } - - saw_data = true; - const char ch = Traits::to_char_type(next); - - if (in_quotes) { - if (ch == '"') { - const Traits::int_type peek = buffer->sgetc(); - if (Traits::to_char_type(peek) == '"') { - buffer->sbumpc(); - row.back().push_back('"'); - } else { - in_quotes = false; - } - } else { - row.back().push_back(ch); - } - continue; - } - - if (ch == '"') { - if (row.back().empty()) { - in_quotes = true; - } else { - row.back().push_back('"'); - } - continue; - } - - if (ch == ',') { - row.emplace_back(); - continue; - } - - if (ch == '\n') { - return true; - } - - if (ch == '\r') { - if (Traits::to_char_type(buffer->sgetc()) == '\n') { - buffer->sbumpc(); - } - return true; - } - - row.back().push_back(ch); - } -} - -CsvWriter::CsvWriter(std::ostream& out) : out_(&out) {} - -CsvWriter::CsvWriter(const std::filesystem::path& path) : owned_out_(OpenOutputFile(path)), out_(&owned_out_) {} - -CsvWriter::CsvWriter(CsvWriter&& other) noexcept : owned_out_(std::move(other.owned_out_)) { - RebindAfterMove(other.out_ == &other.owned_out_, other.out_); - other.out_ = nullptr; -} - -CsvWriter& CsvWriter::operator=(CsvWriter&& other) noexcept { - if (this != &other) { - owned_out_ = std::move(other.owned_out_); - RebindAfterMove(other.out_ == &other.owned_out_, other.out_); - other.out_ = nullptr; - } - return *this; -} - -void CsvWriter::RebindAfterMove(const bool uses_owned_stream, std::ostream* source_stream) noexcept { - if (uses_owned_stream) { - out_ = &owned_out_; - return; - } - out_ = source_stream; -} - -void CsvWriter::WriteRow(const std::vector& row) const { - for (size_t i = 0; i < row.size(); ++i) { - if (i > 0) { - *out_ << ','; - } - - const std::string& value = row[i]; - if (!NeedsCsvQuotes(value)) { - *out_ << value; - continue; - } - - *out_ << '"'; - for (const char ch : value) { - if (ch == '"') { - *out_ << "\"\""; - } else { - *out_ << ch; - } - } - *out_ << '"'; - } - *out_ << '\n'; - if (!*out_) { - throw Error::Io("csv", "failed to write csv row"); - } -} - -void CsvWriter::Flush() const { - out_->flush(); - if (!*out_) { - throw Error::Io("csv", "failed to write csv"); - } -} - -std::vector> ReadRows(const std::filesystem::path& path) { - const CsvReader reader(path); - - std::vector> rows; - std::vector row; - - while (reader.ReadRow(row)) { - rows.push_back(row); - } - - return rows; -} - -void WriteRows(const std::filesystem::path& path, const std::vector>& rows) { - const CsvWriter writer(path); - - for (const auto& row : rows) { - writer.WriteRow(row); - } - - writer.Flush(); -} diff --git a/src/io/fileio.cpp b/src/io/fileio.cpp deleted file mode 100644 index a65ac38..0000000 --- a/src/io/fileio.cpp +++ /dev/null @@ -1,126 +0,0 @@ -#include "fileio.h" - -#include - -#include "error.h" - -std::optional TryGetFileMetadata(const std::filesystem::path& path) { - std::error_code ec; - if (!std::filesystem::exists(path, ec)) { - if (ec) { - throw Error::PathIo("fileio", path, "check existence"); - } - return std::nullopt; - } - - FileMetadata metadata; - const auto status = std::filesystem::status(path, ec); - if (ec) { - throw Error::PathIo("fileio", path, "read status"); - } - - metadata.is_regular = std::filesystem::is_regular_file(status); - metadata.is_directory = std::filesystem::is_directory(status); - if (metadata.is_regular) { - metadata.size = std::filesystem::file_size(path, ec); - if (ec) { - throw Error::PathIo("fileio", path, "read file size"); - } - } - - metadata.last_write_time = std::filesystem::last_write_time(path, ec); - if (ec) { - throw Error::PathIo("fileio", path, "read last write time"); - } - - return metadata; -} - -bool FileExists(const std::filesystem::path& path) { - std::error_code ec; - const bool exists = std::filesystem::exists(path, ec); - if (ec) { - throw Error::PathIo("fileio", path, "check existence"); - } - return exists; -} - -void EnsureParentDirectory(const std::filesystem::path& path) { - const auto parent = path.parent_path(); - if (parent.empty()) { - return; - } - - std::error_code ec; - std::filesystem::create_directories(parent, ec); - if (ec) { - throw Error::PathIo("fileio", path, "create parent directories"); - } -} - -std::ifstream OpenInputFile(const std::filesystem::path& path) { - std::ifstream file(path, std::ios::binary); - if (!file.is_open()) { - throw Error::PathIo("fileio", path, "open for read"); - } - return file; -} - -std::ofstream OpenOutputFile(const std::filesystem::path& path, const FileOpenMode mode) { - std::ofstream file(path, std::ios::binary | (mode == FileOpenMode::Append ? std::ios::app : std::ios::trunc)); - if (!file.is_open()) { - throw Error::PathIo("fileio", path, mode == FileOpenMode::Append ? "open for append" : "open for write"); - } - return file; -} - -std::string ReadTextFile(const std::filesystem::path& path) { - std::ifstream file = OpenInputFile(path); - return std::string(std::istreambuf_iterator(file), std::istreambuf_iterator()); -} - -std::vector ReadFileBytes(const std::filesystem::path& path) { - std::ifstream file = OpenInputFile(path); - file.seekg(0, std::ios::end); - if (!file) { - throw Error::PathIo("fileio", path, "read file size"); - } - - const auto end_pos = file.tellg(); - if (end_pos < 0) { - throw Error::PathIo("fileio", path, "read file size"); - } - - std::vector data(end_pos); - if (end_pos == 0) { - return data; - } - - file.seekg(0, std::ios::beg); - file.read(reinterpret_cast(data.data()), data.size()); - - if (!file) { - throw Error::PathIo("fileio", path, "read file"); - } - - return data; -} - -void WriteFileBytes(const std::filesystem::path& path, const std::span bytes) { - std::ofstream file = OpenOutputFile(path); - if (!bytes.empty()) { - file.write(reinterpret_cast(bytes.data()), bytes.size()); - if (!file) { - throw Error::PathIo("fileio", path, "write file"); - } - } - - file.flush(); - if (!file) { - throw Error::PathIo("fileio", path, "write file"); - } -} - -void WriteFileBytes(const std::filesystem::path& path, const std::vector& bytes) { - WriteFileBytes(path, std::span(bytes.data(), bytes.size())); -} diff --git a/src/io/stream.cpp b/src/io/stream.cpp deleted file mode 100644 index 5193bfb..0000000 --- a/src/io/stream.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "stream.h" - -#include "error.h" - -void ReadBytes(std::istream& in, char* dst, const size_t size) { - if (size == 0) { - return; - } - in.read(dst, size); - if (in.gcount() != static_cast(size) || !in) { - throw Error::Io("stream", "failed to read bytes"); - } -} - -void WriteBytes(std::ostream& out, const std::string_view bytes) { - if (bytes.empty()) { - return; - } - out.write(bytes.data(), bytes.size()); - if (!out) { - throw Error::Io("stream", "failed to write bytes"); - } -} diff --git a/src/model/batch.cpp b/src/model/batch.cpp deleted file mode 100644 index bad890d..0000000 --- a/src/model/batch.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "batch.h" - -#include - -#include "error.h" - -static std::vector> CloneColumns(const std::vector>& columns) { - std::vector> clones; - clones.reserve(columns.size()); - for (const auto& column : columns) { - clones.push_back(column->Clone()); - } - return clones; -} - -Batch::Batch(Schema schema) : schema_(std::move(schema)) { - columns_.reserve(schema_.columns.size()); - for (const auto& [name, type] : schema_.columns) { - columns_.push_back(CreateColumn(type)); - } -} - -Batch::Batch(Schema schema, const size_t reserve_rows) : Batch(std::move(schema)) { Reserve(reserve_rows); } - -Batch::Batch(const Batch& other) : schema_(other.schema_), columns_(CloneColumns(other.columns_)) {} - -Batch& Batch::operator=(const Batch& other) { - if (this == &other) { - return *this; - } - - auto columns = CloneColumns(other.columns_); - schema_ = other.schema_; - columns_ = std::move(columns); - - return *this; -} - -size_t Batch::ColumnsCount() const { return columns_.size(); } - -size_t Batch::RowsCount() const { - if (columns_.empty()) { - return 0; - } - return columns_.front()->Size(); -} - -void Batch::Reserve(const size_t n) const { - for (auto& column : columns_) { - column->Reserve(n); - } -} - -const Column& Batch::ColumnAt(const size_t i) const { - if (i >= columns_.size()) { - throw Error::OutOfRange("batch", "column index out of range"); - } - return *columns_[i]; -} - -Column& Batch::ColumnAt(const size_t i) { - if (i >= columns_.size()) { - throw Error::OutOfRange("batch", "column index out of range"); - } - return *columns_[i]; -} - -void Batch::Validate() const { - if (schema_.columns.size() != columns_.size()) { - throw Error::Mismatch("batch", "column count mismatch"); - } - - if (columns_.empty()) { - return; - } - - const size_t rows = columns_.front()->Size(); - for (size_t i = 0; i < columns_.size(); ++i) { - if (columns_[i]->Type() != schema_.columns[i].type) { - throw Error::Mismatch("batch", "column type mismatch"); - } - if (columns_[i]->Size() != rows) { - throw Error::Mismatch("batch", "column row count mismatch"); - } - } -} diff --git a/src/model/column.cpp b/src/model/column.cpp deleted file mode 100644 index 7eded02..0000000 --- a/src/model/column.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "column.h" - -#include - -#include "column_boolean.h" -#include "column_character.h" -#include "column_date.h" -#include "column_int128.h" -#include "column_int16.h" -#include "column_int32.h" -#include "column_int64.h" -#include "column_string.h" -#include "column_timestamp.h" -#include "error.h" - -std::unique_ptr CreateColumn(const ColumnType type) { - switch (type) { - case ColumnType::Int64: - return std::make_unique(); - case ColumnType::String: - return std::make_unique(); - case ColumnType::Boolean: - return std::make_unique(); - case ColumnType::Int16: - return std::make_unique(); - case ColumnType::Int32: - return std::make_unique(); - case ColumnType::Int128: - return std::make_unique(); - case ColumnType::Date: - return std::make_unique(); - case ColumnType::Timestamp: - return std::make_unique(); - case ColumnType::Character: - return std::make_unique(); - } - throw Error::Unsupported("column", "unsupported column type"); -} - -Column::Column(const ColumnType type) : type_(type) {} - -void Column::CheckRowIndex(const char* module, const size_t row, const size_t size) { - if (row >= size) { - throw Error::OutOfRange(module, "row index out of range"); - } -} diff --git a/src/model/column_boolean.cpp b/src/model/column_boolean.cpp deleted file mode 100644 index 4959ec5..0000000 --- a/src/model/column_boolean.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_boolean.h" - -#include "parsing.h" - -void BooleanColumn::AppendFromString(const std::string& value) { AppendValue(ParseBoolean(value) ? 1 : 0); } - -std::string BooleanColumn::ValueAsString(const size_t row) const { return BooleanToString(ValueAt(row) != 0); } diff --git a/src/model/column_character.cpp b/src/model/column_character.cpp deleted file mode 100644 index 5184b18..0000000 --- a/src/model/column_character.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_character.h" - -#include "parsing.h" - -void CharacterColumn::AppendFromString(const std::string& value) { AppendValue(ParseCharacter(value)); } - -std::string CharacterColumn::ValueAsString(const size_t row) const { return std::string(1, ValueAt(row)); } diff --git a/src/model/column_date.cpp b/src/model/column_date.cpp deleted file mode 100644 index 4bfcc9e..0000000 --- a/src/model/column_date.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_date.h" - -#include "parsing.h" - -void DateColumn::AppendFromString(const std::string& value) { AppendValue(ParseDate(value)); } - -std::string DateColumn::ValueAsString(const size_t row) const { return DateToString(ValueAt(row)); } diff --git a/src/model/column_int128.cpp b/src/model/column_int128.cpp deleted file mode 100644 index 161ab39..0000000 --- a/src/model/column_int128.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_int128.h" - -#include "parsing.h" - -void Int128Column::AppendFromString(const std::string& value) { AppendValue(ParseInt128(value)); } - -std::string Int128Column::ValueAsString(const size_t row) const { return Int128ToString(ValueAt(row)); } diff --git a/src/model/column_int16.cpp b/src/model/column_int16.cpp deleted file mode 100644 index 94415e8..0000000 --- a/src/model/column_int16.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_int16.h" - -#include "parsing.h" - -void Int16Column::AppendFromString(const std::string& value) { AppendValue(ParseInt16(value)); } - -std::string Int16Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_int32.cpp b/src/model/column_int32.cpp deleted file mode 100644 index b9353b9..0000000 --- a/src/model/column_int32.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_int32.h" - -#include "parsing.h" - -void Int32Column::AppendFromString(const std::string& value) { AppendValue(ParseInt32(value)); } - -std::string Int32Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_int64.cpp b/src/model/column_int64.cpp deleted file mode 100644 index f6c2399..0000000 --- a/src/model/column_int64.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_int64.h" - -#include "parsing.h" - -void Int64Column::AppendFromString(const std::string& value) { AppendValue(ParseInt64(value)); } - -std::string Int64Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_string.cpp b/src/model/column_string.cpp deleted file mode 100644 index 31cea45..0000000 --- a/src/model/column_string.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "column_string.h" - -void StringColumn::AppendFromString(const std::string& value) { AppendValue(value); } - -std::string StringColumn::ValueAsString(const size_t row) const { return ValueAt(row); } diff --git a/src/model/column_timestamp.cpp b/src/model/column_timestamp.cpp deleted file mode 100644 index 42867fe..0000000 --- a/src/model/column_timestamp.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_timestamp.h" - -#include "parsing.h" - -void TimestampColumn::AppendFromString(const std::string& value) { AppendValue(ParseTimestamp(value)); } - -std::string TimestampColumn::ValueAsString(const size_t row) const { return TimestampToString(ValueAt(row)); } diff --git a/src/model/metadata.cpp b/src/model/metadata.cpp deleted file mode 100644 index 33736bc..0000000 --- a/src/model/metadata.cpp +++ /dev/null @@ -1,100 +0,0 @@ -#include "metadata.h" - -#include -#include - -#include "error.h" -#include "stream.h" - -static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { - switch (static_cast(type_byte)) { - case ColumnType::Int64: - case ColumnType::String: - case ColumnType::Boolean: - case ColumnType::Int16: - case ColumnType::Int32: - case ColumnType::Int128: - case ColumnType::Date: - case ColumnType::Timestamp: - case ColumnType::Character: - return static_cast(type_byte); - } - - throw Error::InvalidData("columnar", "unknown column type in metadata"); -} - -ColumnarMetadata ReadMetadata(std::istream& in) { - ColumnarMetadata metadata; - - const uint32_t column_count = ReadStream(in); - metadata.schema.columns.reserve(column_count); - - for (uint32_t i = 0; i < column_count; ++i) { - const uint32_t name_size = ReadStream(in); - std::string name(name_size, '\0'); - - ReadBytes(in, name.data(), name.size()); - const uint8_t type_byte = ReadStream(in); - const ColumnType type = ColumnTypeFromByte(type_byte); - - metadata.schema.columns.push_back(ColumnSchema{std::move(name), type}); - } - - const uint32_t row_group_count = ReadStream(in); - metadata.row_groups.reserve(row_group_count); - - for (uint32_t i = 0; i < row_group_count; ++i) { - RowGroupMetadata group; - group.row_count = ReadStream(in); - - const uint32_t columns_in_group = ReadStream(in); - if (columns_in_group != column_count) { - throw Error::Mismatch("columnar", "row group column count mismatch"); - } - - group.columns.reserve(columns_in_group); - - for (uint32_t col = 0; col < columns_in_group; ++col) { - ColumnChunkMetadata chunk; - chunk.offset = ReadStream(in); - chunk.size = ReadStream(in); - group.columns.push_back(chunk); - } - - metadata.row_groups.push_back(std::move(group)); - } - - return metadata; -} - -void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { - if (metadata.schema.columns.size() > std::numeric_limits::max()) { - throw Error::Overflow("columnar", "too many columns for metadata"); - } - - WriteStream(out, metadata.schema.columns.size()); - - for (const auto& [name, type] : metadata.schema.columns) { - if (name.size() > std::numeric_limits::max()) { - throw Error::Overflow("columnar", "column name too long"); - } - WriteStream(out, name.size()); - WriteBytes(out, name); - WriteStream(out, static_cast(type)); - } - - if (metadata.row_groups.size() > std::numeric_limits::max()) { - throw Error::Overflow("columnar", "too many row groups for metadata"); - } - - WriteStream(out, metadata.row_groups.size()); - - for (const auto& [row_count, columns] : metadata.row_groups) { - WriteStream(out, row_count); - WriteStream(out, columns.size()); - for (const auto& [offset, size] : columns) { - WriteStream(out, offset); - WriteStream(out, size); - } - } -} diff --git a/src/model/schema.cpp b/src/model/schema.cpp deleted file mode 100644 index 3c0aae4..0000000 --- a/src/model/schema.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include "schema.h" - -#include - -#include "csv.h" -#include "error.h" -#include "fileio.h" -#include "parsing.h" - -static bool CanParseAs(const ColumnType type, const std::string& value) { - try { - switch (type) { - case ColumnType::Boolean: - static_cast(ParseBoolean(value)); - return true; - case ColumnType::Int16: - static_cast(ParseInt16(value)); - return true; - case ColumnType::Int32: - static_cast(ParseInt32(value)); - return true; - case ColumnType::Int64: - static_cast(ParseInt64(value)); - return true; - case ColumnType::Int128: - static_cast(ParseInt128(value)); - return true; - case ColumnType::Date: - static_cast(ParseDate(value)); - return true; - case ColumnType::Timestamp: - static_cast(ParseTimestamp(value)); - return true; - case ColumnType::Character: - static_cast(ParseCharacter(value)); - return true; - case ColumnType::String: - return true; - } - } catch (const Error&) { - return false; - } - - throw Error::Unsupported("schema", "unknown column type"); -} - -static ColumnType InferColumnType(const std::vector& values) { - static constexpr std::array kInferenceOrder = { - ColumnType::Boolean, ColumnType::Int16, ColumnType::Int32, ColumnType::Int64, ColumnType::Int128, - ColumnType::Date, ColumnType::Timestamp, ColumnType::Character, ColumnType::String, - }; - - for (const ColumnType type : kInferenceOrder) { - bool matches = true; - for (const auto& value : values) { - if (!CanParseAs(type, value)) { - matches = false; - break; - } - } - if (matches) { - return type; - } - } - - return ColumnType::String; -} - -Schema ReadSchemaCsv(const std::filesystem::path& path) { - const CsvReader reader(path); - - Schema schema; - std::vector row; - - while (reader.ReadRow(row)) { - if (row.size() == 1 && row[0].empty()) { - continue; - } - if (row.size() != 2) { - throw Error::InvalidData("schema", "schema csv must have 2 columns", path.string()); - } - schema.columns.push_back(ColumnSchema{ - row[0], - ParseColumnType(row[1]), - }); - } - - if (schema.columns.empty()) { - throw Error::InvalidData("schema", "schema csv is empty", path.string()); - } - - return schema; -} - -Schema InferSchemaCsv(const std::filesystem::path& path) { - const CsvReader reader(path); - - std::vector> values_by_column; - std::vector row; - - while (reader.ReadRow(row)) { - if (values_by_column.empty()) { - values_by_column.resize(row.size()); - } else if (row.size() != values_by_column.size()) { - throw Error::InvalidData("schema", "csv rows have inconsistent column count", path.string()); - } - - for (size_t i = 0; i < row.size(); ++i) { - values_by_column[i].push_back(row[i]); - } - } - - if (values_by_column.empty()) { - throw Error::InvalidData("schema", "csv is empty", path.string()); - } - - Schema schema; - schema.columns.reserve(values_by_column.size()); - - for (size_t i = 0; i < values_by_column.size(); ++i) { - schema.columns.push_back(ColumnSchema{ - "column_" + std::to_string(i + 1), - InferColumnType(values_by_column[i]), - }); - } - - return schema; -} - -void WriteSchemaCsv(const std::filesystem::path& path, const Schema& schema) { - const CsvWriter writer(path); - - for (const auto& [name, type] : schema.columns) { - writer.WriteRow({name, ColumnTypeToString(type)}); - } - - writer.Flush(); -} diff --git a/src/sql_parser/tokenizer.cpp b/src/sql_parser/tokenizer.cpp deleted file mode 100644 index 04c5038..0000000 --- a/src/sql_parser/tokenizer.cpp +++ /dev/null @@ -1,174 +0,0 @@ -#include "tokenizer.h" - -#include -#include -#include -#include -#include - -static bool IsIdentifierStart(const char ch) { - const auto uch = static_cast(ch); - return std::isalpha(uch) != 0 || ch == '_'; -} - -static bool IsIdentifierPart(const char ch) { - const auto uch = static_cast(ch); - return std::isalnum(uch) != 0 || ch == '_'; -} - -Tokenizer::Tokenizer(std::string data) : query_(std::move(data)) {} - -Expected Tokenizer::GetNext() { - while (pos_ < query_.size() && std::isspace(static_cast(query_[pos_])) != 0) { - ++pos_; - } - - if (pos_ >= query_.size()) { - return MakeToken(Tokens::EOI, {}, query_.size()); - } - - const char ch = query_[pos_]; - if (IsIdentifierStart(ch)) { - const size_t start = pos_; - ++pos_; - - while (pos_ < query_.size() && IsIdentifierPart(query_[pos_])) { - ++pos_; - } - - const std::string_view text = std::string_view(query_).substr(start, pos_ - start); - - return MakeToken(ResolveIdentifierType(text), std::string(text), start); - } - - if (std::isdigit(static_cast(ch)) != 0) { - const size_t start = pos_; - ++pos_; - - while (pos_ < query_.size() && std::isdigit(static_cast(query_[pos_])) != 0) { - ++pos_; - } - - if (pos_ + 1 < query_.size() && query_[pos_] == '.' && - std::isdigit(static_cast(query_[pos_ + 1])) != 0) { - ++pos_; - while (pos_ < query_.size() && std::isdigit(static_cast(query_[pos_])) != 0) { - ++pos_; - } - } - - const std::string_view text = std::string_view(query_).substr(start, pos_ - start); - - return MakeToken(Tokens::NumericLiteral, std::string(text), start); - } - - if (ch == '\'') { - const size_t start = pos_; - ++pos_; - - bool terminated = false; - - while (pos_ < query_.size()) { - if (query_[pos_] != '\'') { - ++pos_; - continue; - } - - if (pos_ + 1 < query_.size() && query_[pos_ + 1] == '\'') { - pos_ += 2; - continue; - } - - ++pos_; - terminated = true; - - break; - } - - if (!terminated) { - return MakeUnterminatedString(start); - } - - const std::string_view text = std::string_view(query_).substr(start, pos_ - start); - - return MakeToken(Tokens::StringLiteral, std::string(text), start); - } - - const size_t start = pos_; - switch (ch) { - case '+': - ++pos_; - return MakeToken(Tokens::Plus, "+", start); - case '-': - ++pos_; - return MakeToken(Tokens::Minus, "-", start); - case '*': - ++pos_; - return MakeToken(Tokens::Asterisk, "*", start); - case '(': - ++pos_; - return MakeToken(Tokens::OpenBracket, "(", start); - case ')': - ++pos_; - return MakeToken(Tokens::CloseBracket, ")", start); - case ',': - ++pos_; - return MakeToken(Tokens::Comma, ",", start); - case '.': - ++pos_; - return MakeToken(Tokens::Dot, ".", start); - case ';': - ++pos_; - return MakeToken(Tokens::Semicolon, ";", start); - case '=': - ++pos_; - return MakeToken(Tokens::Equal, "=", start); - case '<': - ++pos_; - if (pos_ < query_.size() && query_[pos_] == '=') { - ++pos_; - return MakeToken(Tokens::LessOrEqual, "<=", start); - } - if (pos_ < query_.size() && query_[pos_] == '>') { - ++pos_; - return MakeToken(Tokens::NotEqual, "<>", start); - } - return MakeToken(Tokens::Less, "<", start); - case '>': - ++pos_; - if (pos_ < query_.size() && query_[pos_] == '=') { - ++pos_; - return MakeToken(Tokens::GreaterOrEqual, ">=", start); - } - return MakeToken(Tokens::Greater, ">", start); - case '!': - if (pos_ + 1 < query_.size() && query_[pos_ + 1] == '=') { - pos_ += 2; - return MakeToken(Tokens::NotEqual, "!=", start); - } - return MakeUnexpectedCharacter(ch, start); - default: - return MakeUnexpectedCharacter(ch, start); - } -} - -Expected> TokenizeSql(const std::string_view input) { - Tokenizer tokenizer{std::string(input)}; - - std::vector tokens; - tokens.reserve(input.size() / 2 + 1); - - while (true) { - auto token = tokenizer.GetNext(); - if (!token.has_value()) { - return tl::unexpected(token.error()); - } - - tokens.push_back(token.value()); - if (token.value()->GetType() == Tokens::EOI) { - break; - } - } - - return tokens; -} diff --git a/src/sql_parser/tokenizer_factory.cpp b/src/sql_parser/tokenizer_factory.cpp deleted file mode 100644 index 7ee46eb..0000000 --- a/src/sql_parser/tokenizer_factory.cpp +++ /dev/null @@ -1,130 +0,0 @@ -#include -#include -#include -#include -#include - -#include "ascii.h" -#include "tokenizer.h" - -static const std::unordered_map kKeywords = { - {"AS", Tokens::As}, {"AND", Tokens::And}, {"AVG", Tokens::Avg}, {"BY", Tokens::By}, - {"COUNT", Tokens::Count}, {"CREATE", Tokens::Create}, {"DISTINCT", Tokens::Distinct}, {"FROM", Tokens::From}, - {"GROUP", Tokens::Group}, {"LENGTH", Tokens::Length}, {"LIMIT", Tokens::Limit}, {"MAX", Tokens::Max}, - {"MIN", Tokens::Min}, {"ORDER", Tokens::Order}, {"SELECT", Tokens::Select}, {"SUM", Tokens::Sum}, - {"WHERE", Tokens::Where}, -}; - -static std::string DescribeCharacter(const char ch) { - switch (ch) { - case '\n': - return "\\n"; - case '\r': - return "\\r"; - case '\t': - return "\\t"; - default: - break; - } - - if (std::isprint(static_cast(ch)) != 0) { - return std::string(1, ch); - } - - return "code " + std::to_string(static_cast(static_cast(ch))); -} - -Tokens ResolveIdentifierType(const std::string_view text) { - const std::string upper = ToUpperAscii(text); - const auto it = kKeywords.find(upper); - if (it != kKeywords.end()) { - return it->second; - } - return Tokens::NameToken; -} - -TokenPtr MakeToken(const Tokens type, std::string text, const size_t offset) { - switch (type) { - case Tokens::NameToken: - return std::make_shared(std::move(text), offset); - case Tokens::NumericLiteral: - return std::make_shared(std::move(text), offset); - case Tokens::StringLiteral: - return std::make_shared(std::move(text), offset); - case Tokens::Select: - return std::make_shared(std::move(text), offset); - case Tokens::From: - return std::make_shared(std::move(text), offset); - case Tokens::Create: - return std::make_shared(std::move(text), offset); - case Tokens::As: - return std::make_shared(std::move(text), offset); - case Tokens::Sum: - return std::make_shared(std::move(text), offset); - case Tokens::Count: - return std::make_shared(std::move(text), offset); - case Tokens::Avg: - return std::make_shared(std::move(text), offset); - case Tokens::Max: - return std::make_shared(std::move(text), offset); - case Tokens::Min: - return std::make_shared(std::move(text), offset); - case Tokens::And: - return std::make_shared(std::move(text), offset); - case Tokens::Distinct: - return std::make_shared(std::move(text), offset); - case Tokens::Length: - return std::make_shared(std::move(text), offset); - case Tokens::Where: - return std::make_shared(std::move(text), offset); - case Tokens::By: - return std::make_shared(std::move(text), offset); - case Tokens::Group: - return std::make_shared(std::move(text), offset); - case Tokens::Order: - return std::make_shared(std::move(text), offset); - case Tokens::Limit: - return std::make_shared(std::move(text), offset); - case Tokens::Plus: - return std::make_shared(std::move(text), offset); - case Tokens::Minus: - return std::make_shared(std::move(text), offset); - case Tokens::Asterisk: - return std::make_shared(std::move(text), offset); - case Tokens::OpenBracket: - return std::make_shared(std::move(text), offset); - case Tokens::CloseBracket: - return std::make_shared(std::move(text), offset); - case Tokens::Comma: - return std::make_shared(std::move(text), offset); - case Tokens::Dot: - return std::make_shared(std::move(text), offset); - case Tokens::Semicolon: - return std::make_shared(std::move(text), offset); - case Tokens::Equal: - return std::make_shared(std::move(text), offset); - case Tokens::NotEqual: - return std::make_shared(std::move(text), offset); - case Tokens::Less: - return std::make_shared(std::move(text), offset); - case Tokens::LessOrEqual: - return std::make_shared(std::move(text), offset); - case Tokens::Greater: - return std::make_shared(std::move(text), offset); - case Tokens::GreaterOrEqual: - return std::make_shared(std::move(text), offset); - case Tokens::EOI: - return std::make_shared(std::move(text), offset); - } - return nullptr; -} - -Expected MakeUnexpectedCharacter(const char ch, const size_t offset) { - return tl::unexpected(Error::InvalidData("tokenizer_factory", "unexpected character '" + DescribeCharacter(ch) + - "' at position " + std::to_string(offset))); -} - -Expected MakeUnterminatedString(const size_t offset) { - return tl::unexpected( - Error::InvalidData("tokenizer_factory", "unterminated string literal at position " + std::to_string(offset))); -} diff --git a/src/sql_parser/tokenizer_tokens.cpp b/src/sql_parser/tokenizer_tokens.cpp deleted file mode 100644 index b1bc6b5..0000000 --- a/src/sql_parser/tokenizer_tokens.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include - -#include "tokenizer.h" - -std::string_view TokenTypeToString(const Tokens type) noexcept { - switch (type) { - case Tokens::NameToken: - return "name"; - case Tokens::NumericLiteral: - return "numeric_literal"; - case Tokens::StringLiteral: - return "string_literal"; - case Tokens::Select: - return "select"; - case Tokens::From: - return "from"; - case Tokens::Create: - return "create"; - case Tokens::As: - return "as"; - case Tokens::Sum: - return "sum"; - case Tokens::Count: - return "count"; - case Tokens::Avg: - return "avg"; - case Tokens::Max: - return "max"; - case Tokens::Min: - return "min"; - case Tokens::And: - return "and"; - case Tokens::Distinct: - return "distinct"; - case Tokens::Length: - return "length"; - case Tokens::Where: - return "where"; - case Tokens::By: - return "by"; - case Tokens::Group: - return "group"; - case Tokens::Order: - return "order"; - case Tokens::Limit: - return "limit"; - case Tokens::Plus: - return "plus"; - case Tokens::Minus: - return "minus"; - case Tokens::Asterisk: - return "asterisk"; - case Tokens::OpenBracket: - return "open_bracket"; - case Tokens::CloseBracket: - return "close_bracket"; - case Tokens::Comma: - return "comma"; - case Tokens::Dot: - return "dot"; - case Tokens::Semicolon: - return "semicolon"; - case Tokens::Equal: - return "equal"; - case Tokens::NotEqual: - return "not_equal"; - case Tokens::Less: - return "less"; - case Tokens::LessOrEqual: - return "less_or_equal"; - case Tokens::Greater: - return "greater"; - case Tokens::GreaterOrEqual: - return "greater_or_equal"; - case Tokens::EOI: - return "end_of_input"; - } - return "unknown"; -} - -Token::Token(std::string text, const size_t offset) : text_(std::move(text)), offset_(offset) {} - -std::string_view Token::GetText() const noexcept { return text_; } - -size_t Token::GetOffset() const noexcept { return offset_; } - -void Command::AddArg(TokenPtr arg) { args_.push_back(std::move(arg)); } - -const std::vector& Command::GetArgs() const noexcept { return args_; } - -// alert: не типобезопасная вещь, лучше переписать -#define COLUMNAR_DEFINE_TOKEN_TYPE(token_class, token_value) \ - Tokens token_class::GetType() const noexcept { return Tokens::token_value; } - -COLUMNAR_DEFINE_TOKEN_TYPE(TSelectToken, Select) -COLUMNAR_DEFINE_TOKEN_TYPE(TCreateToken, Create) -COLUMNAR_DEFINE_TOKEN_TYPE(TFromToken, From) -COLUMNAR_DEFINE_TOKEN_TYPE(TLimitToken, Limit) -COLUMNAR_DEFINE_TOKEN_TYPE(TOrderToken, Order) -COLUMNAR_DEFINE_TOKEN_TYPE(TWhereToken, Where) -COLUMNAR_DEFINE_TOKEN_TYPE(TGroupToken, Group) -COLUMNAR_DEFINE_TOKEN_TYPE(TSumToken, Sum) -COLUMNAR_DEFINE_TOKEN_TYPE(TMinToken, Min) -COLUMNAR_DEFINE_TOKEN_TYPE(TMaxToken, Max) -COLUMNAR_DEFINE_TOKEN_TYPE(TCountToken, Count) -COLUMNAR_DEFINE_TOKEN_TYPE(TDistinctToken, Distinct) -COLUMNAR_DEFINE_TOKEN_TYPE(TAvgToken, Avg) -COLUMNAR_DEFINE_TOKEN_TYPE(TLengthToken, Length) -COLUMNAR_DEFINE_TOKEN_TYPE(TPlusToken, Plus) -COLUMNAR_DEFINE_TOKEN_TYPE(TMinusToken, Minus) -COLUMNAR_DEFINE_TOKEN_TYPE(TNameToken, NameToken) -COLUMNAR_DEFINE_TOKEN_TYPE(TNumericLiteralToken, NumericLiteral) -COLUMNAR_DEFINE_TOKEN_TYPE(TStringLiteralToken, StringLiteral) -COLUMNAR_DEFINE_TOKEN_TYPE(TAsToken, As) -COLUMNAR_DEFINE_TOKEN_TYPE(TAndToken, And) -COLUMNAR_DEFINE_TOKEN_TYPE(TByToken, By) -COLUMNAR_DEFINE_TOKEN_TYPE(TOpenBracketToken, OpenBracket) -COLUMNAR_DEFINE_TOKEN_TYPE(TCommaToken, Comma) -COLUMNAR_DEFINE_TOKEN_TYPE(TCloseBracketToken, CloseBracket) -COLUMNAR_DEFINE_TOKEN_TYPE(TAsteriskToken, Asterisk) -COLUMNAR_DEFINE_TOKEN_TYPE(TDotToken, Dot) -COLUMNAR_DEFINE_TOKEN_TYPE(TSemicolonToken, Semicolon) -COLUMNAR_DEFINE_TOKEN_TYPE(TEqualToken, Equal) -COLUMNAR_DEFINE_TOKEN_TYPE(TNotEqualToken, NotEqual) -COLUMNAR_DEFINE_TOKEN_TYPE(TLessToken, Less) -COLUMNAR_DEFINE_TOKEN_TYPE(TLessOrEqualToken, LessOrEqual) -COLUMNAR_DEFINE_TOKEN_TYPE(TGreaterToken, Greater) -COLUMNAR_DEFINE_TOKEN_TYPE(TGreaterOrEqualToken, GreaterOrEqual) -COLUMNAR_DEFINE_TOKEN_TYPE(TEndOfInputToken, EOI) - -#undef COLUMNAR_DEFINE_TOKEN_TYPE - -std::string_view TNameToken::GetName() const noexcept { return GetText(); } diff --git a/src/support/ascii.cpp b/src/support/ascii.cpp deleted file mode 100644 index 19246ec..0000000 --- a/src/support/ascii.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include "ascii.h" - -#include - -std::string ToLowerAscii(const std::string_view input) { - std::string output; - output.reserve(input.size()); - for (const unsigned char ch : input) { - output.push_back(static_cast(std::tolower(ch))); - } - return output; -} - -std::string ToUpperAscii(const std::string_view input) { - std::string output; - output.reserve(input.size()); - for (const unsigned char ch : input) { - output.push_back(static_cast(std::toupper(ch))); - } - return output; -} diff --git a/src/support/error.cpp b/src/support/error.cpp deleted file mode 100644 index f374ff2..0000000 --- a/src/support/error.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "error.h" - -#include - -std::string Error::BuildWhat(const std::string_view module, const std::string_view message) { - return std::string(module) + ": " + std::string(message); -} - -Error::Error(const Code code, std::string module, std::string module_instance, const Type type, std::string message) - : code_(code), - module_(std::move(module)), - module_instance_(std::move(module_instance)), - type_(type), - message_(std::move(message)), - what_(BuildWhat(module_, message_)) {} - -Error Error::InvalidArgument(std::string module, std::string message, std::string module_instance) { - return Error(Code::InvalidArgument, std::move(module), std::move(module_instance), Type::Validation, - std::move(message)); -} - -Error Error::InvalidData(std::string module, std::string message, std::string module_instance) { - return Error(Code::InvalidData, std::move(module), std::move(module_instance), Type::Data, std::move(message)); -} - -Error Error::NotFound(std::string module, std::string message, std::string module_instance) { - return Error(Code::NotFound, std::move(module), std::move(module_instance), Type::State, std::move(message)); -} - -Error Error::OutOfRange(std::string module, std::string message, std::string module_instance) { - return Error(Code::OutOfRange, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); -} - -Error Error::Mismatch(std::string module, std::string message, std::string module_instance) { - return Error(Code::Mismatch, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); -} - -Error Error::Overflow(std::string module, std::string message, std::string module_instance) { - return Error(Code::Overflow, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); -} - -Error Error::InvalidState(std::string module, std::string message, std::string module_instance) { - return Error(Code::InvalidState, std::move(module), std::move(module_instance), Type::State, std::move(message)); -} - -Error Error::Unsupported(std::string module, std::string message, std::string module_instance) { - return Error(Code::Unsupported, std::move(module), std::move(module_instance), Type::Validation, - std::move(message)); -} - -Error Error::Io(std::string module, std::string message, std::string module_instance) { - return Error(Code::IoFailure, std::move(module), std::move(module_instance), Type::Io, std::move(message)); -} - -Error Error::PathIo(std::string module, const std::filesystem::path& path, const std::string_view action) { - return Io(std::move(module), "failed to " + std::string(action) + " '" + path.string() + "'", path.string()); -} - -const char* Error::what() const noexcept { return what_.c_str(); } - -Error::Code Error::GetCode() const noexcept { return code_; } - -Error::Type Error::GetType() const noexcept { return type_; } - -const std::string& Error::GetModule() const noexcept { return module_; } - -const std::string& Error::GetModuleInstance() const noexcept { return module_instance_; } - -const std::string& Error::GetMessage() const noexcept { return message_; } - -std::string_view Error::CodeToString(const Code code) noexcept { - switch (code) { - case Code::InvalidArgument: - return "invalid_argument"; - case Code::InvalidData: - return "invalid_data"; - case Code::NotFound: - return "not_found"; - case Code::OutOfRange: - return "out_of_range"; - case Code::Mismatch: - return "mismatch"; - case Code::Overflow: - return "overflow"; - case Code::InvalidState: - return "invalid_state"; - case Code::Unsupported: - return "unsupported"; - case Code::IoFailure: - return "io_failure"; - } - return "unknown"; -} - -std::string_view Error::TypeToString(const Type type) noexcept { - switch (type) { - case Type::Validation: - return "validation"; - case Type::Data: - return "data"; - case Type::State: - return "state"; - case Type::Io: - return "io"; - } - return "unknown"; -} diff --git a/src/support/parsing.cpp b/src/support/parsing.cpp deleted file mode 100644 index 2ddda39..0000000 --- a/src/support/parsing.cpp +++ /dev/null @@ -1,315 +0,0 @@ -#include "parsing.h" - -#include -#include -#include -#include -#include - -#include "ascii.h" -#include "error.h" - -template -T ParseInteger(const std::string& value, const std::string_view type_name) { - if (value.empty()) { - throw Error::InvalidData("parsing", "empty value for " + std::string(type_name)); - } - - T result = 0; - const char* begin = value.data(); - const char* end = begin + value.size(); - - if (const auto [ptr, ec] = std::from_chars(begin, end, result); ec != std::errc() || ptr != end) { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); - } - - return result; -} - -static unsigned ParseUnsignedPart(const std::string_view input, const size_t offset, const size_t length, - const std::string_view type_name) { - unsigned value = 0; - const char* begin = input.data() + offset; - const char* end = begin + length; - - if (const auto [ptr, ec] = std::from_chars(begin, end, value); ec != std::errc() || ptr != end) { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); - } - - return value; -} - -static std::chrono::sys_days ParseDatePoint(const std::string_view value, const std::string_view type_name) { - if (value.size() != 10 || value[4] != '-' || value[7] != '-') { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); - } - - const int year = static_cast(ParseUnsignedPart(value, 0, 4, type_name)); - const unsigned month = ParseUnsignedPart(value, 5, 2, type_name); - const unsigned day = ParseUnsignedPart(value, 8, 2, type_name); - - const std::chrono::year_month_day ymd{ - std::chrono::year{year}, - std::chrono::month{month}, - std::chrono::day{day}, - }; - - if (!ymd.ok()) { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); - } - - return std::chrono::sys_days{ymd}; -} - -static std::string FormatDateParts(const std::chrono::year_month_day ymd) { - std::ostringstream out; - out << std::setfill('0') << std::setw(4) << static_cast(ymd.year()) << '-' << std::setw(2) - << static_cast(ymd.month()) << '-' << std::setw(2) << static_cast(ymd.day()); - return out.str(); -} - -bool ParseBoolean(const std::string& value) { - const std::string lowered = ToLowerAscii(value); - if (lowered == "true" || lowered == "1") { - return true; - } - if (lowered == "false" || lowered == "0") { - return false; - } - throw Error::InvalidData("parsing", "invalid bool value"); -} - -int16_t ParseInt16(const std::string& value) { return ParseInteger(value, "int16"); } - -int32_t ParseInt32(const std::string& value) { return ParseInteger(value, "int32"); } - -int64_t ParseInt64(const std::string& value) { return ParseInteger(value, "int64"); } - -Int128 ParseInt128(const std::string& value) { - if (value.empty()) { - throw Error::InvalidData("parsing", "empty value for int128"); - } - - size_t pos = 0; - bool negative = false; - - if (value[pos] == '+' || value[pos] == '-') { - negative = value[pos] == '-'; - ++pos; - } - - if (pos == value.size()) { - throw Error::InvalidData("parsing", "invalid int128 value"); - } - - const UInt128 positive_limit = ~UInt128{0} >> 1; - const UInt128 negative_limit = positive_limit + 1; - const UInt128 limit = negative ? negative_limit : positive_limit; - - UInt128 magnitude = 0; - - for (; pos < value.size(); ++pos) { - const char ch = value[pos]; - if (ch < '0' || ch > '9') { - throw Error::InvalidData("parsing", "invalid int128 value"); - } - - const auto digit = static_cast(ch - '0'); - if (magnitude > (limit - digit) / 10) { - throw Error::InvalidData("parsing", "invalid int128 value"); - } - magnitude = magnitude * 10 + digit; - } - - if (!negative) { - return static_cast(magnitude); - } - if (magnitude == negative_limit) { - return -static_cast(positive_limit) - 1; - } - - return -static_cast(magnitude); -} - -int32_t ParseDate(const std::string& value) { - const auto day_point = ParseDatePoint(value, "date"); - const auto days_since_epoch = day_point.time_since_epoch().count(); - - if (days_since_epoch < std::numeric_limits::min() || - days_since_epoch > std::numeric_limits::max()) { - throw Error::Overflow("parsing", "date value exceeds supported range"); - } - - return days_since_epoch; -} - -int64_t ParseTimestamp(const std::string& value) { - if (value.size() < 19) { - throw Error::InvalidData("parsing", "invalid timestamp value"); - } - - const auto day_point = ParseDatePoint(std::string_view(value).substr(0, 10), "timestamp"); - - if (value[10] != ' ' && value[10] != 'T') { - throw Error::InvalidData("parsing", "invalid timestamp value"); - } - if (value[13] != ':' || value[16] != ':') { - throw Error::InvalidData("parsing", "invalid timestamp value"); - } - - const unsigned hour = ParseUnsignedPart(value, 11, 2, "timestamp"); - const unsigned minute = ParseUnsignedPart(value, 14, 2, "timestamp"); - const unsigned second = ParseUnsignedPart(value, 17, 2, "timestamp"); - - if (hour > 23 || minute > 59 || second > 59) { - throw Error::InvalidData("parsing", "invalid timestamp value"); - } - - int64_t microseconds = 0; - - if (value.size() > 19) { - if (value[19] != '.') { - throw Error::InvalidData("parsing", "invalid timestamp value"); - } - - const size_t digits = value.size() - 20; - if (digits == 0 || digits > 6) { - throw Error::InvalidData("parsing", "invalid timestamp value"); - } - - const auto fractional = ParseUnsignedPart(value, 20, digits, "timestamp"); - microseconds = static_cast(fractional); - for (size_t i = digits; i < 6; ++i) { - microseconds *= 10; - } - } - - const auto timestamp = day_point + std::chrono::hours{hour} + std::chrono::minutes{minute} + - std::chrono::seconds{second} + std::chrono::microseconds{microseconds}; - - return std::chrono::duration_cast(timestamp.time_since_epoch()).count(); -} - -char ParseCharacter(const std::string& value) { - if (value.size() != 1) { - throw Error::InvalidData("parsing", "invalid char value"); - } - return value[0]; -} - -std::string BooleanToString(const bool value) { return value ? "true" : "false"; } - -std::string Int128ToString(const Int128 value) { - if (value == 0) { - return "0"; - } - - const bool negative = value < 0; - UInt128 magnitude = 0; - - if (negative) { - magnitude = static_cast(-(value + 1)); - ++magnitude; - } else { - magnitude = static_cast(value); - } - - std::string out; - - while (magnitude > 0) { - out.push_back(static_cast('0' + magnitude % 10)); - magnitude /= 10; - } - if (negative) { - out.push_back('-'); - } - - std::reverse(out.begin(), out.end()); - - return out; -} - -std::string DateToString(const int32_t value) { - const auto day_point = std::chrono::sys_days{} + std::chrono::days{value}; - return FormatDateParts(std::chrono::year_month_day{day_point}); -} - -std::string TimestampToString(const int64_t value) { - const std::chrono::sys_time timestamp{std::chrono::microseconds{value}}; - const auto day_point = std::chrono::floor(timestamp); - const std::chrono::hh_mm_ss tod{timestamp - day_point}; - - std::ostringstream out; - out << FormatDateParts(std::chrono::year_month_day{day_point}) << ' ' << std::setfill('0') << std::setw(2) - << tod.hours().count() << ':' << std::setw(2) << tod.minutes().count() << ':' << std::setw(2) - << tod.seconds().count(); - - const int64_t fractional = tod.subseconds().count(); - - if (fractional != 0) { - std::string frac = std::to_string(fractional); - frac.insert(frac.begin(), 6 - static_cast(frac.size()), '0'); - while (!frac.empty() && frac.back() == '0') { - frac.pop_back(); - } - out << '.' << frac; - } - - return out.str(); -} - -ColumnType ParseColumnType(const std::string_view input) { - const std::string lowered = ToLowerAscii(input); - if (lowered == "bool") { - return ColumnType::Boolean; - } - if (lowered == "int16") { - return ColumnType::Int16; - } - if (lowered == "int32") { - return ColumnType::Int32; - } - if (lowered == "int64") { - return ColumnType::Int64; - } - if (lowered == "int128") { - return ColumnType::Int128; - } - if (lowered == "string") { - return ColumnType::String; - } - if (lowered == "date") { - return ColumnType::Date; - } - if (lowered == "timestamp") { - return ColumnType::Timestamp; - } - if (lowered == "char") { - return ColumnType::Character; - } - throw Error::Unsupported("parsing", "unknown column type"); -} - -std::string ColumnTypeToString(const ColumnType type) { - switch (type) { - case ColumnType::Boolean: - return "bool"; - case ColumnType::Int16: - return "int16"; - case ColumnType::Int32: - return "int32"; - case ColumnType::Int64: - return "int64"; - case ColumnType::Int128: - return "int128"; - case ColumnType::String: - return "string"; - case ColumnType::Date: - return "date"; - case ColumnType::Timestamp: - return "timestamp"; - case ColumnType::Character: - return "char"; - } - throw Error::Unsupported("parsing", "unknown column type"); -} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt deleted file mode 100644 index 988c0e4..0000000 --- a/tests/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -include(GoogleTest) - -function(add_component_test name) - add_executable(${name} ${ARGN}) - target_link_libraries(${name} PRIVATE columnar_engine_lib GTest::gtest_main) - gtest_discover_tests(${name}) -endfunction() - -add_component_test(test_fileio test_fileio.cpp) -add_component_test(test_columnar test_columnar.cpp) -add_component_test(test_csv test_csv.cpp) -add_component_test(test_batch test_batch.cpp) -add_component_test(test_columns test_columns.cpp) -add_component_test(test_stream test_stream.cpp) -add_component_test(test_tokenizer test_tokenizer.cpp) -add_component_test(test_executor test_executor.cpp) diff --git a/tests/test_batch.cpp b/tests/test_batch.cpp deleted file mode 100644 index 90ddefa..0000000 --- a/tests/test_batch.cpp +++ /dev/null @@ -1,245 +0,0 @@ -#include -#include -#include - -#include "columnar_batch_io.h" -#include "csv.h" -#include "csv_batch_io.h" -#include "error.h" -#include "gtest/gtest.h" -#include "temp_file.h" - -static_assert(std::is_copy_constructible_v); -static_assert(std::is_copy_assignable_v); -static_assert(std::is_move_constructible_v); -static_assert(std::is_move_assignable_v); - -TEST(batch, csv_reader_respects_max_rows) { - Schema schema; - schema.columns = { - {"id", ColumnType::Int64}, - {"name", ColumnType::String}, - }; - - const TempFile data_in("batch_csv"); - const std::vector> data_rows = { - {"1", "alpha"}, - {"2", "be,ta"}, - {"3", "gamma"}, - }; - - WriteRows(data_in.Path(), data_rows); - - BatchSizing sizing; - sizing.max_rows = 2; - CsvBatchReader reader(data_in.Path(), schema, sizing); - - auto first = reader.ReadNext(); - ASSERT_TRUE(first.has_value()); - EXPECT_EQ(first->RowsCount(), 2u); - EXPECT_EQ(first->ColumnAt(0).ValueAsString(0), "1"); - EXPECT_EQ(first->ColumnAt(1).ValueAsString(1), "be,ta"); - - auto second = reader.ReadNext(); - ASSERT_TRUE(second.has_value()); - EXPECT_EQ(second->RowsCount(), 1u); - EXPECT_EQ(second->ColumnAt(0).ValueAsString(0), "3"); - EXPECT_EQ(second->ColumnAt(1).ValueAsString(0), "gamma"); - - EXPECT_FALSE(reader.ReadNext().has_value()); -} - -TEST(batch, write_batch_csv_writes_single_batch) { - Schema schema; - schema.columns = { - {"id", ColumnType::Int64}, - {"name", ColumnType::String}, - }; - - Batch batch(schema); - batch.ColumnAt(0).AppendFromString("1"); - batch.ColumnAt(1).AppendFromString("alpha"); - batch.ColumnAt(0).AppendFromString("2"); - batch.ColumnAt(1).AppendFromString("be,ta"); - - const TempFile data_out("batch_write_csv"); - WriteBatchCsv(data_out.Path(), batch); - - EXPECT_EQ(ReadRows(data_out.Path()), (std::vector>{ - {"1", "alpha"}, - {"2", "be,ta"}, - })); -} - -TEST(batch, columnar_roundtrip) { - Schema schema; - schema.columns = { - {"id", ColumnType::Int64}, - {"name", ColumnType::String}, - }; - - const TempFile data_in("batch_roundtrip_csv"); - const TempFile columnar_file("batch_roundtrip_columnar"); - - const std::vector> data_rows = { - {"1", "alpha"}, - {"2", "be,ta"}, - {"3", "gamma"}, - }; - - WriteRows(data_in.Path(), data_rows); - - BatchSizing sizing; - sizing.max_rows = 2; - CsvBatchReader csv_reader(data_in.Path(), schema, sizing); - ColumnarBatchWriter columnar_writer(columnar_file.Path(), schema); - - while (auto batch = csv_reader.ReadNext()) { - columnar_writer.Write(*batch); - } - columnar_writer.Finalize(); - - ColumnarBatchReader columnar_reader(columnar_file.Path()); - EXPECT_EQ(columnar_reader.GetMetadata().row_groups.size(), 2u); - - std::vector> roundtrip_rows; - while (auto batch = columnar_reader.ReadNext()) { - AppendBatchRows(*batch, roundtrip_rows); - } - - EXPECT_EQ(roundtrip_rows, data_rows); -} - -TEST(batch, validate_detects_row_count_mismatch) { - Schema schema; - schema.columns = { - {"id", ColumnType::Int64}, - {"name", ColumnType::String}, - }; - - Batch batch(schema); - batch.ColumnAt(0).AppendFromString("1"); - batch.ColumnAt(0).AppendFromString("2"); - batch.ColumnAt(1).AppendFromString("alpha"); - - EXPECT_THROW(batch.Validate(), Error); -} - -TEST(batch, copy_is_deep) { - Schema schema; - schema.columns = { - {"id", ColumnType::Int64}, - {"name", ColumnType::String}, - }; - - Batch original(schema); - original.ColumnAt(0).AppendFromString("1"); - original.ColumnAt(1).AppendFromString("alpha"); - - Batch copied = original; - copied.ColumnAt(0).AppendFromString("2"); - copied.ColumnAt(1).AppendFromString("beta"); - - EXPECT_EQ(original.RowsCount(), 1u); - EXPECT_EQ(original.ColumnAt(0).ValueAsString(0), "1"); - EXPECT_EQ(original.ColumnAt(1).ValueAsString(0), "alpha"); - - EXPECT_EQ(copied.RowsCount(), 2u); - EXPECT_EQ(copied.ColumnAt(0).ValueAsString(1), "2"); - EXPECT_EQ(copied.ColumnAt(1).ValueAsString(1), "beta"); -} - -TEST(batch, csv_reader_respects_max_values) { - Schema schema; - schema.columns = { - {"id", ColumnType::Int64}, - {"name", ColumnType::String}, - }; - - const TempFile data_in("batch_csv_values"); - const std::vector> data_rows = { - {"1", "alpha"}, - {"2", "beta"}, - {"3", "gamma"}, - }; - - WriteRows(data_in.Path(), data_rows); - - BatchSizing sizing; - sizing.max_values = 3; - CsvBatchReader reader(data_in.Path(), schema, sizing); - - for (size_t i = 0; i < data_rows.size(); ++i) { - auto batch = reader.ReadNext(); - ASSERT_TRUE(batch.has_value()); - EXPECT_EQ(batch->RowsCount(), 1u); - EXPECT_EQ(batch->ColumnAt(0).ValueAsString(0), data_rows[i][0]); - EXPECT_EQ(batch->ColumnAt(1).ValueAsString(0), data_rows[i][1]); - } - - EXPECT_FALSE(reader.ReadNext().has_value()); -} - -TEST(batch, csv_reader_respects_max_bytes) { - Schema schema; - schema.columns = { - {"id", ColumnType::Int64}, - {"name", ColumnType::String}, - }; - - const TempFile data_in("batch_csv_bytes"); - const std::vector> data_rows = { - {"1", "aaaa"}, - {"2", "bbbb"}, - {"3", "cccc"}, - }; - - WriteRows(data_in.Path(), data_rows); - - BatchSizing sizing; - sizing.max_bytes = sizeof(int64_t) + 4u; - CsvBatchReader reader(data_in.Path(), schema, sizing); - - for (size_t i = 0; i < data_rows.size(); ++i) { - auto batch = reader.ReadNext(); - ASSERT_TRUE(batch.has_value()); - EXPECT_EQ(batch->RowsCount(), 1u); - EXPECT_EQ(batch->ColumnAt(0).ValueAsString(0), data_rows[i][0]); - EXPECT_EQ(batch->ColumnAt(1).ValueAsString(0), data_rows[i][1]); - } - - EXPECT_FALSE(reader.ReadNext().has_value()); -} - -TEST(batch, csv_reader_throws_on_column_mismatch) { - Schema schema; - schema.columns = { - {"id", ColumnType::Int64}, - {"name", ColumnType::String}, - }; - - const TempFile data_in("batch_csv_mismatch"); - WriteRows(data_in.Path(), {{"1"}}); - - BatchSizing sizing; - sizing.max_rows = 2; - CsvBatchReader reader(data_in.Path(), schema, sizing); - - EXPECT_THROW(reader.ReadNext(), Error); -} - -TEST(batch, columnar_writer_rejects_after_finalize) { - Schema schema; - schema.columns = { - {"id", ColumnType::Int64}, - {"name", ColumnType::String}, - }; - - const TempFile columnar_file("batch_finalize_columnar"); - ColumnarBatchWriter writer(columnar_file.Path(), schema); - writer.Finalize(); - - Batch batch(schema); - EXPECT_THROW(writer.Write(batch), Error); - EXPECT_THROW(writer.Finalize(), Error); -} diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp deleted file mode 100644 index 4132268..0000000 --- a/tests/test_columnar.cpp +++ /dev/null @@ -1,197 +0,0 @@ -#include -#include -#include - -#include "columnar_batch_io.h" -#include "csv_columnar.h" -#include "csv.h" -#include "error.h" -#include "fileio.h" -#include "gtest/gtest.h" -#include "parsing.h" -#include "schema.h" -#include "temp_file.h" - -TEST(columnar, csv_to_columnar_and_back) { - const TempFile schema_in("schema_in"); - const TempFile data_in("data_in"); - const TempFile columnar_file("columnar"); - const TempFile schema_out("schema_out"); - const TempFile data_out("data_out"); - - const std::vector> schema_rows = { - {"a", "int64"}, - {"b", "int64"}, - {"name", "string"}, - {"d", "int64"}, - }; - - const std::vector> data_rows = { - {"1", "2", "first", "4"}, - {"5", "1", "sec,ond", "2"}, - {"8", "17", "thi\"rd", "2"}, - {"9", "5", "line1\nline2", "10"}, - }; - - WriteRows(schema_in.Path(), schema_rows); - WriteRows(data_in.Path(), data_rows); - - ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 2); - ConvertColumnarToCsv(columnar_file.Path(), schema_out.Path(), data_out.Path()); - - const auto [columns] = ReadSchemaCsv(schema_out.Path()); - ASSERT_EQ(columns.size(), schema_rows.size()); - for (size_t i = 0; i < schema_rows.size(); ++i) { - EXPECT_EQ(columns[i].name, schema_rows[i][0]); - EXPECT_EQ(ColumnTypeToString(columns[i].type), schema_rows[i][1]); - } - - const auto data_roundtrip = ReadRows(data_out.Path()); - EXPECT_EQ(data_roundtrip, data_rows); -} - -TEST(columnar, metadata_offsets_and_sizes) { - const TempFile schema_in("schema_meta_in"); - const TempFile data_in("data_meta_in"); - const TempFile columnar_file("columnar_meta"); - - const std::vector> schema_rows = { - {"id", "int64"}, - {"name", "string"}, - }; - - const std::vector> data_rows = { - {"1", "a"}, - {"2", "bb"}, - {"3", "ccc"}, - }; - - WriteRows(schema_in.Path(), schema_rows); - WriteRows(data_in.Path(), data_rows); - - ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 2); - - ColumnarBatchReader reader(columnar_file.Path()); - const auto& [schema, row_groups] = reader.GetMetadata(); - ASSERT_EQ(schema.columns.size(), 2u); - EXPECT_EQ(schema.columns[0].name, "id"); - EXPECT_EQ(schema.columns[0].type, ColumnType::Int64); - EXPECT_EQ(schema.columns[1].name, "name"); - EXPECT_EQ(schema.columns[1].type, ColumnType::String); - - ASSERT_EQ(row_groups.size(), 2u); - EXPECT_EQ(row_groups[0].row_count, 2u); - EXPECT_EQ(row_groups[1].row_count, 1u); - - ASSERT_EQ(row_groups[0].columns.size(), 2u); - ASSERT_EQ(row_groups[1].columns.size(), 2u); - - auto string_chunk_size = [&data_rows](const size_t start, const size_t count) -> uint64_t { - uint64_t total = 0; - for (size_t i = 0; i < count; ++i) { - total += sizeof(uint32_t); - total += data_rows[start + i][1].size(); - } - return total; - }; - - EXPECT_EQ(row_groups[0].columns[0].size, sizeof(int64_t) * 2u); - EXPECT_EQ(row_groups[0].columns[1].size, string_chunk_size(0, 2)); - EXPECT_EQ(row_groups[1].columns[0].size, sizeof(int64_t) * 1u); - EXPECT_EQ(row_groups[1].columns[1].size, string_chunk_size(2, 1)); - - uint64_t expected_offset = 0; - for (const auto& [row_count, columns] : row_groups) { - for (const auto& [offset, size] : columns) { - EXPECT_EQ(offset, expected_offset); - expected_offset += size; - } - } - - const auto file_info = TryGetFileMetadata(columnar_file.Path()); - ASSERT_TRUE(file_info.has_value()); - EXPECT_TRUE(file_info->is_regular); - EXPECT_LT(expected_offset, file_info->size); -} - -TEST(columnar, invalid_int64_throws) { - const TempFile schema_in("schema_bad_int"); - const TempFile data_in("data_bad_int"); - const TempFile columnar_file("columnar_bad_int"); - - WriteRows(schema_in.Path(), {{"value", "int64"}}); - WriteRows(data_in.Path(), {{"not_a_number"}}); - - EXPECT_THROW(ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 4), Error); -} - -TEST(columnar, mixed_supported_types_roundtrip) { - const TempFile schema_in("schema_mixed_in"); - const TempFile data_in("data_mixed_in"); - const TempFile columnar_file("columnar_mixed"); - const TempFile schema_out("schema_mixed_out"); - const TempFile data_out("data_mixed_out"); - - const std::vector> schema_rows = { - {"flag", "bool"}, - {"small", "int16"}, - {"medium", "int32"}, - {"big", "int64"}, - {"huge", "int128"}, - {"name", "string"}, - {"birth_date", "date"}, - {"created_at", "timestamp"}, - {"grade", "char"}, - }; - - const std::vector> data_rows = { - {"true", "-7", "12345", "9999999999", "170141183460469231731687303715884105727", "alpha", "2024-02-29", - "2024-02-29 12:34:56.123456", "A"}, - {"false", "8", "-54321", "-42", "-170141183460469231731687303715884105728", "be,ta", "1970-01-01", - "1970-01-01 00:00:00", "Z"}, - }; - - WriteRows(schema_in.Path(), schema_rows); - WriteRows(data_in.Path(), data_rows); - - ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 1); - ConvertColumnarToCsv(columnar_file.Path(), schema_out.Path(), data_out.Path()); - - EXPECT_EQ(ReadRows(schema_out.Path()), schema_rows); - EXPECT_EQ(ReadRows(data_out.Path()), data_rows); -} - -TEST(columnar, infer_schema_from_csv_is_convert_compatible) { - const TempFile data_in("data_infer_in"); - const TempFile schema_out("schema_infer_out"); - const TempFile columnar_file("columnar_infer"); - const TempFile data_out("data_infer_out"); - - const std::vector> data_rows = { - {"true", "-7", "12345", "9999999999", "170141183460469231731687303715884105727", "alpha", "2024-02-29", - "2024-02-29 12:34:56.123456", "A"}, - {"false", "8", "-54321", "-42", "-170141183460469231731687303715884105728", "be,ta", "1970-01-01", - "1970-01-01 00:00:00", "Z"}, - }; - - WriteRows(data_in.Path(), data_rows); - WriteSchemaCsv(schema_out.Path(), InferSchemaCsv(data_in.Path())); - - EXPECT_EQ(ReadRows(schema_out.Path()), - (std::vector>{ - {"column_1", "bool"}, - {"column_2", "int16"}, - {"column_3", "int32"}, - {"column_4", "int64"}, - {"column_5", "int128"}, - {"column_6", "string"}, - {"column_7", "date"}, - {"column_8", "timestamp"}, - {"column_9", "char"}, - })); - - ConvertCsvToColumnar(schema_out.Path(), data_in.Path(), columnar_file.Path(), 1); - ConvertColumnarToCsv(columnar_file.Path(), schema_out.Path(), data_out.Path()); - - EXPECT_EQ(ReadRows(data_out.Path()), data_rows); -} diff --git a/tests/test_columns.cpp b/tests/test_columns.cpp deleted file mode 100644 index b83e667..0000000 --- a/tests/test_columns.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include -#include -#include - -#include "column.h" -#include "column_int64.h" -#include "column_string.h" -#include "error.h" -#include "gtest/gtest.h" -#include "parsing.h" - -static void ExpectColumnRoundtrip(const ColumnType type, const std::vector& values, - const std::vector& expected_values = {}) { - auto column = CreateColumn(type); - for (const auto& value : values) { - column->AppendFromString(value); - } - - std::stringstream buffer; - column->WriteTo(buffer); - const std::string bytes = buffer.str(); - - auto read_back = CreateColumn(type); - std::stringstream in(bytes); - read_back->ReadFrom(in, values.size(), bytes.size()); - - ASSERT_EQ(read_back->Size(), values.size()); - const auto& expected = expected_values.empty() ? values : expected_values; - ASSERT_EQ(expected.size(), values.size()); - for (size_t i = 0; i < expected.size(); ++i) { - EXPECT_EQ(read_back->ValueAsString(i), expected[i]); - } -} - -TEST(columns, int64_roundtrip) { - Int64Column values; - values.AppendFromString("10"); - values.AppendFromString("-5"); - - std::stringstream buffer; - values.WriteTo(buffer); - const std::string bytes = buffer.str(); - - Int64Column read_back; - std::stringstream in(bytes); - read_back.ReadFrom(in, 2, bytes.size()); - - EXPECT_EQ(read_back.Size(), 2u); - EXPECT_EQ(read_back.ValueAsString(0), "10"); - EXPECT_EQ(read_back.ValueAsString(1), "-5"); -} - -TEST(columns, string_roundtrip) { - StringColumn values; - values.AppendFromString("alpha"); - values.AppendFromString("line1\nline2"); - values.AppendFromString("be,ta"); - - std::stringstream buffer; - values.WriteTo(buffer); - const std::string bytes = buffer.str(); - - StringColumn read_back; - std::stringstream in(bytes); - read_back.ReadFrom(in, 3, bytes.size()); - - EXPECT_EQ(read_back.Size(), 3u); - EXPECT_EQ(read_back.ValueAsString(0), "alpha"); - EXPECT_EQ(read_back.ValueAsString(1), "line1\nline2"); - EXPECT_EQ(read_back.ValueAsString(2), "be,ta"); -} - -TEST(columns, int64_invalid_value_throws) { - Int64Column values; - EXPECT_THROW(values.AppendFromString("not_a_number"), Error); -} - -TEST(columns, string_out_of_range_throws) { - StringColumn values; - values.AppendFromString("alpha"); - EXPECT_THROW(values.ValueAsString(1), Error); -} - -TEST(columns, supported_scalar_types_roundtrip) { - ExpectColumnRoundtrip(ColumnType::Boolean, {"true", "false", "1"}, {"true", "false", "true"}); - ExpectColumnRoundtrip(ColumnType::Int16, {"-32768", "0", "32767"}); - ExpectColumnRoundtrip(ColumnType::Int32, {"-2147483648", "7", "2147483647"}); - ExpectColumnRoundtrip( - ColumnType::Int128, - {"-170141183460469231731687303715884105728", "0", "170141183460469231731687303715884105727"}); - ExpectColumnRoundtrip(ColumnType::Date, {"1970-01-01", "2024-02-29", "2030-12-31"}); - ExpectColumnRoundtrip( - ColumnType::Timestamp, - {"1970-01-01 00:00:00", "2024-02-29 12:34:56.123456", "2030-12-31T23:59:59.5"}, - {"1970-01-01 00:00:00", "2024-02-29 12:34:56.123456", "2030-12-31 23:59:59.5"}); - ExpectColumnRoundtrip(ColumnType::Character, {"A", ",", "\n"}); -} - -TEST(columns, scalar_invalid_values_throw) { - auto boolean = CreateColumn(ColumnType::Boolean); - EXPECT_THROW(boolean->AppendFromString("maybe"), Error); - - auto int16 = CreateColumn(ColumnType::Int16); - EXPECT_THROW(int16->AppendFromString("40000"), Error); - - auto int32 = CreateColumn(ColumnType::Int32); - EXPECT_THROW(int32->AppendFromString("2147483648"), Error); - - auto int128 = CreateColumn(ColumnType::Int128); - EXPECT_THROW(int128->AppendFromString("170141183460469231731687303715884105728"), Error); - - auto date = CreateColumn(ColumnType::Date); - EXPECT_THROW(date->AppendFromString("2024-02-30"), Error); - - auto timestamp = CreateColumn(ColumnType::Timestamp); - EXPECT_THROW(timestamp->AppendFromString("2024-02-29 24:00:00"), Error); - - auto character = CreateColumn(ColumnType::Character); - EXPECT_THROW(character->AppendFromString("ab"), Error); -} - -TEST(columns, type_names_roundtrip) { - const std::vector types = { - ColumnType::Boolean, - ColumnType::Int16, - ColumnType::Int32, - ColumnType::Int64, - ColumnType::Int128, - ColumnType::String, - ColumnType::Date, - ColumnType::Timestamp, - ColumnType::Character, - }; - - for (const auto type : types) { - EXPECT_EQ(ParseColumnType(ColumnTypeToString(type)), type); - } -} diff --git a/tests/test_csv.cpp b/tests/test_csv.cpp deleted file mode 100644 index 7e9c4c6..0000000 --- a/tests/test_csv.cpp +++ /dev/null @@ -1,65 +0,0 @@ -#include -#include -#include -#include - -#include "csv.h" -#include "gtest/gtest.h" - -static_assert(!std::is_copy_constructible_v); -static_assert(!std::is_copy_assignable_v); -static_assert(std::is_move_constructible_v); -static_assert(std::is_move_assignable_v); -static_assert(!std::is_copy_constructible_v); -static_assert(!std::is_copy_assignable_v); -static_assert(std::is_move_constructible_v); -static_assert(std::is_move_assignable_v); - -TEST(csv, read_write_roundtrip_with_quotes) { - std::stringstream buffer; - CsvWriter writer(buffer); - CsvReader reader(buffer); - - writer.WriteRow({"1", "2,3", "he\"llo", "line1\nline2"}); - writer.WriteRow({"", "plain"}); - - std::vector row; - ASSERT_TRUE(reader.ReadRow(row)); - EXPECT_EQ(row, (std::vector{"1", "2,3", "he\"llo", "line1\nline2"})); - - ASSERT_TRUE(reader.ReadRow(row)); - EXPECT_EQ(row, (std::vector{"", "plain"})); - - EXPECT_FALSE(reader.ReadRow(row)); -} - -TEST(csv, handles_crlf_newlines) { - std::istringstream input("a,b\r\nc,d\r\n"); - CsvReader reader(input); - std::vector row; - - ASSERT_TRUE(reader.ReadRow(row)); - EXPECT_EQ(row, (std::vector{"a", "b"})); - - ASSERT_TRUE(reader.ReadRow(row)); - EXPECT_EQ(row, (std::vector{"c", "d"})); - - EXPECT_FALSE(reader.ReadRow(row)); -} - -TEST(csv, moved_reader_and_writer_remain_usable) { - std::stringstream buffer; - CsvWriter writer(buffer); - CsvWriter moved_writer(std::move(writer)); - moved_writer.WriteRow({"1", "value"}); - moved_writer.Flush(); - - std::istringstream input(buffer.str()); - CsvReader reader(input); - CsvReader moved_reader(std::move(reader)); - - std::vector row; - ASSERT_TRUE(moved_reader.ReadRow(row)); - EXPECT_EQ(row, (std::vector{"1", "value"})); - EXPECT_FALSE(moved_reader.ReadRow(row)); -} diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp deleted file mode 100644 index e44a044..0000000 --- a/tests/test_executor.cpp +++ /dev/null @@ -1,114 +0,0 @@ -#include -#include - -#include "csv.h" -#include "csv_columnar.h" -#include "executor.h" -#include "executor_test_utils.h" -#include "gtest/gtest.h" -#include "temp_file.h" - -static Batch BuildHitsTable(const std::string_view query) { - const TempFile schema_file("executor_schema"); - const TempFile data_file("executor_data"); - const TempFile columnar_file("executor_columnar"); - - WriteRows(schema_file.Path(), { - {"AdvEngineID", "int64"}, - {"ResolutionWidth", "int32"}, - {"UserID", "int64"}, - {"SearchPhrase", "string"}, - {"EventDate", "date"}, - }); - - WriteRows(data_file.Path(), { - {"0", "100", "1", "alpha", "2024-01-05"}, - {"7", "200", "2", "beta", "2024-01-03"}, - {"3", "301", "2", "alpha", "2024-01-07"}, - {"7", "500", "5", "gamma", "2024-01-01"}, - {"1", "400", "11", "delta", "2024-01-09"}, - }); - - ConvertCsvToColumnar(schema_file.Path(), data_file.Path(), columnar_file.Path(), 2); - - Executor executor; - executor.RegisterTable("hits", columnar_file.Path()); - - auto result = executor.Execute(query); - if (!result.has_value()) { - throw result.error(); - } - return std::move(result.value()); -} - -TEST(executor, executes_basic_aggregate_queries) { - { - const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(*)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"5"})); - } - - { - const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(*)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4"})); - } - - { - const Batch batch = BuildHitsTable("SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), - (std::vector{"SUM(AdvEngineID)", "COUNT(*)", "AVG(ResolutionWidth)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"18", "5", "300.2"})); - } - - { - const Batch batch = BuildHitsTable("SELECT AVG(UserID) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AVG(UserID)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4.2"})); - } - - { - const Batch batch = BuildHitsTable("SELECT COUNT(DISTINCT UserID) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(DISTINCT UserID)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4"})); - } - - { - const Batch batch = BuildHitsTable("SELECT COUNT(DISTINCT SearchPhrase) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(DISTINCT SearchPhrase)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4"})); - } - - { - const Batch batch = BuildHitsTable("SELECT MIN(EventDate), MAX(EventDate) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"MIN(EventDate)", "MAX(EventDate)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"2024-01-01", "2024-01-09"})); - } - - { - const Batch batch = BuildHitsTable("select sum(AdvEngineID), count(*) from hits where SearchPhrase = 'alpha';"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"SUM(AdvEngineID)", "COUNT(*)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"3", "2"})); - } -} - -TEST(executor, rejects_unknown_aggregate_functions) { - const TempFile schema_file("executor_schema"); - const TempFile data_file("executor_data"); - const TempFile columnar_file("executor_columnar"); - - WriteRows(schema_file.Path(), { - {"Value", "int64"}, - }); - WriteRows(data_file.Path(), { - {"1"}, - }); - ConvertCsvToColumnar(schema_file.Path(), data_file.Path(), columnar_file.Path(), 1); - - Executor executor; - executor.RegisterTable("hits", columnar_file.Path()); - - const ExecuteExpected result = executor.Execute("SELECT median(Value) FROM hits;"); - ASSERT_FALSE(result.has_value()); - EXPECT_EQ(result.error().GetCode(), Error::Code::Unsupported); -} diff --git a/tests/test_fileio.cpp b/tests/test_fileio.cpp deleted file mode 100644 index c919fcd..0000000 --- a/tests/test_fileio.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include -#include -#include -#include - -#include "error.h" -#include "fileio.h" -#include "gtest/gtest.h" -#include "temp_file.h" - -TEST(fileio, write_and_read_roundtrip) { - const TempFile temp("fileio_roundtrip"); - const std::vector payload = {0, 1, 2, 3, 255}; - - WriteFileBytes(temp.Path(), payload); - const auto read_back = ReadFileBytes(temp.Path()); - - EXPECT_EQ(read_back, payload); -} - -TEST(fileio, append_bytes) { - const TempFile temp("fileio_append"); - const std::vector initial = {10, 11}; - const std::vector extra = {12, 13}; - - WriteFileBytes(temp.Path(), initial); - std::ofstream out = OpenOutputFile(temp.Path(), FileOpenMode::Append); - out.write(reinterpret_cast(extra.data()), extra.size()); - ASSERT_TRUE(out.good()); - out.flush(); - ASSERT_TRUE(out.good()); - - const auto read_back = ReadFileBytes(temp.Path()); - const std::vector expected = {10, 11, 12, 13}; - EXPECT_EQ(read_back, expected); -} - -TEST(fileio, metadata_for_file) { - const TempFile temp("fileio_meta"); - const std::vector payload = {42, 43, 44}; - - WriteFileBytes(temp.Path(), payload); - - const auto metadata = TryGetFileMetadata(temp.Path()); - ASSERT_TRUE(metadata.has_value()); - EXPECT_TRUE(metadata->is_regular); - EXPECT_FALSE(metadata->is_directory); - EXPECT_EQ(metadata->size, payload.size()); -} - -TEST(fileio, metadata_for_missing_file) { - const auto missing = UniqueTempPath("fileio_missing"); - const auto metadata = TryGetFileMetadata(missing); - - EXPECT_FALSE(metadata.has_value()); -} - -TEST(fileio, read_missing_file_throws) { - const auto missing = UniqueTempPath("fileio_missing_read"); - - EXPECT_THROW(ReadFileBytes(missing), Error); -} - -TEST(fileio, read_text_roundtrip) { - const TempFile temp("fileio_text_roundtrip"); - const std::string payload = "SELECT COUNT(*)\nFROM hits\nWHERE UserID = 42;\n"; - - std::ofstream out = OpenOutputFile(temp.Path()); - out << payload; - ASSERT_TRUE(out.good()); - out.flush(); - ASSERT_TRUE(out.good()); - - EXPECT_EQ(ReadTextFile(temp.Path()), payload); -} - -TEST(fileio, file_exists_checks) { - const TempFile temp("fileio_exists"); - EXPECT_FALSE(FileExists(temp.Path())); - - WriteFileBytes(temp.Path(), std::vector{1}); - - EXPECT_TRUE(FileExists(temp.Path())); -} - -TEST(fileio, ensure_parent_directory_creates_missing_directories) { - const auto file_path = UniqueTempPath("fileio_nested") / "a" / "b" / "query.sql"; - const auto parent = file_path.parent_path(); - - ASSERT_FALSE(std::filesystem::exists(parent)); - EnsureParentDirectory(file_path); - EXPECT_TRUE(std::filesystem::exists(parent)); - EXPECT_TRUE(std::filesystem::is_directory(parent)); -} - -TEST(fileio, stream_roundtrip) { - const TempFile temp("fileio_stream_roundtrip"); - const std::vector payload = {7, 8, 9, 10, 11}; - - std::ofstream out = OpenOutputFile(temp.Path()); - out.write(reinterpret_cast(payload.data()), payload.size()); - ASSERT_TRUE(out.good()); - out.flush(); - ASSERT_TRUE(out.good()); - - std::ifstream in = OpenInputFile(temp.Path()); - std::vector read_back(payload.size()); - in.read(reinterpret_cast(read_back.data()), read_back.size()); - ASSERT_EQ(static_cast(in.gcount()), read_back.size()); - EXPECT_EQ(read_back, payload); -} - -TEST(fileio, stream_append) { - const TempFile temp("fileio_stream_append"); - const std::vector initial = {1, 2}; - const std::vector extra = {3, 4, 5}; - - WriteFileBytes(temp.Path(), initial); - - std::ofstream out = OpenOutputFile(temp.Path(), FileOpenMode::Append); - out.write(reinterpret_cast(extra.data()), extra.size()); - ASSERT_TRUE(out.good()); - out.flush(); - ASSERT_TRUE(out.good()); - - const auto read_back = ReadFileBytes(temp.Path()); - const std::vector expected = {1, 2, 3, 4, 5}; - EXPECT_EQ(read_back, expected); -} - -TEST(fileio, stream_seek_and_tell) { - const TempFile temp("fileio_stream_seek"); - const std::vector payload = {100, 101, 102, 103}; - - WriteFileBytes(temp.Path(), payload); - - std::ifstream in = OpenInputFile(temp.Path()); - EXPECT_EQ(in.tellg(), std::streampos(0)); - in.seekg(std::streampos(2)); - ASSERT_TRUE(in.good()); - EXPECT_EQ(in.tellg(), std::streampos(2)); - - uint8_t value = 0; - in.read(reinterpret_cast(&value), 1); - ASSERT_EQ(in.gcount(), 1); - EXPECT_EQ(value, payload[2]); -} - -TEST(fileio, stream_open_missing_throws) { - const auto missing = UniqueTempPath("fileio_stream_missing"); - - EXPECT_THROW({ std::ifstream in = OpenInputFile(missing); }, Error); -} diff --git a/tests/test_stream.cpp b/tests/test_stream.cpp deleted file mode 100644 index a3617c8..0000000 --- a/tests/test_stream.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include -#include -#include - -#include "gtest/gtest.h" -#include "stream.h" - -TEST(stream, roundtrip_signed_integrals) { - std::stringstream buffer; - - WriteStream(buffer, std::numeric_limits::min()); - WriteStream(buffer, -1); - WriteStream(buffer, 0); - WriteStream(buffer, std::numeric_limits::max()); - - EXPECT_EQ(ReadStream(buffer), std::numeric_limits::min()); - EXPECT_EQ(ReadStream(buffer), -1); - EXPECT_EQ(ReadStream(buffer), 0); - EXPECT_EQ(ReadStream(buffer), std::numeric_limits::max()); -} diff --git a/tests/test_tokenizer.cpp b/tests/test_tokenizer.cpp deleted file mode 100644 index ed9f019..0000000 --- a/tests/test_tokenizer.cpp +++ /dev/null @@ -1,117 +0,0 @@ -#include -#include -#include - -#include "gtest/gtest.h" -#include "tokenizer.h" - -static std::vector CollectTypes(const std::vector& tokens) { - std::vector types; - types.reserve(tokens.size()); - - for (const TokenPtr& token : tokens) { - types.push_back(token->GetType()); - } - - return types; -} - -TEST(sql_tokenizer, tokenizes_select_query) { - const auto result = - TokenizeSql("SELECT DISTINCT name, COUNT(*), LENGTH(city) FROM sales GROUP BY name ORDER BY name LIMIT 10;"); - - ASSERT_TRUE(result.has_value()); - const auto& tokens = result.value(); - - EXPECT_EQ( - CollectTypes(tokens), - (std::vector{ - Tokens::Select, Tokens::Distinct, Tokens::NameToken, Tokens::Comma, Tokens::Count, - Tokens::OpenBracket, Tokens::Asterisk, Tokens::CloseBracket, Tokens::Comma, Tokens::Length, - Tokens::OpenBracket, Tokens::NameToken, Tokens::CloseBracket, Tokens::From, Tokens::NameToken, - Tokens::Group, Tokens::By, Tokens::NameToken, Tokens::Order, Tokens::By, - Tokens::NameToken, Tokens::Limit, Tokens::NumericLiteral, Tokens::Semicolon, Tokens::EOI, - })); - - ASSERT_EQ(tokens.size(), 25U); - EXPECT_NE(std::dynamic_pointer_cast(tokens[0]), nullptr); - EXPECT_NE(std::dynamic_pointer_cast(tokens[1]), nullptr); - EXPECT_NE(std::dynamic_pointer_cast(tokens[2]), nullptr); - EXPECT_EQ(tokens[2]->GetText(), "name"); - EXPECT_EQ(tokens[14]->GetText(), "sales"); - EXPECT_EQ(tokens[22]->GetText(), "10"); - EXPECT_EQ(tokens[24]->GetOffset(), std::string_view("SELECT DISTINCT name, COUNT(*), LENGTH(city) FROM sales " - "GROUP BY name ORDER BY name LIMIT 10;") - .size()); -} - -TEST(sql_tokenizer, tokenizes_create_with_literals_and_case_insensitive_keywords) { - const auto result = TokenizeSql("create report as SeLeCt 'abc''def', total_sum - 10.5 FROM source_table"); - - ASSERT_TRUE(result.has_value()); - const auto& tokens = result.value(); - - EXPECT_EQ(CollectTypes(tokens), (std::vector{ - Tokens::Create, - Tokens::NameToken, - Tokens::As, - Tokens::Select, - Tokens::StringLiteral, - Tokens::Comma, - Tokens::NameToken, - Tokens::Minus, - Tokens::NumericLiteral, - Tokens::From, - Tokens::NameToken, - Tokens::EOI, - })); - - ASSERT_EQ(tokens.size(), 12U); - EXPECT_NE(std::dynamic_pointer_cast(tokens[0]), nullptr); - EXPECT_NE(std::dynamic_pointer_cast(tokens[4]), nullptr); - EXPECT_EQ(tokens[1]->GetText(), "report"); - EXPECT_EQ(tokens[4]->GetText(), "'abc''def'"); - EXPECT_EQ(tokens[8]->GetText(), "10.5"); - EXPECT_EQ(tokens[10]->GetText(), "source_table"); -} - -TEST(sql_tokenizer, tokenizer_returns_tokens_one_by_one) { - Tokenizer tokenizer("WHERE value >= 10 AND value <> 20 AND value != 30"); - - std::vector types; - while (true) { - auto token = tokenizer.GetNext(); - ASSERT_TRUE(token.has_value()); - types.push_back(token.value()->GetType()); - - if (token.value()->GetType() == Tokens::EOI) { - break; - } - } - - EXPECT_EQ(types, (std::vector{ - Tokens::Where, - Tokens::NameToken, - Tokens::GreaterOrEqual, - Tokens::NumericLiteral, - Tokens::And, - Tokens::NameToken, - Tokens::NotEqual, - Tokens::NumericLiteral, - Tokens::And, - Tokens::NameToken, - Tokens::NotEqual, - Tokens::NumericLiteral, - Tokens::EOI, - })); -} - -TEST(sql_tokenizer, rejects_invalid_input) { - const auto unterminated = TokenizeSql("SELECT 'oops"); - ASSERT_FALSE(unterminated.has_value()); - EXPECT_EQ(unterminated.error().GetCode(), Error::Code::InvalidData); - - const auto invalid_character = TokenizeSql("SELECT @oops"); - ASSERT_FALSE(invalid_character.has_value()); - EXPECT_EQ(invalid_character.error().GetCode(), Error::Code::InvalidData); -} From af2524e241c0bc5f23e20ac0c0474916ffcdcb4a Mon Sep 17 00:00:00 2001 From: ik4rd Date: Tue, 12 May 2026 23:04:44 +0300 Subject: [PATCH 15/29] fix: exit 0 breaks German --- script/run_query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run_query.sh b/script/run_query.sh index 2996f56..6fe4cdf 100755 --- a/script/run_query.sh +++ b/script/run_query.sh @@ -29,7 +29,7 @@ columnar_path=${columnar_path} query_file=${query_file} message=Add SQL text to benchmarks/queries/query_${query_num}.sql EOF - exit 0 + return 0 2>/dev/null fi if [[ ! -x "${binary}" ]]; then From b6a7e98bc57b1b7c213ce683d529ea33ef06b176 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Tue, 12 May 2026 23:08:00 +0300 Subject: [PATCH 16/29] fix: exit 0 again... --- script/setup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/setup.sh b/script/setup.sh index 3e6f8f2..087e96a 100755 --- a/script/setup.sh +++ b/script/setup.sh @@ -11,7 +11,7 @@ packages=( if ! command -v apt-get >/dev/null 2>&1; then echo "apt-get not found; assuming build dependencies are already installed." - exit 0 + return 0 2>/dev/null fi missing=() @@ -23,14 +23,14 @@ done if ((${#missing[@]} == 0)); then echo "Build dependencies already installed." - exit 0 + return 0 2>/dev/null fi if [[ "$(id -u)" -eq 0 ]]; then apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${missing[@]}" rm -rf /var/lib/apt/lists/* - exit 0 + return 0 2>/dev/null fi sudo apt-get update From 4b715f8908dc0c4322d259f25e933bd5713e0d0d Mon Sep 17 00:00:00 2001 From: ik4rd Date: Tue, 12 May 2026 23:28:02 +0300 Subject: [PATCH 17/29] fix: scheme forgot --- .gitignore | 1 - benchmarks/CMakeLists.txt | 2 +- benchmarks/benchmark_csv_to_columnar.cpp | 4 +- benchmarks/scheme.csv | 105 +++++++++++++++++++++++ 4 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 benchmarks/scheme.csv diff --git a/.gitignore b/.gitignore index 3eef705..14918bf 100644 --- a/.gitignore +++ b/.gitignore @@ -104,7 +104,6 @@ examples/out_columnar # benchmarks benchmarks/hits_sample.csv benchmarks/hits_sample_new.csv -benchmarks/schema_sample.csv benchmarks/schema_sample_new.csv benchmarks/hits_sample.columnar benchmarks/benchmark_csv_to_columnar diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 5559b8d..7fe3cd2 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -22,7 +22,7 @@ target_link_libraries(benchmark_queries target_compile_definitions(benchmark_csv_to_columnar PRIVATE COLUMNAR_BENCHMARK_DEFAULT_DATA="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample.csv" - COLUMNAR_BENCHMARK_DEFAULT_SCHEMA="${PROJECT_SOURCE_DIR}/benchmarks/schema_sample.csv" + COLUMNAR_BENCHMARK_DEFAULT_SCHEMA="${PROJECT_SOURCE_DIR}/benchmarks/scheme.csv" COLUMNAR_BENCHMARK_DEFAULT_OUTPUT="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample.columnar" COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_DATA="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample_new.csv" COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA="${PROJECT_SOURCE_DIR}/benchmarks/schema_sample_new.csv" diff --git a/benchmarks/benchmark_csv_to_columnar.cpp b/benchmarks/benchmark_csv_to_columnar.cpp index db124f5..42810e3 100644 --- a/benchmarks/benchmark_csv_to_columnar.cpp +++ b/benchmarks/benchmark_csv_to_columnar.cpp @@ -4,8 +4,8 @@ #include #include -#include "io/csv.h" #include "convert/csv_columnar.h" +#include "io/csv.h" #include "support/error.h" #ifndef COLUMNAR_BENCHMARK_DEFAULT_DATA @@ -13,7 +13,7 @@ #endif #ifndef COLUMNAR_BENCHMARK_DEFAULT_SCHEMA -#define COLUMNAR_BENCHMARK_DEFAULT_SCHEMA "benchmarks/schema_sample.csv" +#define COLUMNAR_BENCHMARK_DEFAULT_SCHEMA "benchmarks/scheme.csv" #endif #ifndef COLUMNAR_BENCHMARK_DEFAULT_OUTPUT diff --git a/benchmarks/scheme.csv b/benchmarks/scheme.csv new file mode 100644 index 0000000..881bb54 --- /dev/null +++ b/benchmarks/scheme.csv @@ -0,0 +1,105 @@ +WatchID,int64 +JavaEnable,int16 +Title,string +GoodEvent,int16 +EventTime,TIMESTAMP +EventDate,DATE +CounterID,int32 +ClientIP,int32 +RegionID,int32 +UserID,int64 +CounterClass,int16 +OS,int16 +UserAgent,int16 +URL,string +Referer,string +IsRefresh,int16 +RefererCategoryID,int16 +RefererRegionID,int32 +URLCategoryID,int16 +URLRegionID,int32 +ResolutionWidth,int16 +ResolutionHeight,int16 +ResolutionDepth,int16 +FlashMajor,int16 +FlashMinor,int16 +FlashMinor2,string +NetMajor,int16 +NetMinor,int16 +UserAgentMajor,int16 +UserAgentMinor,string +CookieEnable,int16 +JavascriptEnable,int16 +IsMobile,int16 +MobilePhone,int16 +MobilePhoneModel,string +Params,string +IPNetworkID,int32 +TraficSourceID,int16 +SearchEngineID,int16 +SearchPhrase,string +AdvEngineID,int16 +IsArtifical,int16 +WindowClientWidth,int16 +WindowClientHeight,int16 +ClientTimeZone,int16 +ClientEventTime,TIMESTAMP +SilverlightVersion1,int16 +SilverlightVersion2,int16 +SilverlightVersion3,int32 +SilverlightVersion4,int16 +PageCharset,string +CodeVersion,int32 +IsLink,int16 +IsDownload,int16 +IsNotBounce,int16 +FUniqID,int64 +OriginalURL,string +HID,int32 +IsOldCounter,int16 +IsEvent,int16 +IsParameter,int16 +DontCountHits,int16 +WithHash,int16 +HitColor,string +LocalEventTime,TIMESTAMP +Age,int16 +Sex,int16 +Income,int16 +Interests,int16 +Robotness,int16 +RemoteIP,int32 +WindowName,int32 +OpenerName,int32 +HistoryLength,int16 +BrowserLanguage,string +BrowserCountry,string +SocialNetwork,string +SocialAction,string +HTTPError,int16 +SendTiming,int32 +DNSTiming,int32 +ConnectTiming,int32 +ResponseStartTiming,int32 +ResponseEndTiming,int32 +FetchTiming,int32 +SocialSourceNetworkID,int16 +SocialSourcePage,string +ParamPrice,int64 +ParamOrderID,string +ParamCurrency,string +ParamCurrencyID,int16 +OpenstatServiceName,string +OpenstatCampaignID,string +OpenstatAdID,string +OpenstatSourceID,string +UTMSource,string +UTMMedium,string +UTMCampaign,string +UTMContent,string +UTMTerm,string +FromTag,string +HasGCLID,int16 +RefererHash,int64 +URLHash,int64 +CLID,int32 From 2db82cda47f965c6fe9526dabddf74079cb2ab53 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Tue, 12 May 2026 23:45:09 +0300 Subject: [PATCH 18/29] fix: watafa --- script/convert.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/script/convert.sh b/script/convert.sh index 9cd90cc..d6bc604 100755 --- a/script/convert.sh +++ b/script/convert.sh @@ -8,13 +8,15 @@ fi input_csv="$1" columnar_path="$2" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(cd "${script_dir}/.." && pwd)" build_dir="${BUILD_DIR:-build}" row_group_size="${ROW_GROUP_SIZE:-50000}" -schema_path="${COLUMNAR_SCHEMA:-${columnar_path}.schema.csv}" -binary="./${build_dir}/columnar_engine" +schema_path="${COLUMNAR_SCHEMA:-${repo_dir}/benchmarks/scheme.csv}" +binary="${repo_dir}/${build_dir}/columnar_engine" if [[ ! -x "${binary}" ]]; then - binary="./${build_dir}/src/columnar_engine" + binary="${repo_dir}/${build_dir}/src/columnar_engine" fi if [[ ! -x "${binary}" ]]; then @@ -22,12 +24,12 @@ if [[ ! -x "${binary}" ]]; then exit 1 fi -mkdir -p "$(dirname "${columnar_path}")" -mkdir -p "$(dirname "${schema_path}")" +if [[ ! -f "${schema_path}" ]]; then + echo "schema file not found: ${schema_path}" >&2 + exit 1 +fi -"${binary}" infer-schema \ - --input "${input_csv}" \ - --output "${schema_path}" +mkdir -p "$(dirname "${columnar_path}")" "${binary}" convert \ --schema "${schema_path}" \ From 225225fae7f8a9232304b3ce10c509e535fbeecc Mon Sep 17 00:00:00 2001 From: ik4rd Date: Fri, 15 May 2026 13:28:14 +0300 Subject: [PATCH 19/29] review: executor module done --- include/executor/aggregate_function.h | 1 - include/executor/aggregate_state.h | 24 ++++ include/executor/executor.h | 143 ----------------------- include/executor/operator.h | 22 ++++ include/executor/operator_utils.h | 2 +- include/executor/planner_utils.h | 8 +- include/executor/query.h | 88 +++++++++++++++ include/executor/query_parser.h | 7 ++ include/executor/query_plan.h | 69 +++++++++++ include/executor/query_planner.h | 10 ++ src/executor/aggregate_states.cpp | 21 +--- src/executor/executor.cpp | 10 +- src/executor/operator_utils.cpp | 1 + src/executor/operators.cpp | 27 +++-- src/executor/planner_utils.cpp | 19 ++-- src/executor/query_parser.cpp | 142 +++++++++++++++-------- src/executor/query_planner.cpp | 157 +++++++++++++++++++------- tests/test_executor.cpp | 12 +- 18 files changed, 488 insertions(+), 275 deletions(-) create mode 100644 include/executor/aggregate_state.h create mode 100644 include/executor/operator.h create mode 100644 include/executor/query.h create mode 100644 include/executor/query_parser.h create mode 100644 include/executor/query_plan.h create mode 100644 include/executor/query_planner.h diff --git a/include/executor/aggregate_function.h b/include/executor/aggregate_function.h index 69435ef..1d18168 100644 --- a/include/executor/aggregate_function.h +++ b/include/executor/aggregate_function.h @@ -40,4 +40,3 @@ struct AggRegistrar { const AggFuncDefinition& ResolveAggFunc(std::string_view name); bool AggSupportsInputType(const AggFuncDefinition& definition, ColumnType type); -std::string FormatAgg(const AggFuncDefinition& definition, std::string_view column_name, bool distinct, bool star); diff --git a/include/executor/aggregate_state.h b/include/executor/aggregate_state.h new file mode 100644 index 0000000..2ccb3fe --- /dev/null +++ b/include/executor/aggregate_state.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include +#include + +#include "executor/query_plan.h" + +class AggState { + public: + AggState() = default; + AggState(const AggState&) = delete; + AggState(AggState&&) noexcept = default; + AggState& operator=(const AggState&) = delete; + AggState& operator=(AggState&&) noexcept = default; + virtual ~AggState() = default; + + public: + virtual void Consume(std::optional value) = 0; + virtual std::string Finalize() const = 0; +}; + +std::unique_ptr CreateAggState(const PlannedAgg& aggregate); diff --git a/include/executor/executor.h b/include/executor/executor.h index 53da8b7..5ce4591 100644 --- a/include/executor/executor.h +++ b/include/executor/executor.h @@ -1,34 +1,16 @@ #pragma once #include -#include -#include #include #include #include -#include -#include "executor/aggregate_function.h" #include "model/batch.h" #include "support/error.h" #include "tl/expected.hpp" -#include "sql_parser/tokenizer.h" using ExecuteExpected = tl::expected; -class Operator { - public: - Operator() = default; - Operator(const Operator&) = delete; - Operator(Operator&&) noexcept = default; - Operator& operator=(const Operator&) = delete; - Operator& operator=(Operator&&) noexcept = default; - virtual ~Operator() = default; - - public: - virtual std::optional Next() = 0; -}; - class Executor { public: Executor() = default; @@ -40,128 +22,3 @@ class Executor { private: std::unordered_map tables_; }; - -enum class ComparisonKind { - Equal, - NotEqual, - Less, - LessOrEqual, - Greater, - GreaterOrEqual, -}; - -struct ParsedLiteral { - std::string text; - Tokens token_type = Tokens::StringLiteral; -}; - -struct ColumnRef { - std::string qualifier; - std::string name; -}; - -struct FilterSpec { - ColumnRef column; - ComparisonKind comparison = ComparisonKind::Equal; - ParsedLiteral literal; -}; - -struct AggSpec { - const AggFuncDefinition* function = nullptr; - bool distinct = false; - bool star = false; - ColumnRef column; - std::string output_name; -}; - -enum class SelectItemKind { - GroupKey, - Aggregate, -}; - -struct SelectItemSpec { - SelectItemKind kind = SelectItemKind::GroupKey; - ColumnRef column; - AggSpec aggregate; - std::string output_name; -}; - -struct OrderBySpec { - SelectItemSpec item; - bool descending = false; -}; - -struct ParsedQuery { - std::string table_name; - std::string table_alias; - std::vector select_items; - std::vector group_by; - std::optional filter; - std::vector order_by; - std::optional limit; -}; - -struct PlannedAgg { - const AggFuncDefinition* function = nullptr; - bool distinct = false; - bool star = false; - std::optional column_index; - ColumnType input_type = ColumnType::String; - std::string output_name; -}; - -struct PlannedGroupKey { - size_t column_index = 0; - ColumnType column_type = ColumnType::String; - std::string output_name; -}; - -struct PlannedSelectItem { - SelectItemKind kind = SelectItemKind::GroupKey; - size_t index = 0; - ColumnType output_type = ColumnType::String; - std::string output_name; -}; - -struct PlannedOrderBy { - size_t result_column_index = 0; - bool descending = false; - ColumnType value_type = ColumnType::String; -}; - -struct PlannedFilter { - size_t column_index = 0; - ColumnType column_type = ColumnType::String; - ComparisonKind comparison = ComparisonKind::Equal; - std::string literal_text; -}; - -struct PlannedQuery { - std::filesystem::path table_path; - std::vector projection_indexes; - std::optional filter; - std::vector group_keys; - std::vector aggregates; - std::vector select_items; - std::vector order_by; - std::optional limit; -}; - -class AggState { - public: - AggState() = default; - AggState(const AggState&) = delete; - AggState(AggState&&) noexcept = default; - AggState& operator=(const AggState&) = delete; - AggState& operator=(AggState&&) noexcept = default; - virtual ~AggState() = default; - - public: - virtual void Consume(std::optional value) = 0; - virtual std::string Finalize() const = 0; -}; - -ParsedQuery ParseQuery(std::string_view query); -PlannedQuery PlanQuery(const ParsedQuery& parsed, const std::unordered_map& tables); -std::unique_ptr CreateAggState(const PlannedAgg& aggregate); -std::unique_ptr BuildPlan(const PlannedQuery& planned); diff --git a/include/executor/operator.h b/include/executor/operator.h new file mode 100644 index 0000000..60ae1c3 --- /dev/null +++ b/include/executor/operator.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include "executor/query_plan.h" +#include "model/batch.h" + +class Operator { + public: + Operator() = default; + Operator(const Operator&) = delete; + Operator(Operator&&) noexcept = default; + Operator& operator=(const Operator&) = delete; + Operator& operator=(Operator&&) noexcept = default; + virtual ~Operator() = default; + + public: + virtual std::optional Next() = 0; +}; + +std::unique_ptr BuildPlan(const PlannedQuery& planned); diff --git a/include/executor/operator_utils.h b/include/executor/operator_utils.h index 7a26006..44f9d18 100644 --- a/include/executor/operator_utils.h +++ b/include/executor/operator_utils.h @@ -2,7 +2,7 @@ #include -#include "executor/executor.h" +#include "executor/query_plan.h" int CompareValues(ColumnType type, std::string_view lhs, std::string_view rhs); bool MatchesComparison(ColumnType type, std::string_view lhs, std::string_view rhs, ComparisonKind kind); diff --git a/include/executor/planner_utils.h b/include/executor/planner_utils.h index f5b4b90..70d19af 100644 --- a/include/executor/planner_utils.h +++ b/include/executor/planner_utils.h @@ -3,11 +3,13 @@ #include #include -#include "executor/executor.h" +#include "executor/query.h" +#include "model/schema.h" size_t FindColumnIndex(const Schema& schema, std::string_view column_name); -void ValidateColumnQualifier(const ParsedQuery& parsed, const ColumnRef& column); -std::string NormalizeLiteral(const ParsedLiteral& literal, ColumnType type); +void ValidateColumnQualifier(const Query& query, const ColumnRef& column); +std::string NormalizeLiteral(const QueryLiteral& literal, ColumnType type); + bool SameColumnName(std::string_view lhs, std::string_view rhs); bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs); bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs); diff --git a/include/executor/query.h b/include/executor/query.h new file mode 100644 index 0000000..2cd4b88 --- /dev/null +++ b/include/executor/query.h @@ -0,0 +1,88 @@ +#pragma once + +#include +#include +#include +#include + +enum class ComparisonKind { + Equal, + NotEqual, + Less, + LessOrEqual, + Greater, + GreaterOrEqual, +}; + +enum class LiteralKind { + Numeric, + String, + Identifier, +}; + +struct QueryLiteral { + std::string text; + LiteralKind kind = LiteralKind::String; +}; + +struct ColumnRef { + std::string qualifier; + std::string name; +}; + +enum class QueryValueKind { + Column, + Literal, +}; + +struct QueryValue { + QueryValueKind kind = QueryValueKind::Literal; + ColumnRef column; + QueryLiteral literal; +}; + +struct FilterSpec { + QueryValue left; + ComparisonKind comparison = ComparisonKind::Equal; + QueryValue right; +}; + +enum class AggArgumentKind { + Column, + Star, +}; + +struct AggSpec { + std::string function_name; + bool distinct = false; + AggArgumentKind argument_kind = AggArgumentKind::Column; + ColumnRef column; + std::string output_name; +}; + +enum class SelectItemKind { + GroupKey, + Aggregate, +}; + +struct SelectItemSpec { + SelectItemKind kind = SelectItemKind::GroupKey; + ColumnRef column; + AggSpec aggregate; + std::string output_name; +}; + +struct OrderBySpec { + SelectItemSpec item; + bool descending = false; +}; + +struct Query { + std::string table_name; + std::string table_alias; + std::vector select_items; + std::vector group_by; + std::optional filter; + std::vector order_by; + std::optional limit; +}; diff --git a/include/executor/query_parser.h b/include/executor/query_parser.h new file mode 100644 index 0000000..5b823d0 --- /dev/null +++ b/include/executor/query_parser.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +#include "executor/query.h" + +Query ParseQuery(std::string_view query); diff --git a/include/executor/query_plan.h b/include/executor/query_plan.h new file mode 100644 index 0000000..7cfaba7 --- /dev/null +++ b/include/executor/query_plan.h @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "executor/aggregate_function.h" +#include "executor/query.h" +#include "model/schema.h" + +struct PlannedAgg { + const AggFuncDefinition* function = nullptr; + bool distinct = false; + AggArgumentKind argument_kind = AggArgumentKind::Column; + size_t column_index = 0; + ColumnType input_type = ColumnType::String; + std::string output_name; +}; + +struct PlannedGroupKey { + size_t column_index = 0; + ColumnType column_type = ColumnType::String; + std::string output_name; +}; + +struct PlannedSelectItem { + SelectItemKind kind = SelectItemKind::GroupKey; + size_t index = 0; + ColumnType output_type = ColumnType::String; + std::string output_name; +}; + +struct PlannedOrderBy { + size_t result_column_index = 0; + bool descending = false; + ColumnType value_type = ColumnType::String; +}; + +enum class PlannedValueKind { + Column, + Literal, +}; + +struct PlannedValue { + PlannedValueKind kind = PlannedValueKind::Literal; + ColumnType column_type = ColumnType::String; + size_t column_index = 0; + std::string literal_text; +}; + +struct PlannedFilter { + PlannedValue left; + ComparisonKind comparison = ComparisonKind::Equal; + PlannedValue right; + ColumnType comparison_type = ColumnType::String; +}; + +struct PlannedQuery { + std::filesystem::path table_path; + std::vector projection_indexes; + std::optional filter; + std::vector group_keys; + std::vector aggregates; + std::vector select_items; + std::vector order_by; + std::optional limit; +}; diff --git a/include/executor/query_planner.h b/include/executor/query_planner.h new file mode 100644 index 0000000..913d6f4 --- /dev/null +++ b/include/executor/query_planner.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include +#include + +#include "executor/query.h" +#include "executor/query_plan.h" + +PlannedQuery PlanQuery(const Query& query, const std::unordered_map& tables); diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index bda9e89..d24ce7d 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -4,9 +4,10 @@ #include #include +#include "executor/aggregate_state.h" #include "executor/aggregate_function.h" #include "support/ascii.h" -#include "executor/executor.h" +#include "support/error.h" #include "support/int128.h" #include "support/parsing.h" @@ -306,22 +307,6 @@ bool AggSupportsInputType(const AggFuncDefinition& definition, const ColumnType return definition.supports_type(type); } -std::string FormatAgg(const AggFuncDefinition& definition, const std::string_view column_name, const bool distinct, - const bool star) { - std::string output(definition.canonical_name); - output += '('; - if (star) { - output += '*'; - } else { - if (distinct) { - output += "DISTINCT "; - } - output += column_name; - } - output += ')'; - return output; -} - std::unique_ptr CreateAggState(const PlannedAgg& aggregate) { if (aggregate.function == nullptr || aggregate.function->factory == nullptr) { throw Error::Unsupported("executor", "aggregate is not registered"); @@ -330,7 +315,7 @@ std::unique_ptr CreateAggState(const PlannedAgg& aggregate) { throw Error::Unsupported("executor", "DISTINCT is not supported for aggregate '" + std::string(aggregate.function->canonical_name) + "'"); } - if (aggregate.star && !aggregate.function->star) { + if (aggregate.argument_kind == AggArgumentKind::Star && !aggregate.function->star) { throw Error::Unsupported( "executor", "aggregate '" + std::string(aggregate.function->canonical_name) + "' does not support '*'"); } diff --git a/src/executor/executor.cpp b/src/executor/executor.cpp index 91e222d..49e44d7 100644 --- a/src/executor/executor.cpp +++ b/src/executor/executor.cpp @@ -1,7 +1,11 @@ #include "executor/executor.h" +#include #include +#include "executor/operator.h" +#include "executor/query_parser.h" +#include "executor/query_planner.h" #include "support/ascii.h" void Executor::RegisterTable(const std::string& name, std::filesystem::path path) { @@ -10,10 +14,10 @@ void Executor::RegisterTable(const std::string& name, std::filesystem::path path ExecuteExpected Executor::Execute(const std::string_view query) const { try { - const ParsedQuery parsed = ParseQuery(query); - const PlannedQuery planned = PlanQuery(parsed, tables_); + const Query query_ast = ParseQuery(query); + const PlannedQuery planned = PlanQuery(query_ast, tables_); - std::unique_ptr root = BuildPlan(planned); + const std::unique_ptr root = BuildPlan(planned); auto batch = root->Next(); if (!batch.has_value()) { diff --git a/src/executor/operator_utils.cpp b/src/executor/operator_utils.cpp index 6b047f4..4cafce0 100644 --- a/src/executor/operator_utils.cpp +++ b/src/executor/operator_utils.cpp @@ -3,6 +3,7 @@ #include #include "support/ascii.h" +#include "support/error.h" #include "support/parsing.h" template diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index 886708e..9c2ee79 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -4,10 +4,12 @@ #include #include -#include "executor/executor.h" +#include "executor/aggregate_state.h" +#include "executor/operator.h" #include "executor/operator_utils.h" #include "io/columnar_batch_io.h" #include "io/fileio.h" +#include "support/error.h" class ScanOperator final : public Operator { public: @@ -68,9 +70,10 @@ class FilterOperator final : public Operator { Batch filtered(batch->GetSchema()); for (size_t row = 0; row < batch->RowsCount(); ++row) { - const std::string value = batch->ColumnAt(filter_.column_index).ValueAsString(row); + const std::string left = MaterializeValue(filter_.left, *batch, row); + const std::string right = MaterializeValue(filter_.right, *batch, row); - if (!MatchesComparison(filter_.column_type, value, filter_.literal_text, filter_.comparison)) { + if (!MatchesComparison(filter_.comparison_type, left, right, filter_.comparison)) { continue; } @@ -87,6 +90,14 @@ class FilterOperator final : public Operator { return std::nullopt; } + private: + static std::string MaterializeValue(const PlannedValue& value, const Batch& batch, const size_t row) { + if (value.kind == PlannedValueKind::Literal) { + return value.literal_text; + } + return batch.ColumnAt(value.column_index).ValueAsString(row); + } + private: std::unique_ptr child_; PlannedFilter filter_; @@ -118,9 +129,9 @@ class AggOperator final : public Operator { while (auto batch = child_->Next()) { for (size_t row = 0; row < batch->RowsCount(); ++row) { - for (auto& binding : bindings_) { - if (!binding.aggregate.star) { - std::string materialized = batch->ColumnAt(*binding.aggregate.column_index).ValueAsString(row); + for (const auto& binding : bindings_) { + if (binding.aggregate.argument_kind == AggArgumentKind::Column) { + std::string materialized = batch->ColumnAt(binding.aggregate.column_index).ValueAsString(row); binding.state->Consume(materialized); } else { binding.state->Consume(std::nullopt); @@ -203,8 +214,8 @@ class GroupAggOperator final : public Operator { GroupState& group = groups_[group_index]; for (size_t i = 0; i < aggregates_.size(); ++i) { const PlannedAgg& aggregate = aggregates_[i]; - if (!aggregate.star) { - std::string materialized = batch->ColumnAt(*aggregate.column_index).ValueAsString(row); + if (aggregate.argument_kind == AggArgumentKind::Column) { + std::string materialized = batch->ColumnAt(aggregate.column_index).ValueAsString(row); group.states[i]->Consume(materialized); } else { group.states[i]->Consume(std::nullopt); diff --git a/src/executor/planner_utils.cpp b/src/executor/planner_utils.cpp index 1aa3eb2..0012cf2 100644 --- a/src/executor/planner_utils.cpp +++ b/src/executor/planner_utils.cpp @@ -1,6 +1,7 @@ #include "executor/planner_utils.h" #include "support/ascii.h" +#include "support/error.h" size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { const std::string needle = ToLowerAscii(column_name); @@ -16,15 +17,15 @@ static bool SameQualifier(const std::string_view lhs, const std::string_view rhs return ToLowerAscii(lhs) == ToLowerAscii(rhs); } -void ValidateColumnQualifier(const ParsedQuery& parsed, const ColumnRef& column) { +void ValidateColumnQualifier(const Query& query, const ColumnRef& column) { if (column.qualifier.empty()) { return; } - if (SameQualifier(column.qualifier, parsed.table_name)) { + if (SameQualifier(column.qualifier, query.table_name)) { return; } - if (!parsed.table_alias.empty() && SameQualifier(column.qualifier, parsed.table_alias)) { + if (!query.table_alias.empty() && SameQualifier(column.qualifier, query.table_alias)) { return; } @@ -51,13 +52,13 @@ static std::string UnescapeSqlString(const std::string_view text) { return result; } -std::string NormalizeLiteral(const ParsedLiteral& literal, const ColumnType type) { - switch (literal.token_type) { - case Tokens::StringLiteral: +std::string NormalizeLiteral(const QueryLiteral& literal, const ColumnType type) { + switch (literal.kind) { + case LiteralKind::String: return UnescapeSqlString(literal.text); - case Tokens::NumericLiteral: - case Tokens::NameToken: - if (type == ColumnType::String && literal.token_type == Tokens::NameToken) { + case LiteralKind::Numeric: + case LiteralKind::Identifier: + if (type == ColumnType::String && literal.kind == LiteralKind::Identifier) { return literal.text; } return literal.text; diff --git a/src/executor/query_parser.cpp b/src/executor/query_parser.cpp index 86a1e87..f139ba7 100644 --- a/src/executor/query_parser.cpp +++ b/src/executor/query_parser.cpp @@ -1,9 +1,12 @@ #include #include #include +#include -#include "executor/executor.h" +#include "executor/query_parser.h" +#include "sql_parser/tokenizer.h" #include "support/ascii.h" +#include "support/error.h" class TokenCursor { public: @@ -110,6 +113,76 @@ static ColumnRef ParseColumnRef(TokenCursor& cursor, const std::string_view mess return column; } +static QueryValue QueryColumnValue(ColumnRef column) { + return QueryValue{ + .kind = QueryValueKind::Column, + .column = std::move(column), + .literal = {}, + }; +} + +static QueryValue QueryLiteralValue(QueryLiteral literal) { + return QueryValue{ + .kind = QueryValueKind::Literal, + .column = {}, + .literal = std::move(literal), + }; +} + +static QueryValue ParseFilterValue(TokenCursor& cursor, const std::string_view message, + const bool bare_identifier_is_column) { + if (cursor.TryConsume(Tokens::Minus)) { + const Token& literal = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after '-'"); + return QueryLiteralValue(QueryLiteral{ + .text = "-" + std::string(literal.GetText()), + .kind = LiteralKind::Numeric, + }); + } + + const Token& token = cursor.Peek(); + switch (token.GetType()) { + case Tokens::NumericLiteral: + cursor.Consume(Tokens::NumericLiteral, message); + return QueryLiteralValue(QueryLiteral{ + .text = std::string(token.GetText()), + .kind = LiteralKind::Numeric, + }); + case Tokens::StringLiteral: + cursor.Consume(Tokens::StringLiteral, message); + return QueryLiteralValue(QueryLiteral{ + .text = std::string(token.GetText()), + .kind = LiteralKind::String, + }); + case Tokens::NameToken: + if (bare_identifier_is_column || cursor.Match(Tokens::Dot, 1)) { + return QueryColumnValue(ParseColumnRef(cursor, message)); + } + cursor.Consume(Tokens::NameToken, message); + return QueryLiteralValue(QueryLiteral{ + .text = std::string(token.GetText()), + .kind = LiteralKind::Identifier, + }); + default: + throw Error::InvalidArgument("query_parser", std::string(message)); + } +} + +static std::string FormatAggregateCall(const std::string_view function_name, const std::string_view column_name, + const bool distinct, const AggArgumentKind argument_kind) { + std::string output(function_name); + output += '('; + if (argument_kind == AggArgumentKind::Star) { + output += '*'; + } else { + if (distinct) { + output += "DISTINCT "; + } + output += column_name; + } + output += ')'; + return output; +} + static std::optional ParseOptionalLimit(TokenCursor& cursor) { if (!cursor.TryConsume(Tokens::Limit)) { return std::nullopt; @@ -131,27 +204,20 @@ static AggSpec ParseAggregate(TokenCursor& cursor) { const Token& function_token = cursor.ConsumeFunctionToken("expected aggregate function"); AggSpec spec; - spec.function = &ResolveAggFunc(function_token.GetText()); + spec.function_name = ToUpperAscii(function_token.GetText()); cursor.Consume(Tokens::OpenBracket, "expected '(' after aggregate function"); if (cursor.TryConsume(Tokens::Asterisk)) { - if (!spec.function->star) { - throw Error::InvalidArgument("query_parser", - std::string(spec.function->canonical_name) + " does not support '*'"); - } - spec.star = true; + spec.argument_kind = AggArgumentKind::Star; } else { spec.distinct = cursor.TryConsume(Tokens::Distinct); - if (spec.distinct && !spec.function->distinct) { - throw Error::InvalidArgument("query_parser", - std::string(spec.function->canonical_name) + " does not support DISTINCT"); - } spec.column = ParseColumnRef(cursor, "expected column name in aggregate"); } cursor.Consume(Tokens::CloseBracket, "expected ')' after aggregate arguments"); - spec.output_name = FormatAgg(*spec.function, FormatColumnRef(spec.column), spec.distinct, spec.star); + spec.output_name = FormatAggregateCall(spec.function_name, FormatColumnRef(spec.column), spec.distinct, + spec.argument_kind); return spec; } @@ -216,7 +282,7 @@ static OrderBySpec ParseOrderByItem(TokenCursor& cursor) { }; } -ParsedQuery ParseQuery(const std::string_view query) { +Query ParseQuery(const std::string_view query) { auto tokenized = TokenizeSql(query); if (!tokenized.has_value()) { throw tokenized.error(); @@ -224,74 +290,54 @@ ParsedQuery ParseQuery(const std::string_view query) { TokenCursor cursor(std::move(tokenized.value())); - ParsedQuery parsed; + Query query_ast; cursor.Consume(Tokens::Select, "expected SELECT"); - parsed.select_items.push_back(ParseSelectItem(cursor)); + query_ast.select_items.push_back(ParseSelectItem(cursor)); while (cursor.TryConsume(Tokens::Comma)) { - parsed.select_items.push_back(ParseSelectItem(cursor)); + query_ast.select_items.push_back(ParseSelectItem(cursor)); } cursor.Consume(Tokens::From, "expected FROM"); - parsed.table_name = std::string(cursor.Consume(Tokens::NameToken, "expected table name").GetText()); + query_ast.table_name = std::string(cursor.Consume(Tokens::NameToken, "expected table name").GetText()); if (cursor.TryConsume(Tokens::As)) { - parsed.table_alias = std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); + query_ast.table_alias = std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); } else if (cursor.Match(Tokens::NameToken)) { - parsed.table_alias = std::string(cursor.Consume(Tokens::NameToken, "expected table alias").GetText()); + query_ast.table_alias = std::string(cursor.Consume(Tokens::NameToken, "expected table alias").GetText()); } if (cursor.TryConsume(Tokens::Where)) { FilterSpec filter; - filter.column = ParseColumnRef(cursor, "expected column name after WHERE"); + filter.left = ParseFilterValue(cursor, "expected left operand after WHERE", true); const Tokens comparison_token = cursor.Peek().GetType(); filter.comparison = ComparisonKindFromToken(comparison_token); cursor.Consume(comparison_token, "expected comparison operator in WHERE"); + filter.right = ParseFilterValue(cursor, "expected right operand after comparison operator", false); - if (cursor.TryConsume(Tokens::Minus)) { - const Token& literal = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after '-'"); - filter.literal = - ParsedLiteral{.text = "-" + std::string(literal.GetText()), .token_type = Tokens::NumericLiteral}; - } else { - const Token& literal = cursor.Peek(); - switch (literal.GetType()) { - case Tokens::NumericLiteral: - case Tokens::StringLiteral: - case Tokens::NameToken: - filter.literal = ParsedLiteral{ - .text = std::string(literal.GetText()), - .token_type = literal.GetType(), - }; - cursor.Consume(literal.GetType(), "expected literal after comparison operator"); - break; - default: - throw Error::InvalidArgument("query_parser", "unsupported literal in WHERE"); - } - } - - parsed.filter = std::move(filter); + query_ast.filter = std::move(filter); } if (cursor.TryConsume(Tokens::Group)) { cursor.Consume(Tokens::By, "expected BY after GROUP"); - parsed.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); + query_ast.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); while (cursor.TryConsume(Tokens::Comma)) { - parsed.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); + query_ast.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); } } if (cursor.TryConsume(Tokens::Order)) { cursor.Consume(Tokens::By, "expected BY after ORDER"); - parsed.order_by.push_back(ParseOrderByItem(cursor)); + query_ast.order_by.push_back(ParseOrderByItem(cursor)); while (cursor.TryConsume(Tokens::Comma)) { - parsed.order_by.push_back(ParseOrderByItem(cursor)); + query_ast.order_by.push_back(ParseOrderByItem(cursor)); } } - parsed.limit = ParseOptionalLimit(cursor); + query_ast.limit = ParseOptionalLimit(cursor); cursor.TryConsume(Tokens::Semicolon); cursor.Consume(Tokens::EndOfInput, "unexpected trailing tokens"); - return parsed; + return query_ast; } diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index 9de1d36..44865a2 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -1,18 +1,28 @@ +#include "executor/query_planner.h" + #include -#include "executor/executor.h" #include "executor/operator_utils.h" #include "executor/planner_utils.h" #include "io/columnar_batch_io.h" #include "support/ascii.h" +#include "support/error.h" + +static std::optional TryFindColumnIndex(const Schema& schema, const std::string_view column_name) { + for (size_t i = 0; i < schema.columns.size(); ++i) { + if (SameColumnName(schema.columns[i].name, column_name)) { + return i; + } + } + return std::nullopt; +} -PlannedQuery PlanQuery(const ParsedQuery& parsed, - const std::unordered_map& tables) { +PlannedQuery PlanQuery(const Query& query_ast, const std::unordered_map& tables) { PlannedQuery planned; - const auto table_it = tables.find(ToLowerAscii(parsed.table_name)); + const auto table_it = tables.find(ToLowerAscii(query_ast.table_name)); if (table_it == tables.end()) { - throw Error::NotFound("query_planner", "unknown table", parsed.table_name); + throw Error::NotFound("query_planner", "unknown table", query_ast.table_name); } planned.table_path = table_it->second; @@ -29,22 +39,82 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, return planned.projection_indexes.size() - 1; }; - if (parsed.filter.has_value()) { - ValidateColumnQualifier(parsed, parsed.filter->column); - const size_t source_index = FindColumnIndex(schema, parsed.filter->column.name); - const ColumnType column_type = schema.columns[source_index].type; + struct PlannedFilterValue { + PlannedValue value; + std::optional literal; + }; + + const auto plan_filter_value = [&](const QueryValue& value) -> PlannedFilterValue { + if (value.kind == QueryValueKind::Column) { + ValidateColumnQualifier(query_ast, value.column); + const size_t source_index = FindColumnIndex(schema, value.column.name); + return PlannedFilterValue{ + .value = + PlannedValue{ + .kind = PlannedValueKind::Column, + .column_type = schema.columns[source_index].type, + .column_index = register_projection(source_index), + .literal_text = {}, + }, + .literal = std::nullopt, + }; + } + + if (value.literal.kind == LiteralKind::Identifier) { + if (const std::optional source_index = TryFindColumnIndex(schema, value.literal.text); + source_index.has_value()) { + return PlannedFilterValue{ + .value = + PlannedValue{ + .kind = PlannedValueKind::Column, + .column_type = schema.columns[*source_index].type, + .column_index = register_projection(*source_index), + .literal_text = {}, + }, + .literal = std::nullopt, + }; + } + } + + return PlannedFilterValue{ + .value = + PlannedValue{ + .kind = PlannedValueKind::Literal, + .column_type = ColumnType::String, + .column_index = 0, + .literal_text = {}, + }, + .literal = value.literal, + }; + }; + + if (query_ast.filter.has_value()) { + PlannedFilterValue left = plan_filter_value(query_ast.filter->left); + PlannedFilterValue right = plan_filter_value(query_ast.filter->right); + + const ColumnType comparison_type = left.value.kind == PlannedValueKind::Column ? left.value.column_type + : right.value.kind == PlannedValueKind::Column ? right.value.column_type + : ColumnType::String; + if (left.literal.has_value()) { + left.value.column_type = comparison_type; + left.value.literal_text = NormalizeLiteral(*left.literal, comparison_type); + } + if (right.literal.has_value()) { + right.value.column_type = comparison_type; + right.value.literal_text = NormalizeLiteral(*right.literal, comparison_type); + } planned.filter = PlannedFilter{ - .column_index = register_projection(source_index), - .column_type = column_type, - .comparison = parsed.filter->comparison, - .literal_text = NormalizeLiteral(parsed.filter->literal, column_type), + .left = std::move(left.value), + .comparison = query_ast.filter->comparison, + .right = std::move(right.value), + .comparison_type = comparison_type, }; } - planned.group_keys.reserve(parsed.group_by.size()); - for (const ColumnRef& group_column : parsed.group_by) { - ValidateColumnQualifier(parsed, group_column); + planned.group_keys.reserve(query_ast.group_by.size()); + for (const ColumnRef& group_column : query_ast.group_by) { + ValidateColumnQualifier(query_ast, group_column); const size_t source_index = FindColumnIndex(schema, group_column.name); planned.group_keys.push_back(PlannedGroupKey{ .column_index = register_projection(source_index), @@ -54,23 +124,23 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, } bool has_aggregate = false; - for (const auto& item : parsed.select_items) { + for (const auto& item : query_ast.select_items) { if (item.kind == SelectItemKind::Aggregate) { has_aggregate = true; break; } } - planned.select_items.reserve(parsed.select_items.size()); - planned.aggregates.reserve(parsed.select_items.size()); + planned.select_items.reserve(query_ast.select_items.size()); + planned.aggregates.reserve(query_ast.select_items.size()); - for (const auto& item : parsed.select_items) { + for (const auto& item : query_ast.select_items) { if (item.kind == SelectItemKind::GroupKey) { - if (parsed.group_by.empty()) { + if (query_ast.group_by.empty()) { throw Error::Unsupported("query_planner", "plain column SELECT items require GROUP BY"); } - ValidateColumnQualifier(parsed, item.column); + ValidateColumnQualifier(query_ast, item.column); const auto it = std::ranges::find_if(planned.group_keys, [&](const PlannedGroupKey& group_key) { return SameColumnName(group_key.output_name, item.column.name); }); @@ -89,24 +159,35 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, } const AggSpec& aggregate = item.aggregate; + const AggFuncDefinition& function = ResolveAggFunc(aggregate.function_name); + + if (aggregate.distinct && !function.distinct) { + throw Error::InvalidArgument("query_planner", + std::string(function.canonical_name) + " does not support DISTINCT"); + } + if (aggregate.argument_kind == AggArgumentKind::Star && !function.star) { + throw Error::InvalidArgument("query_planner", + std::string(function.canonical_name) + " does not support '*'"); + } + PlannedAgg planned_aggregate{ - .function = aggregate.function, + .function = &function, .distinct = aggregate.distinct, - .star = aggregate.star, - .column_index = std::nullopt, + .argument_kind = aggregate.argument_kind, + .column_index = 0, .input_type = ColumnType::String, .output_name = item.output_name, }; - if (!aggregate.star) { - ValidateColumnQualifier(parsed, aggregate.column); + if (aggregate.argument_kind == AggArgumentKind::Column) { + ValidateColumnQualifier(query_ast, aggregate.column); const size_t source_index = FindColumnIndex(schema, aggregate.column.name); planned_aggregate.column_index = register_projection(source_index); planned_aggregate.input_type = schema.columns[source_index].type; - if (!AggSupportsInputType(*aggregate.function, planned_aggregate.input_type)) { - throw Error::Unsupported("query_planner", - "aggregate " + std::string(aggregate.function->canonical_name) + - " does not support column '" + aggregate.column.name + "'"); + if (!AggSupportsInputType(function, planned_aggregate.input_type)) { + throw Error::Unsupported("query_planner", "aggregate " + std::string(function.canonical_name) + + " does not support column '" + aggregate.column.name + + "'"); } } @@ -120,20 +201,20 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, }); } - if (parsed.group_by.empty() && !has_aggregate) { + if (query_ast.group_by.empty() && !has_aggregate) { throw Error::Unsupported("query_planner", "plain SELECT without aggregates is not supported"); } - planned.order_by.reserve(parsed.order_by.size()); - for (const auto& order_item : parsed.order_by) { - const auto it = std::ranges::find_if(parsed.select_items, [&](const SelectItemSpec& select_item) { + planned.order_by.reserve(query_ast.order_by.size()); + for (const auto& order_item : query_ast.order_by) { + const auto it = std::ranges::find_if(query_ast.select_items, [&](const SelectItemSpec& select_item) { return SameSelectItem(select_item, order_item.item) || SameOutputName(select_item, order_item.item); }); - if (it == parsed.select_items.end()) { + if (it == query_ast.select_items.end()) { throw Error::InvalidArgument("query_planner", "ORDER BY expression must appear in SELECT"); } - const size_t result_column_index = static_cast(std::distance(parsed.select_items.begin(), it)); + const size_t result_column_index = static_cast(std::distance(query_ast.select_items.begin(), it)); planned.order_by.push_back(PlannedOrderBy{ .result_column_index = result_column_index, .descending = order_item.descending, @@ -141,7 +222,7 @@ PlannedQuery PlanQuery(const ParsedQuery& parsed, }); } - planned.limit = parsed.limit; + planned.limit = query_ast.limit; if (planned.projection_indexes.empty() && !schema.columns.empty()) { planned.projection_indexes.push_back(0); diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index a17bbda..2ce8be1 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -103,6 +103,13 @@ TEST(executor, supports_from_with_bare_alias) { EXPECT_EQ(actual_rows, expected_rows); } +TEST(executor, supports_column_to_column_filter) { + const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits h WHERE h.AdvEngineID > h.SearchEngineID;"); + + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"3"}); +} + TEST(executor, supports_select_alias_order_by_alias_and_limit) { const Batch batch = BuildHitsTable( "SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 2;"); @@ -123,9 +130,8 @@ TEST(executor, supports_multiple_aggregates_with_alias_and_limit) { "SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) " "FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 2;"); - EXPECT_EQ(BatchColumnNames(batch), - (std::vector{"RegionID", "SUM(AdvEngineID)", "c", "AVG(ResolutionWidth)", - "COUNT(DISTINCT UserID)"})); + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"RegionID", "SUM(AdvEngineID)", "c", + "AVG(ResolutionWidth)", "COUNT(DISTINCT UserID)"})); auto actual_rows = BatchRows(batch); auto expected_rows = std::vector>{ {"10", "3", "2", "200", "2"}, From 55365403aa3b7e7966e3594cd1960a13f0bff64b Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sat, 16 May 2026 02:29:41 +0300 Subject: [PATCH 20/29] review: io module done --- include/io/columnar_batch_io.h | 6 +- include/io/fileio.h | 28 ++ include/model/schema.h | 5 - include/model/schema_csv.h | 9 + src/CMakeLists.txt | 56 +++- src/app/main.cpp | 1 + src/convert/csv_columnar.cpp | 2 +- src/executor/operators.cpp | 8 +- src/io/batch_io.cpp | 312 +---------------------- src/io/columnar_batch_io.cpp | 146 +++++++++++ src/io/csv_batch_io.cpp | 163 ++++++++++++ src/io/fileio.cpp | 24 ++ src/model/{schema.cpp => schema_csv.cpp} | 5 +- tests/test_columnar.cpp | 1 + tests/test_fileio.cpp | 11 + 15 files changed, 449 insertions(+), 328 deletions(-) create mode 100644 include/model/schema_csv.h create mode 100644 src/io/columnar_batch_io.cpp create mode 100644 src/io/csv_batch_io.cpp rename src/model/{schema.cpp => schema_csv.cpp} (98%) diff --git a/include/io/columnar_batch_io.h b/include/io/columnar_batch_io.h index 6cafa3b..27c3ebe 100644 --- a/include/io/columnar_batch_io.h +++ b/include/io/columnar_batch_io.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include "io/batch_io.h" @@ -28,7 +27,7 @@ class ColumnarBatchReader final : public BatchReader { private: std::filesystem::path path_; - std::ifstream in_; + InputFile input_; ColumnarMetadata metadata_; size_t next_group_ = 0; @@ -51,9 +50,6 @@ class ColumnarBatchWriter final : public BatchWriter { const ColumnarMetadata& GetMetadata() const { return metadata_; } - private: - void FinalizeImpl(); - private: std::filesystem::path path_; std::ofstream out_; diff --git a/include/io/fileio.h b/include/io/fileio.h index 2e62c62..5394e9f 100644 --- a/include/io/fileio.h +++ b/include/io/fileio.h @@ -1,9 +1,11 @@ #pragma once +#include #include #include #include #include +#include #include struct FileMetadata { @@ -21,6 +23,32 @@ void EnsureParentDirectory(const std::filesystem::path& path); std::ifstream OpenInputFile(const std::filesystem::path& path); void SeekInputFile(std::istream& in, const std::filesystem::path& path, uint64_t offset); +class InputFile { + public: + explicit InputFile(std::filesystem::path path); + InputFile(const InputFile&) = delete; + InputFile(InputFile&&) noexcept = default; + InputFile& operator=(const InputFile&) = delete; + InputFile& operator=(InputFile&&) noexcept = default; + ~InputFile() = default; + + public: + void ReadAt(char* dst, size_t size, uint64_t offset); + std::string ReadStringAt(uint64_t offset, size_t size); + std::istream& StreamAt(uint64_t offset); + + template + T ReadAt(const uint64_t offset) { + T value = 0; + ReadAt(reinterpret_cast(&value), sizeof(value), offset); + return value; + } + + private: + std::filesystem::path path_; + std::ifstream in_; +}; + enum class FileOpenMode { Truncate, Append, diff --git a/include/model/schema.h b/include/model/schema.h index 8cce2c9..85f3c40 100644 --- a/include/model/schema.h +++ b/include/model/schema.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include @@ -29,7 +28,3 @@ struct Schema { bool operator==(const Schema& other) const = default; }; - -Schema ReadSchemaCsv(const std::filesystem::path& path); -Schema InferSchemaCsv(const std::filesystem::path& path); -void WriteSchemaCsv(const std::filesystem::path& path, const Schema& schema); diff --git a/include/model/schema_csv.h b/include/model/schema_csv.h new file mode 100644 index 0000000..cc08543 --- /dev/null +++ b/include/model/schema_csv.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +#include "model/schema.h" + +Schema ReadSchemaCsv(const std::filesystem::path& path); +Schema InferSchemaCsv(const std::filesystem::path& path); +void WriteSchemaCsv(const std::filesystem::path& path, const Schema& schema); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8f5ac71..df4b0f5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,5 @@ -add_library(columnar_engine_lib +add_library(columnar_engine_core model/batch.cpp - io/batch_io.cpp model/column.cpp model/column_boolean.cpp model/column_character.cpp @@ -11,15 +10,42 @@ add_library(columnar_engine_lib model/column_int64.cpp model/column_string.cpp model/column_timestamp.cpp - convert/csv_columnar.cpp - io/csv.cpp + model/metadata.cpp + io/batch_io.cpp + io/fileio.cpp + io/stream.cpp support/error.cpp support/ascii.cpp - io/fileio.cpp - model/metadata.cpp support/parsing.cpp - model/schema.cpp - io/stream.cpp +) + +target_include_directories(columnar_engine_core PUBLIC ${COLUMNAR_INCLUDE_DIRS}) + +add_library(columnar_engine_columnar + io/columnar_batch_io.cpp +) + +target_link_libraries(columnar_engine_columnar PUBLIC columnar_engine_core) + +add_library(columnar_engine_csv + io/csv.cpp + io/csv_batch_io.cpp + model/schema_csv.cpp +) + +target_link_libraries(columnar_engine_csv PUBLIC columnar_engine_core) + +add_library(columnar_engine_convert + convert/csv_columnar.cpp +) + +target_link_libraries(columnar_engine_convert + PUBLIC + columnar_engine_columnar + columnar_engine_csv +) + +add_library(columnar_engine_executor executor/executor.cpp executor/query_parser.cpp executor/query_planner.cpp @@ -32,6 +58,18 @@ add_library(columnar_engine_lib sql_parser/tokenizer.cpp ) -target_include_directories(columnar_engine_lib PUBLIC ${COLUMNAR_INCLUDE_DIRS}) +target_link_libraries(columnar_engine_executor PUBLIC columnar_engine_columnar) + +add_library(columnar_engine_lib INTERFACE) + +target_link_libraries(columnar_engine_lib + INTERFACE + columnar_engine_core + columnar_engine_columnar + columnar_engine_csv + columnar_engine_convert + columnar_engine_executor +) + add_executable(columnar_engine app/main.cpp) target_link_libraries(columnar_engine PRIVATE columnar_engine_lib argparse) diff --git a/src/app/main.cpp b/src/app/main.cpp index 21b790f..721cc3b 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -9,6 +9,7 @@ #include "executor/executor.h" #include "io/fileio.h" #include "model/schema.h" +#include "model/schema_csv.h" void ConfigureInferSchemaCommand(argparse::ArgumentParser& command) { command.add_description("Infer a schema from a CSV file."); diff --git a/src/convert/csv_columnar.cpp b/src/convert/csv_columnar.cpp index 590ee89..e119935 100644 --- a/src/convert/csv_columnar.cpp +++ b/src/convert/csv_columnar.cpp @@ -4,8 +4,8 @@ #include "io/columnar_batch_io.h" #include "io/csv_batch_io.h" +#include "model/schema_csv.h" #include "support/error.h" -#include "model/schema.h" void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, const std::filesystem::path& output_path, size_t max_rows_per_group) { diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index 9c2ee79..d3f9dbf 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -14,7 +13,7 @@ class ScanOperator final : public Operator { public: ScanOperator(std::filesystem::path path, std::vector projection_indexes) - : path_(std::move(path)), in_(OpenInputFile(path_)), metadata_(ColumnarBatchReader(path_).GetMetadata()) { + : path_(std::move(path)), input_(path_), metadata_(ColumnarBatchReader(path_).GetMetadata()) { if (metadata_.schema.columns.empty()) { throw Error::InvalidData("operators", "columnar schema is empty", path_.string()); } @@ -42,8 +41,7 @@ class ScanOperator final : public Operator { for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { const size_t source_index = projection_indexes_[projected_index]; const auto& chunk = row_group.columns[source_index]; - SeekInputFile(in_, path_, chunk.offset); - batch.ColumnAt(projected_index).ReadFrom(in_, row_group.row_count, chunk.size); + batch.ColumnAt(projected_index).ReadFrom(input_.StreamAt(chunk.offset), row_group.row_count, chunk.size); } return batch; @@ -51,7 +49,7 @@ class ScanOperator final : public Operator { private: std::filesystem::path path_; - std::ifstream in_; + InputFile input_; ColumnarMetadata metadata_; std::vector projection_indexes_; diff --git a/src/io/batch_io.cpp b/src/io/batch_io.cpp index 786caf5..f918fba 100644 --- a/src/io/batch_io.cpp +++ b/src/io/batch_io.cpp @@ -1,70 +1,23 @@ -#include -#include -#include +#include "io/batch_io.h" -#include "io/columnar_batch_io.h" -#include "io/csv.h" -#include "io/csv_batch_io.h" -#include "io/stream.h" -#include "model/metadata.h" -#include "support/error.h" - -static constexpr std::string_view ColumnarMagic = "CLMN"; - -static uint64_t TellWrite(const std::filesystem::path& path, std::ofstream& out) { - const auto pos = out.tellp(); - if (pos == -1) { - throw Error::PathIo("batch_io", path, "tell file"); - } - return pos; -} - -static void FlushWrite(const std::filesystem::path& path, std::ofstream& out) { - out.flush(); - if (!out) { - throw Error::PathIo("batch_io", path, "write file"); - } -} +#include +#include -static uint64_t AddChecked(const uint64_t current, const uint64_t add) { - if (add > std::numeric_limits::max() - current) { - throw Error::Overflow("batch_io", "batch byte size overflow"); - } - return current + add; -} +#include "support/error.h" -static uint64_t EstimateRowBytes(const std::vector>& columns, - const std::vector& row) { - uint64_t bytes = 0; - for (size_t i = 0; i < columns.size(); ++i) { - bytes = AddChecked(bytes, columns[i]->EstimateSizeFromString(row[i])); +static void ValidateOptionalLimit(const std::optional& value, const std::string_view name) { + if (value && *value == 0) { + throw Error::InvalidArgument("batch_io", std::string(name) + " must be > 0"); } - return bytes; } -static void ValidateSizing(const BatchSizing& sizing) { - if (sizing.max_rows && *sizing.max_rows == 0) { - throw Error::InvalidArgument("batch_io", "max rows must be > 0"); - } - if (sizing.max_values && *sizing.max_values == 0) { - throw Error::InvalidArgument("batch_io", "max values must be > 0"); - } - if (sizing.max_bytes && *sizing.max_bytes == 0) { +bool BatchSizing::WouldExceed(const size_t next_rows, const size_t column_count, const uint64_t next_bytes) const { + ValidateOptionalLimit(max_rows, "max rows"); + ValidateOptionalLimit(max_values, "max values"); + if (max_bytes && *max_bytes == 0) { throw Error::InvalidArgument("batch_io", "max bytes must be > 0"); } -} - -static void ReserveForSizing(const Batch& batch, const BatchSizing& sizing, const size_t column_count) { - if (sizing.max_rows) { - batch.Reserve(*sizing.max_rows); - return; - } - if (sizing.max_values && column_count > 0) { - batch.Reserve(std::max(1, *sizing.max_values / column_count)); - } -} -bool BatchSizing::WouldExceed(const size_t next_rows, const size_t column_count, const uint64_t next_bytes) const { if (max_rows && next_rows > *max_rows) { return true; } @@ -78,246 +31,3 @@ bool BatchSizing::WouldExceed(const size_t next_rows, const size_t column_count, } return false; } - -CsvBatchReader::CsvBatchReader(const std::filesystem::path& path, Schema schema, BatchSizing sizing) - : csv_reader_(path), schema_(std::move(schema)), sizing_(std::move(sizing)) { - if (schema_.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns"); - } - ValidateSizing(sizing_); -} - -std::optional CsvBatchReader::ReadNext() { - if (reached_eof_ && !pending_row_) { - return std::nullopt; - } - - Batch batch(schema_); - - const size_t column_count = schema_.columns.size(); - ReserveForSizing(batch, sizing_, column_count); - const auto& columns = batch.GetColumns(); - - std::vector row; - row.reserve(column_count); - - size_t rows = 0; - uint64_t bytes = 0; - - while (true) { - if (pending_row_) { - row = std::move(*pending_row_); - pending_row_.reset(); - } else { - if (!csv_reader_.ReadRow(row)) { - reached_eof_ = true; - break; - } - } - - if (row.size() != column_count) { - throw Error::Mismatch("batch_io", "data csv column count mismatch"); - } - - const size_t next_rows = rows + 1; - uint64_t next_bytes = bytes; - - if (sizing_.max_bytes) { - next_bytes = AddChecked(bytes, EstimateRowBytes(columns, row)); - } - - if (rows > 0 && sizing_.WouldExceed(next_rows, column_count, next_bytes)) { - pending_row_ = std::move(row); - break; - } - - for (size_t col = 0; col < column_count; ++col) { - columns[col]->AppendFromString(row[col]); - } - - rows = next_rows; - bytes = next_bytes; - } - - if (rows == 0 && reached_eof_ && !pending_row_) { - return std::nullopt; - } - - return batch; -} - -CsvBatchWriter::CsvBatchWriter(const std::filesystem::path& path, Schema schema) - : csv_writer_(path), schema_(std::move(schema)) { - if (schema_.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns"); - } -} - -void CsvBatchWriter::Write(const Batch& batch) { - batch.Validate(); - if (batch.GetSchema() != schema_) { - throw Error::Mismatch("batch_io", "batch schema mismatch"); - } - - const size_t column_count = schema_.columns.size(); - const size_t row_count = batch.RowsCount(); - - std::vector row(column_count); - - for (size_t row_index = 0; row_index < row_count; ++row_index) { - for (size_t col = 0; col < column_count; ++col) { - row[col] = batch.ColumnAt(col).ValueAsString(row_index); - } - csv_writer_.WriteRow(row); - } -} - -void CsvBatchWriter::Flush() { csv_writer_.Flush(); } - -void AppendBatchRows(const Batch& batch, std::vector>& rows) { - const size_t row_count = batch.RowsCount(); - const size_t column_count = batch.ColumnsCount(); - - for (size_t row = 0; row < row_count; ++row) { - std::vector values; - values.reserve(column_count); - - for (size_t col = 0; col < column_count; ++col) { - values.push_back(batch.ColumnAt(col).ValueAsString(row)); - } - - rows.push_back(std::move(values)); - } -} - -void WriteBatchCsv(const std::filesystem::path& path, const Batch& batch) { - CsvBatchWriter writer(path, batch.GetSchema()); - writer.Write(batch); - writer.Flush(); -} - -ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::path& path) { - const auto file_metadata = TryGetFileMetadata(path); - - if (!file_metadata || !file_metadata->is_regular) { - throw Error::NotFound("batch_io", "columnar file not found", path.string()); - } - - const uint64_t file_size = file_metadata->size; - constexpr uint64_t FooterSize = sizeof(uint64_t) + ColumnarMagic.size(); - - if (file_size < FooterSize) { - throw Error::InvalidData("batch_io", "columnar file is too small", path.string()); - } - - std::ifstream in = OpenInputFile(path); - SeekInputFile(in, path, file_size - FooterSize); - - const uint64_t metadata_size = ReadStream(in); - std::string magic_read(ColumnarMagic.size(), '\0'); - ReadBytes(in, magic_read.data(), magic_read.size()); - - if (magic_read != ColumnarMagic) { - throw Error::InvalidData("batch_io", "invalid columnar magic", path.string()); - } - - if (metadata_size > file_size - FooterSize) { - throw Error::InvalidData("batch_io", "metadata size exceeds file size", path.string()); - } - - SeekInputFile(in, path, file_size - FooterSize - metadata_size); - - return ReadMetadata(in); -} - -ColumnarBatchReader::ColumnarBatchReader(const std::filesystem::path& path) - : path_(path), in_(OpenInputFile(path)), metadata_(ReadFileMetadata(path)) { - if (metadata_.schema.columns.empty()) { - throw Error::InvalidData("batch_io", "columnar schema is empty", path.string()); - } -} - -std::optional ColumnarBatchReader::ReadNext() { - if (next_group_ >= metadata_.row_groups.size()) { - return std::nullopt; - } - - const auto& [row_count, row_group_columns] = metadata_.row_groups[next_group_++]; - Batch batch(metadata_.schema, row_count); - - const auto& columns = batch.GetColumns(); - - if (row_group_columns.size() != columns.size()) { - throw Error::Mismatch("batch_io", "row group column count mismatch", path_.string()); - } - - for (size_t col = 0; col < columns.size(); ++col) { - const auto& [offset, size] = row_group_columns[col]; - SeekInputFile(in_, path_, offset); - columns[col]->ReadFrom(in_, row_count, size); - } - - return batch; -} - -ColumnarBatchWriter::ColumnarBatchWriter(const std::filesystem::path& path, Schema schema) - : path_(path), out_(OpenOutputFile(path)) { - if (schema.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns", path.string()); - } - metadata_.schema = std::move(schema); -} - -void ColumnarBatchWriter::Write(const Batch& batch) { - if (finalized_) { - throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); - } - - batch.Validate(); - if (batch.GetSchema() != metadata_.schema) { - throw Error::Mismatch("batch_io", "batch schema mismatch", path_.string()); - } - - const size_t row_count = batch.RowsCount(); - - if (row_count == 0) { - return; - } - if (row_count > std::numeric_limits::max()) { - throw Error::Overflow("batch_io", "row group exceeds supported size", path_.string()); - } - - RowGroupMetadata group; - group.row_count = row_count; - group.columns.reserve(batch.ColumnsCount()); - - for (const auto& column : batch.GetColumns()) { - const uint64_t offset = TellWrite(path_, out_); - column->WriteTo(out_); - const uint64_t end = TellWrite(path_, out_); - group.columns.push_back(ColumnChunkMetadata{offset, end - offset}); - } - - metadata_.row_groups.push_back(std::move(group)); -} - -void ColumnarBatchWriter::Flush() { FlushWrite(path_, out_); } - -void ColumnarBatchWriter::Finalize() && { FinalizeImpl(); } - -void ColumnarBatchWriter::FinalizeImpl() { - if (finalized_) { - throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); - } - - const uint64_t metadata_start = TellWrite(path_, out_); - WriteMetadata(out_, metadata_); - const uint64_t metadata_end = TellWrite(path_, out_); - const uint64_t metadata_size = metadata_end - metadata_start; - - WriteStream(out_, metadata_size); - WriteBytes(out_, ColumnarMagic); - - FlushWrite(path_, out_); - finalized_ = true; -} diff --git a/src/io/columnar_batch_io.cpp b/src/io/columnar_batch_io.cpp new file mode 100644 index 0000000..b8dc931 --- /dev/null +++ b/src/io/columnar_batch_io.cpp @@ -0,0 +1,146 @@ +#include "io/columnar_batch_io.h" + +#include +#include +#include +#include +#include +#include + +#include "io/stream.h" +#include "support/error.h" + +static constexpr std::string_view ColumnarMagic = "CLMN"; + +static uint64_t TellWrite(const std::filesystem::path& path, std::ofstream& out) { + const auto pos = out.tellp(); + if (pos == -1) { + throw Error::PathIo("batch_io", path, "tell file"); + } + return pos; +} + +static void FlushWrite(const std::filesystem::path& path, std::ofstream& out) { + out.flush(); + if (!out) { + throw Error::PathIo("batch_io", path, "write file"); + } +} + +ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::path& path) { + const auto file_metadata = TryGetFileMetadata(path); + + if (!file_metadata || !file_metadata->is_regular) { + throw Error::NotFound("batch_io", "columnar file not found", path.string()); + } + + const uint64_t file_size = file_metadata->size; + constexpr uint64_t FooterSize = sizeof(uint64_t) + ColumnarMagic.size(); + + if (file_size < FooterSize) { + throw Error::InvalidData("batch_io", "columnar file is too small", path.string()); + } + + InputFile file(path); + const uint64_t metadata_size = file.ReadAt(file_size - FooterSize); + const std::string magic_read = file.ReadStringAt(file_size - ColumnarMagic.size(), ColumnarMagic.size()); + + if (magic_read != ColumnarMagic) { + throw Error::InvalidData("batch_io", "invalid columnar magic", path.string()); + } + + if (metadata_size > file_size - FooterSize) { + throw Error::InvalidData("batch_io", "metadata size exceeds file size", path.string()); + } + + return ReadMetadata(file.StreamAt(file_size - FooterSize - metadata_size)); +} + +ColumnarBatchReader::ColumnarBatchReader(const std::filesystem::path& path) + : path_(path), input_(path), metadata_(ReadFileMetadata(path)) { + if (metadata_.schema.columns.empty()) { + throw Error::InvalidData("batch_io", "columnar schema is empty", path.string()); + } +} + +std::optional ColumnarBatchReader::ReadNext() { + if (next_group_ >= metadata_.row_groups.size()) { + return std::nullopt; + } + + const auto& [row_count, row_group_columns] = metadata_.row_groups[next_group_++]; + Batch batch(metadata_.schema, row_count); + + const auto& columns = batch.GetColumns(); + + if (row_group_columns.size() != columns.size()) { + throw Error::Mismatch("batch_io", "row group column count mismatch", path_.string()); + } + + for (size_t col = 0; col < columns.size(); ++col) { + const auto& [offset, size] = row_group_columns[col]; + columns[col]->ReadFrom(input_.StreamAt(offset), row_count, size); + } + + return batch; +} + +ColumnarBatchWriter::ColumnarBatchWriter(const std::filesystem::path& path, Schema schema) + : path_(path), out_(OpenOutputFile(path)) { + if (schema.columns.empty()) { + throw Error::InvalidArgument("batch_io", "schema has no columns", path.string()); + } + metadata_.schema = std::move(schema); +} + +void ColumnarBatchWriter::Write(const Batch& batch) { + if (finalized_) { + throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); + } + + batch.Validate(); + if (batch.GetSchema() != metadata_.schema) { + throw Error::Mismatch("batch_io", "batch schema mismatch", path_.string()); + } + + const size_t row_count = batch.RowsCount(); + + if (row_count == 0) { + return; + } + if (row_count > std::numeric_limits::max()) { + throw Error::Overflow("batch_io", "row group exceeds supported size", path_.string()); + } + + RowGroupMetadata group; + group.row_count = row_count; + group.columns.reserve(batch.ColumnsCount()); + + for (const auto& column : batch.GetColumns()) { + const uint64_t offset = TellWrite(path_, out_); + column->WriteTo(out_); + const uint64_t end = TellWrite(path_, out_); + group.columns.push_back(ColumnChunkMetadata{offset, end - offset}); + } + + metadata_.row_groups.push_back(std::move(group)); +} + +void ColumnarBatchWriter::Flush() { FlushWrite(path_, out_); } + +void ColumnarBatchWriter::Finalize() && { + if (finalized_) { + throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); + } + + const uint64_t metadata_start = TellWrite(path_, out_); + WriteMetadata(out_, metadata_); + const uint64_t metadata_end = TellWrite(path_, out_); + const uint64_t metadata_size = metadata_end - metadata_start; + + WriteStream(out_, metadata_size); + WriteBytes(out_, ColumnarMagic); + + FlushWrite(path_, out_); + finalized_ = true; +} diff --git a/src/io/csv_batch_io.cpp b/src/io/csv_batch_io.cpp new file mode 100644 index 0000000..cf64ab2 --- /dev/null +++ b/src/io/csv_batch_io.cpp @@ -0,0 +1,163 @@ +#include "io/csv_batch_io.h" + +#include +#include +#include +#include + +#include "support/error.h" + +static uint64_t AddChecked(const uint64_t current, const uint64_t add) { + if (add > std::numeric_limits::max() - current) { + throw Error::Overflow("batch_io", "batch byte size overflow"); + } + return current + add; +} + +static uint64_t EstimateRowBytes(const std::vector>& columns, + const std::vector& row) { + uint64_t bytes = 0; + for (size_t i = 0; i < columns.size(); ++i) { + bytes = AddChecked(bytes, columns[i]->EstimateSizeFromString(row[i])); + } + return bytes; +} + +static void ValidateSizing(const BatchSizing& sizing) { + if (sizing.max_rows && *sizing.max_rows == 0) { + throw Error::InvalidArgument("batch_io", "max rows must be > 0"); + } + if (sizing.max_values && *sizing.max_values == 0) { + throw Error::InvalidArgument("batch_io", "max values must be > 0"); + } + if (sizing.max_bytes && *sizing.max_bytes == 0) { + throw Error::InvalidArgument("batch_io", "max bytes must be > 0"); + } +} + +static void ReserveForSizing(const Batch& batch, const BatchSizing& sizing, const size_t column_count) { + if (sizing.max_rows) { + batch.Reserve(*sizing.max_rows); + return; + } + if (sizing.max_values && column_count > 0) { + batch.Reserve(std::max(1, *sizing.max_values / column_count)); + } +} + +CsvBatchReader::CsvBatchReader(const std::filesystem::path& path, Schema schema, BatchSizing sizing) + : csv_reader_(path), schema_(std::move(schema)), sizing_(std::move(sizing)) { + if (schema_.columns.empty()) { + throw Error::InvalidArgument("batch_io", "schema has no columns"); + } + ValidateSizing(sizing_); +} + +std::optional CsvBatchReader::ReadNext() { + if (reached_eof_ && !pending_row_) { + return std::nullopt; + } + + Batch batch(schema_); + + const size_t column_count = schema_.columns.size(); + ReserveForSizing(batch, sizing_, column_count); + const auto& columns = batch.GetColumns(); + + std::vector row; + row.reserve(column_count); + + size_t rows = 0; + uint64_t bytes = 0; + + while (true) { + if (pending_row_) { + row = std::move(*pending_row_); + pending_row_.reset(); + } else { + if (!csv_reader_.ReadRow(row)) { + reached_eof_ = true; + break; + } + } + + if (row.size() != column_count) { + throw Error::Mismatch("batch_io", "data csv column count mismatch"); + } + + const size_t next_rows = rows + 1; + uint64_t next_bytes = bytes; + + if (sizing_.max_bytes) { + next_bytes = AddChecked(bytes, EstimateRowBytes(columns, row)); + } + + if (rows > 0 && sizing_.WouldExceed(next_rows, column_count, next_bytes)) { + pending_row_ = std::move(row); + break; + } + + for (size_t col = 0; col < column_count; ++col) { + columns[col]->AppendFromString(row[col]); + } + + rows = next_rows; + bytes = next_bytes; + } + + if (rows == 0 && reached_eof_ && !pending_row_) { + return std::nullopt; + } + + return batch; +} + +CsvBatchWriter::CsvBatchWriter(const std::filesystem::path& path, Schema schema) + : csv_writer_(path), schema_(std::move(schema)) { + if (schema_.columns.empty()) { + throw Error::InvalidArgument("batch_io", "schema has no columns"); + } +} + +void CsvBatchWriter::Write(const Batch& batch) { + batch.Validate(); + if (batch.GetSchema() != schema_) { + throw Error::Mismatch("batch_io", "batch schema mismatch"); + } + + const size_t column_count = schema_.columns.size(); + const size_t row_count = batch.RowsCount(); + + std::vector row(column_count); + + for (size_t row_index = 0; row_index < row_count; ++row_index) { + for (size_t col = 0; col < column_count; ++col) { + row[col] = batch.ColumnAt(col).ValueAsString(row_index); + } + csv_writer_.WriteRow(row); + } +} + +void CsvBatchWriter::Flush() { csv_writer_.Flush(); } + +void AppendBatchRows(const Batch& batch, std::vector>& rows) { + const size_t row_count = batch.RowsCount(); + const size_t column_count = batch.ColumnsCount(); + + for (size_t row = 0; row < row_count; ++row) { + std::vector values; + values.reserve(column_count); + + for (size_t col = 0; col < column_count; ++col) { + values.push_back(batch.ColumnAt(col).ValueAsString(row)); + } + + rows.push_back(std::move(values)); + } +} + +void WriteBatchCsv(const std::filesystem::path& path, const Batch& batch) { + CsvBatchWriter writer(path, batch.GetSchema()); + writer.Write(batch); + writer.Flush(); +} diff --git a/src/io/fileio.cpp b/src/io/fileio.cpp index b332524..35c7a7d 100644 --- a/src/io/fileio.cpp +++ b/src/io/fileio.cpp @@ -1,7 +1,9 @@ #include "io/fileio.h" #include +#include +#include "io/stream.h" #include "support/error.h" std::optional TryGetFileMetadata(const std::filesystem::path& path) { @@ -73,6 +75,28 @@ void SeekInputFile(std::istream& in, const std::filesystem::path& path, const ui } } +InputFile::InputFile(std::filesystem::path path) : path_(std::move(path)), in_(OpenInputFile(path_)) {} + +void InputFile::ReadAt(char* dst, const size_t size, const uint64_t offset) { + SeekInputFile(in_, path_, offset); + try { + ReadBytes(in_, dst, size); + } catch (const Error&) { + throw Error::PathIo("fileio", path_, "read file"); + } +} + +std::string InputFile::ReadStringAt(const uint64_t offset, const size_t size) { + std::string result(size, '\0'); + ReadAt(result.data(), result.size(), offset); + return result; +} + +std::istream& InputFile::StreamAt(const uint64_t offset) { + SeekInputFile(in_, path_, offset); + return in_; +} + std::ofstream OpenOutputFile(const std::filesystem::path& path, const FileOpenMode mode) { std::ofstream file(path, std::ios::binary | (mode == FileOpenMode::Append ? std::ios::app : std::ios::trunc)); if (!file.is_open()) { diff --git a/src/model/schema.cpp b/src/model/schema_csv.cpp similarity index 98% rename from src/model/schema.cpp rename to src/model/schema_csv.cpp index c5344f1..7be6762 100644 --- a/src/model/schema.cpp +++ b/src/model/schema_csv.cpp @@ -1,10 +1,11 @@ -#include "model/schema.h" +#include "model/schema_csv.h" #include +#include +#include #include "io/csv.h" #include "support/error.h" -#include "io/fileio.h" #include "support/parsing.h" static bool CanParseAs(const ColumnType type, const std::string& value) { diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp index f6051f5..8310dc9 100644 --- a/tests/test_columnar.cpp +++ b/tests/test_columnar.cpp @@ -10,6 +10,7 @@ #include "gtest/gtest.h" #include "support/parsing.h" #include "model/schema.h" +#include "model/schema_csv.h" #include "testing/temp_file.h" TEST(columnar, csv_to_columnar_and_back) { diff --git a/tests/test_fileio.cpp b/tests/test_fileio.cpp index 5427f41..31a8cca 100644 --- a/tests/test_fileio.cpp +++ b/tests/test_fileio.cpp @@ -146,6 +146,17 @@ TEST(fileio, stream_seek_and_tell) { EXPECT_EQ(value, payload[2]); } +TEST(fileio, input_file_reads_at_offsets) { + const TempFile temp("fileio_input_file_read_at"); + const std::vector payload = {10, 11, 12, 13, 14}; + + WriteFileBytes(temp.Path(), payload); + + InputFile file(temp.Path()); + EXPECT_EQ(file.ReadAt(3), payload[3]); + EXPECT_EQ(file.ReadStringAt(1, 3), std::string("\x0B\x0C\x0D", 3)); +} + TEST(fileio, stream_open_missing_throws) { const auto missing = UniqueTempPath("fileio_stream_missing"); From 7d19dedd7bfd0b512b96886bacdbc11913b8891b Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sat, 16 May 2026 02:33:22 +0300 Subject: [PATCH 21/29] review: io module done (finally) --- src/io/csv.cpp | 55 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/src/io/csv.cpp b/src/io/csv.cpp index 0291c4a..a12835f 100644 --- a/src/io/csv.cpp +++ b/src/io/csv.cpp @@ -1,10 +1,40 @@ #include "io/csv.h" +#include #include #include "io/fileio.h" #include "support/error.h" +class CsvCharReader { + public: + explicit CsvCharReader(std::istream& in) : buffer_(in.rdbuf()) {} + + public: + std::optional Read() const { + const Traits::int_type next = buffer_->sbumpc(); + if (Traits::eq_int_type(next, Traits::eof())) { + return std::nullopt; + } + return Traits::to_char_type(next); + } + + std::optional Peek() const { + const Traits::int_type next = buffer_->sgetc(); + if (Traits::eq_int_type(next, Traits::eof())) { + return std::nullopt; + } + return Traits::to_char_type(next); + } + + void Discard() const { static_cast(Read()); } + + private: + using Traits = std::streambuf::traits_type; + + std::streambuf* buffer_; +}; + static bool NeedsCsvQuotes(const std::string_view value) { return value.find_first_of(",\"\n\r") != std::string_view::npos; } @@ -39,22 +69,14 @@ bool CsvReader::ReadRow(std::vector& row) const { row.clear(); row.emplace_back(); - auto* buffer = in_->rdbuf(); - using Traits = std::streambuf::traits_type; - /* До этого чтение происходило через istream::get() и peek() - * Traits требуется только как вспомогательный тип для: - * - eof() - * - to_chat_type() - * - eq_int_type() - */ + const CsvCharReader input(*in_); bool in_quotes = false; bool saw_data = false; while (true) { - const Traits::int_type next = buffer->sbumpc(); - - if (Traits::eq_int_type(next, Traits::eof())) { + const std::optional next = input.Read(); + if (!next.has_value()) { if (!saw_data) { return false; } @@ -65,13 +87,12 @@ bool CsvReader::ReadRow(std::vector& row) const { } saw_data = true; - const char ch = Traits::to_char_type(next); + const char ch = *next; if (in_quotes) { if (ch == '"') { - const Traits::int_type peek = buffer->sgetc(); - if (Traits::to_char_type(peek) == '"') { - buffer->sbumpc(); + if (input.Peek() == '"') { + input.Discard(); row.back().push_back('"'); } else { in_quotes = false; @@ -101,8 +122,8 @@ bool CsvReader::ReadRow(std::vector& row) const { } if (ch == '\r') { - if (Traits::to_char_type(buffer->sgetc()) == '\n') { - buffer->sbumpc(); + if (input.Peek() == '\n') { + input.Discard(); } return true; } From 463b614cf340886f07ff61aae277b991f4223bd9 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sat, 16 May 2026 02:39:00 +0300 Subject: [PATCH 22/29] review: executor edits --- include/executor/aggregate_state.h | 4 +-- src/executor/aggregate_states.cpp | 47 ++++++++++++------------------ src/executor/operators.cpp | 8 ++--- 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/include/executor/aggregate_state.h b/include/executor/aggregate_state.h index 2ccb3fe..4149b73 100644 --- a/include/executor/aggregate_state.h +++ b/include/executor/aggregate_state.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include @@ -17,7 +16,8 @@ class AggState { virtual ~AggState() = default; public: - virtual void Consume(std::optional value) = 0; + virtual void ConsumeValue(std::string_view value) = 0; + virtual void ConsumeRow() = 0; virtual std::string Finalize() const = 0; }; diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index d24ce7d..231be1c 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -4,8 +4,8 @@ #include #include -#include "executor/aggregate_state.h" #include "executor/aggregate_function.h" +#include "executor/aggregate_state.h" #include "support/ascii.h" #include "support/error.h" #include "support/int128.h" @@ -91,10 +91,11 @@ static std::string FormatAverage(const Int128 sum, const uint64_t count) { class CountAS final : public AggState { public: - void Consume(const std::optional value) override { + void ConsumeValue(const std::string_view value) override { (void)value; - ++count_; + ConsumeRow(); } + void ConsumeRow() override { ++count_; } std::string Finalize() const override { return std::to_string(count_); } @@ -107,12 +108,8 @@ class SumAS final : public AggState { explicit SumAS(const ColumnType type) : type_(type) {} public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "SUM requires a value"); - } - sum_ += ParseToInt128(type_, *value); - } + void ConsumeValue(const std::string_view value) override { sum_ += ParseToInt128(type_, value); } + void ConsumeRow() override { throw Error::InvalidState("executor", "SUM requires a value"); } std::string Finalize() const override { return Int128ToString(sum_); } @@ -126,13 +123,11 @@ class AvgAS final : public AggState { explicit AvgAS(const ColumnType type) : type_(type) {} public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "AVG requires a value"); - } - sum_ += ParseToInt128(type_, *value); + void ConsumeValue(const std::string_view value) override { + sum_ += ParseToInt128(type_, value); ++count_; } + void ConsumeRow() override { throw Error::InvalidState("executor", "AVG requires a value"); } std::string Finalize() const override { return FormatAverage(sum_, count_); } @@ -147,20 +142,17 @@ class ExtremumAS final : public AggState { ExtremumAS(const ColumnType type, const bool is_min) : type_(type), is_min_(is_min) {} public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "MIN/MAX requires a value"); - } - + void ConsumeValue(const std::string_view value) override { if (!extremum_.has_value()) { - extremum_ = std::string(*value); + extremum_ = std::string(value); return; } - if (ShouldReplaceExtremum(type_, *value, *extremum_, is_min_)) { - extremum_ = std::string(*value); + if (ShouldReplaceExtremum(type_, value, *extremum_, is_min_)) { + extremum_ = std::string(value); } } + void ConsumeRow() override { throw Error::InvalidState("executor", "MIN/MAX requires a value"); } std::string Finalize() const override { return extremum_.value_or(""); } @@ -175,15 +167,12 @@ class DistinctAS final : public AggState { explicit DistinctAS(std::unique_ptr nested) : nested_(std::move(nested)) {} public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "DISTINCT requires a value"); - } - - if (seen_.insert(std::string(*value)).second) { - nested_->Consume(value); + void ConsumeValue(const std::string_view value) override { + if (seen_.insert(std::string(value)).second) { + nested_->ConsumeValue(value); } } + void ConsumeRow() override { throw Error::InvalidState("executor", "DISTINCT requires a value"); } std::string Finalize() const override { return nested_->Finalize(); } diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index d3f9dbf..07a9a33 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -130,9 +130,9 @@ class AggOperator final : public Operator { for (const auto& binding : bindings_) { if (binding.aggregate.argument_kind == AggArgumentKind::Column) { std::string materialized = batch->ColumnAt(binding.aggregate.column_index).ValueAsString(row); - binding.state->Consume(materialized); + binding.state->ConsumeValue(materialized); } else { - binding.state->Consume(std::nullopt); + binding.state->ConsumeRow(); } } } @@ -214,9 +214,9 @@ class GroupAggOperator final : public Operator { const PlannedAgg& aggregate = aggregates_[i]; if (aggregate.argument_kind == AggArgumentKind::Column) { std::string materialized = batch->ColumnAt(aggregate.column_index).ValueAsString(row); - group.states[i]->Consume(materialized); + group.states[i]->ConsumeValue(materialized); } else { - group.states[i]->Consume(std::nullopt); + group.states[i]->ConsumeRow(); } } } From ad44f586bc395344c46a1cddcf90ac3de6b15dde Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sat, 16 May 2026 02:41:07 +0300 Subject: [PATCH 23/29] review: -io suffix --- benchmarks/benchmark_queries.cpp | 2 +- benchmarks/support/query_result_checks.cpp | 2 +- include/io/{batch_io.h => batch.h} | 0 include/io/{columnar_batch_io.h => columnar_batch.h} | 4 ++-- include/io/{csv_batch_io.h => csv_batch.h} | 2 +- include/io/{fileio.h => file.h} | 0 include/testing/executor_test_utils.h | 2 +- src/CMakeLists.txt | 8 ++++---- src/app/main.cpp | 6 +++--- src/convert/csv_columnar.cpp | 8 ++++---- src/executor/operators.cpp | 4 ++-- src/executor/query_planner.cpp | 2 +- src/io/{batch_io.cpp => batch.cpp} | 3 +-- src/io/{columnar_batch_io.cpp => columnar_batch.cpp} | 3 +-- src/io/csv.cpp | 2 +- src/io/{csv_batch_io.cpp => csv_batch.cpp} | 2 +- src/io/{fileio.cpp => file.cpp} | 3 +-- tests/test_batch.cpp | 6 +++--- tests/test_columnar.cpp | 10 +++++----- tests/test_fileio.cpp | 4 ++-- 20 files changed, 35 insertions(+), 38 deletions(-) rename include/io/{batch_io.h => batch.h} (100%) rename include/io/{columnar_batch_io.h => columnar_batch.h} (97%) rename include/io/{csv_batch_io.h => csv_batch.h} (98%) rename include/io/{fileio.h => file.h} (100%) rename src/io/{batch_io.cpp => batch.cpp} (97%) rename src/io/{columnar_batch_io.cpp => columnar_batch.cpp} (99%) rename src/io/{csv_batch_io.cpp => csv_batch.cpp} (99%) rename src/io/{fileio.cpp => file.cpp} (99%) diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index 2d5469f..64cf866 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -6,7 +6,7 @@ #include #include "executor/executor.h" -#include "io/fileio.h" +#include "io/file.h" #include "support/query_result_checks.h" #include "support/query_runners.h" diff --git a/benchmarks/support/query_result_checks.cpp b/benchmarks/support/query_result_checks.cpp index da89f53..3b136bc 100644 --- a/benchmarks/support/query_result_checks.cpp +++ b/benchmarks/support/query_result_checks.cpp @@ -4,7 +4,7 @@ #include #include -#include "io/csv_batch_io.h" +#include "io/csv_batch.h" #include "support/ascii.h" std::string BatchToString(const Batch& batch) { diff --git a/include/io/batch_io.h b/include/io/batch.h similarity index 100% rename from include/io/batch_io.h rename to include/io/batch.h diff --git a/include/io/columnar_batch_io.h b/include/io/columnar_batch.h similarity index 97% rename from include/io/columnar_batch_io.h rename to include/io/columnar_batch.h index 27c3ebe..50771da 100644 --- a/include/io/columnar_batch_io.h +++ b/include/io/columnar_batch.h @@ -3,8 +3,8 @@ #include #include -#include "io/batch_io.h" -#include "io/fileio.h" +#include "io/batch.h" +#include "io/file.h" #include "model/metadata.h" class ColumnarBatchReader final : public BatchReader { diff --git a/include/io/csv_batch_io.h b/include/io/csv_batch.h similarity index 98% rename from include/io/csv_batch_io.h rename to include/io/csv_batch.h index 881bd7c..6514738 100644 --- a/include/io/csv_batch_io.h +++ b/include/io/csv_batch.h @@ -4,7 +4,7 @@ #include #include -#include "io/batch_io.h" +#include "io/batch.h" #include "io/csv.h" class CsvBatchReader final : public BatchReader { diff --git a/include/io/fileio.h b/include/io/file.h similarity index 100% rename from include/io/fileio.h rename to include/io/file.h diff --git a/include/testing/executor_test_utils.h b/include/testing/executor_test_utils.h index bf8c3e3..7a800f8 100644 --- a/include/testing/executor_test_utils.h +++ b/include/testing/executor_test_utils.h @@ -4,7 +4,7 @@ #include #include "gtest/gtest.h" -#include "io/csv_batch_io.h" +#include "io/csv_batch.h" inline std::vector SingleRowValues(const Batch& batch) { EXPECT_EQ(batch.RowsCount(), 1u); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index df4b0f5..7ccaa3d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,8 +11,8 @@ add_library(columnar_engine_core model/column_string.cpp model/column_timestamp.cpp model/metadata.cpp - io/batch_io.cpp - io/fileio.cpp + io/batch.cpp + io/file.cpp io/stream.cpp support/error.cpp support/ascii.cpp @@ -22,14 +22,14 @@ add_library(columnar_engine_core target_include_directories(columnar_engine_core PUBLIC ${COLUMNAR_INCLUDE_DIRS}) add_library(columnar_engine_columnar - io/columnar_batch_io.cpp + io/columnar_batch.cpp ) target_link_libraries(columnar_engine_columnar PUBLIC columnar_engine_core) add_library(columnar_engine_csv io/csv.cpp - io/csv_batch_io.cpp + io/csv_batch.cpp model/schema_csv.cpp ) diff --git a/src/app/main.cpp b/src/app/main.cpp index 721cc3b..9dcbc94 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -3,13 +3,13 @@ #include #include -#include "io/csv_batch_io.h" #include "convert/csv_columnar.h" -#include "support/error.h" #include "executor/executor.h" -#include "io/fileio.h" +#include "io/csv_batch.h" +#include "io/file.h" #include "model/schema.h" #include "model/schema_csv.h" +#include "support/error.h" void ConfigureInferSchemaCommand(argparse::ArgumentParser& command) { command.add_description("Infer a schema from a CSV file."); diff --git a/src/convert/csv_columnar.cpp b/src/convert/csv_columnar.cpp index e119935..f69e27f 100644 --- a/src/convert/csv_columnar.cpp +++ b/src/convert/csv_columnar.cpp @@ -1,9 +1,9 @@ -#include - #include "convert/csv_columnar.h" -#include "io/columnar_batch_io.h" -#include "io/csv_batch_io.h" +#include + +#include "io/columnar_batch.h" +#include "io/csv_batch.h" #include "model/schema_csv.h" #include "support/error.h" diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index 07a9a33..9ff44a9 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -6,8 +6,8 @@ #include "executor/aggregate_state.h" #include "executor/operator.h" #include "executor/operator_utils.h" -#include "io/columnar_batch_io.h" -#include "io/fileio.h" +#include "io/columnar_batch.h" +#include "io/file.h" #include "support/error.h" class ScanOperator final : public Operator { diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index 44865a2..4aa5bde 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -4,7 +4,7 @@ #include "executor/operator_utils.h" #include "executor/planner_utils.h" -#include "io/columnar_batch_io.h" +#include "io/columnar_batch.h" #include "support/ascii.h" #include "support/error.h" diff --git a/src/io/batch_io.cpp b/src/io/batch.cpp similarity index 97% rename from src/io/batch_io.cpp rename to src/io/batch.cpp index f918fba..4a945f1 100644 --- a/src/io/batch_io.cpp +++ b/src/io/batch.cpp @@ -1,8 +1,7 @@ -#include "io/batch_io.h" - #include #include +#include "io/batch.h" #include "support/error.h" static void ValidateOptionalLimit(const std::optional& value, const std::string_view name) { diff --git a/src/io/columnar_batch_io.cpp b/src/io/columnar_batch.cpp similarity index 99% rename from src/io/columnar_batch_io.cpp rename to src/io/columnar_batch.cpp index b8dc931..98ee983 100644 --- a/src/io/columnar_batch_io.cpp +++ b/src/io/columnar_batch.cpp @@ -1,5 +1,3 @@ -#include "io/columnar_batch_io.h" - #include #include #include @@ -7,6 +5,7 @@ #include #include +#include "io/columnar_batch.h" #include "io/stream.h" #include "support/error.h" diff --git a/src/io/csv.cpp b/src/io/csv.cpp index a12835f..0116cbb 100644 --- a/src/io/csv.cpp +++ b/src/io/csv.cpp @@ -3,7 +3,7 @@ #include #include -#include "io/fileio.h" +#include "io/file.h" #include "support/error.h" class CsvCharReader { diff --git a/src/io/csv_batch_io.cpp b/src/io/csv_batch.cpp similarity index 99% rename from src/io/csv_batch_io.cpp rename to src/io/csv_batch.cpp index cf64ab2..5f1f934 100644 --- a/src/io/csv_batch_io.cpp +++ b/src/io/csv_batch.cpp @@ -1,4 +1,4 @@ -#include "io/csv_batch_io.h" +#include "io/csv_batch.h" #include #include diff --git a/src/io/fileio.cpp b/src/io/file.cpp similarity index 99% rename from src/io/fileio.cpp rename to src/io/file.cpp index 35c7a7d..67822c1 100644 --- a/src/io/fileio.cpp +++ b/src/io/file.cpp @@ -1,8 +1,7 @@ -#include "io/fileio.h" - #include #include +#include "io/file.h" #include "io/stream.h" #include "support/error.h" diff --git a/tests/test_batch.cpp b/tests/test_batch.cpp index cfbfb5a..78964e4 100644 --- a/tests/test_batch.cpp +++ b/tests/test_batch.cpp @@ -3,11 +3,11 @@ #include #include -#include "io/columnar_batch_io.h" +#include "gtest/gtest.h" +#include "io/columnar_batch.h" #include "io/csv.h" -#include "io/csv_batch_io.h" +#include "io/csv_batch.h" #include "support/error.h" -#include "gtest/gtest.h" #include "testing/temp_file.h" static_assert(std::is_copy_constructible_v); diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp index 8310dc9..5dd88ed 100644 --- a/tests/test_columnar.cpp +++ b/tests/test_columnar.cpp @@ -2,15 +2,15 @@ #include #include -#include "io/columnar_batch_io.h" #include "convert/csv_columnar.h" -#include "io/csv.h" -#include "support/error.h" -#include "io/fileio.h" #include "gtest/gtest.h" -#include "support/parsing.h" +#include "io/columnar_batch.h" +#include "io/csv.h" +#include "io/file.h" #include "model/schema.h" #include "model/schema_csv.h" +#include "support/error.h" +#include "support/parsing.h" #include "testing/temp_file.h" TEST(columnar, csv_to_columnar_and_back) { diff --git a/tests/test_fileio.cpp b/tests/test_fileio.cpp index 31a8cca..c68acbc 100644 --- a/tests/test_fileio.cpp +++ b/tests/test_fileio.cpp @@ -3,9 +3,9 @@ #include #include -#include "support/error.h" -#include "io/fileio.h" #include "gtest/gtest.h" +#include "io/file.h" +#include "support/error.h" #include "testing/temp_file.h" TEST(fileio, write_and_read_roundtrip) { From 9ffaa70a6803b49370fc988891abb98d971ae164 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sat, 16 May 2026 16:28:33 +0300 Subject: [PATCH 24/29] review: goddamn..... --- CMakeLists.txt | 1 + benchmarks/benchmark_csv_to_columnar.cpp | 2 +- benchmarks/support/query_result_checks.cpp | 2 +- include/{support => common}/ascii.h | 0 include/{support => common}/error.h | 20 +- include/{support => common}/int128.h | 0 include/{support => common}/parsing.h | 12 +- include/executor/aggregate_function.h | 18 +- include/executor/executor.h | 2 +- include/io/file.h | 2 +- include/io/stream.h | 2 +- include/model/batch.h | 11 +- include/model/column.h | 22 +- include/model/column_int128.h | 2 +- include/model/column_string.h | 26 +- include/model/fixed_column.h | 21 +- include/model/schema.h | 6 +- include/model/variant_column.h | 92 ------- include/sql_parser/tokenizer.h | 242 ++++-------------- src/CMakeLists.txt | 6 +- src/app/main.cpp | 2 +- src/{support => common}/ascii.cpp | 2 +- src/{support => common}/error.cpp | 53 ++-- src/{support => common}/parsing.cpp | 208 ++++++++++++---- src/convert/csv_columnar.cpp | 2 +- src/executor/aggregate_states.cpp | 41 +--- src/executor/executor.cpp | 2 +- src/executor/operator_utils.cpp | 6 +- src/executor/operators.cpp | 269 ++++++++++++++------- src/executor/planner_utils.cpp | 12 +- src/executor/query_parser.cpp | 13 +- src/executor/query_planner.cpp | 9 +- src/io/batch.cpp | 8 +- src/io/columnar_batch.cpp | 28 +-- src/io/csv.cpp | 4 +- src/io/csv_batch.cpp | 45 +++- src/io/file.cpp | 4 +- src/io/stream.cpp | 2 +- src/model/batch.cpp | 31 ++- src/model/column.cpp | 4 +- src/model/column_boolean.cpp | 2 +- src/model/column_character.cpp | 2 +- src/model/column_date.cpp | 2 +- src/model/column_int128.cpp | 2 +- src/model/column_int16.cpp | 2 +- src/model/column_int32.cpp | 2 +- src/model/column_int64.cpp | 2 +- src/model/column_string.cpp | 76 +++++- src/model/column_timestamp.cpp | 2 +- src/model/metadata.cpp | 6 +- src/model/schema_csv.cpp | 62 ++--- src/sql_parser/tokenizer_factory.cpp | 6 +- src/sql_parser/tokenizer_tokens.cpp | 42 ---- tests/test_batch.cpp | 24 +- tests/test_columnar.cpp | 6 +- tests/test_columns.cpp | 4 +- tests/test_executor.cpp | 37 +++ tests/test_fileio.cpp | 6 +- tests/test_tokenizer.cpp | 4 +- {include => third-party}/tl/expected.hpp | 0 60 files changed, 793 insertions(+), 730 deletions(-) rename include/{support => common}/ascii.h (100%) rename include/{support => common}/error.h (80%) rename include/{support => common}/int128.h (100%) rename include/{support => common}/parsing.h (57%) delete mode 100644 include/model/variant_column.h rename src/{support => common}/ascii.cpp (95%) rename src/{support => common}/error.cpp (65%) rename src/{support => common}/parsing.cpp (60%) rename {include => third-party}/tl/expected.hpp (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index eba6976..fbd4b6c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ include(cmake/sanitizers.cmake) set(COLUMNAR_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/third-party ) add_compile_options(-Wall -Wextra -Werror -Wpedantic -O2) diff --git a/benchmarks/benchmark_csv_to_columnar.cpp b/benchmarks/benchmark_csv_to_columnar.cpp index db124f5..3db1be1 100644 --- a/benchmarks/benchmark_csv_to_columnar.cpp +++ b/benchmarks/benchmark_csv_to_columnar.cpp @@ -6,7 +6,7 @@ #include "io/csv.h" #include "convert/csv_columnar.h" -#include "support/error.h" +#include "common/error.h" #ifndef COLUMNAR_BENCHMARK_DEFAULT_DATA #define COLUMNAR_BENCHMARK_DEFAULT_DATA "benchmarks/hits_sample.csv" diff --git a/benchmarks/support/query_result_checks.cpp b/benchmarks/support/query_result_checks.cpp index 3b136bc..80a44a3 100644 --- a/benchmarks/support/query_result_checks.cpp +++ b/benchmarks/support/query_result_checks.cpp @@ -5,7 +5,7 @@ #include #include "io/csv_batch.h" -#include "support/ascii.h" +#include "common/ascii.h" std::string BatchToString(const Batch& batch) { std::vector> rows; diff --git a/include/support/ascii.h b/include/common/ascii.h similarity index 100% rename from include/support/ascii.h rename to include/common/ascii.h diff --git a/include/support/error.h b/include/common/error.h similarity index 80% rename from include/support/error.h rename to include/common/error.h index 034fc4f..73783d2 100644 --- a/include/support/error.h +++ b/include/common/error.h @@ -10,25 +10,18 @@ class Error final : public std::exception { public: enum class Code : uint16_t { InvalidArgument = 1, - InvalidData = 2, + MalformedData = 2, NotFound = 3, OutOfRange = 4, - Mismatch = 5, + InconsistentData = 5, Overflow = 6, InvalidState = 7, Unsupported = 8, IoFailure = 9, }; - enum class Type : uint8_t { - Validation, - Data, - State, - Io, - }; - public: - Error(Code code, std::string module, std::string module_instance, Type type, std::string message); + Error(Code code, std::string module, std::string module_instance, std::string message); Error(const Error&) = default; Error(Error&&) noexcept = default; Error& operator=(const Error&) = default; @@ -37,10 +30,10 @@ class Error final : public std::exception { public: static Error InvalidArgument(std::string module, std::string message, std::string module_instance = {}); - static Error InvalidData(std::string module, std::string message, std::string module_instance = {}); + static Error MalformedData(std::string module, std::string message, std::string module_instance = {}); static Error NotFound(std::string module, std::string message, std::string module_instance = {}); static Error OutOfRange(std::string module, std::string message, std::string module_instance = {}); - static Error Mismatch(std::string module, std::string message, std::string module_instance = {}); + static Error InconsistentData(std::string module, std::string message, std::string module_instance = {}); static Error Overflow(std::string module, std::string message, std::string module_instance = {}); static Error InvalidState(std::string module, std::string message, std::string module_instance = {}); static Error Unsupported(std::string module, std::string message, std::string module_instance = {}); @@ -49,13 +42,11 @@ class Error final : public std::exception { const char* what() const noexcept override; Code GetCode() const noexcept; - Type GetType() const noexcept; const std::string& GetModule() const noexcept; const std::string& GetModuleInstance() const noexcept; const std::string& GetMessage() const noexcept; static std::string_view CodeToString(Code code) noexcept; - static std::string_view TypeToString(Type type) noexcept; private: static std::string BuildWhat(std::string_view module, std::string_view message); @@ -64,7 +55,6 @@ class Error final : public std::exception { Code code_; std::string module_; std::string module_instance_; - Type type_; std::string message_; std::string what_; }; diff --git a/include/support/int128.h b/include/common/int128.h similarity index 100% rename from include/support/int128.h rename to include/common/int128.h diff --git a/include/support/parsing.h b/include/common/parsing.h similarity index 57% rename from include/support/parsing.h rename to include/common/parsing.h index 6a4d585..d779adb 100644 --- a/include/support/parsing.h +++ b/include/common/parsing.h @@ -1,11 +1,12 @@ #pragma once #include +#include #include #include #include "model/schema.h" -#include "support/int128.h" +#include "common/int128.h" bool ParseBoolean(const std::string& value); int16_t ParseInt16(const std::string& value); @@ -16,6 +17,15 @@ int32_t ParseDate(const std::string& value); int64_t ParseTimestamp(const std::string& value); char ParseCharacter(const std::string& value); +std::optional TryParseBoolean(std::string_view value); +std::optional TryParseInt16(std::string_view value); +std::optional TryParseInt32(std::string_view value); +std::optional TryParseInt64(std::string_view value); +std::optional TryParseInt128(std::string_view value); +std::optional TryParseDate(std::string_view value); +std::optional TryParseTimestamp(std::string_view value); +std::optional TryParseCharacter(std::string_view value); + std::string BooleanToString(bool value); std::string Int128ToString(Int128 value); std::string DateToString(int32_t value); diff --git a/include/executor/aggregate_function.h b/include/executor/aggregate_function.h index 1d18168..aeecdad 100644 --- a/include/executor/aggregate_function.h +++ b/include/executor/aggregate_function.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -13,10 +14,23 @@ struct PlannedAgg; using AggFactory = std::unique_ptr (*)(const PlannedAgg&); using AggTypeSupport = bool (*)(ColumnType); +enum class AggCallFeature : std::uint8_t { + None = 0, + Distinct = 1, + Star = 2, +}; + +constexpr AggCallFeature operator|(const AggCallFeature lhs, const AggCallFeature rhs) { + return static_cast(static_cast(lhs) | static_cast(rhs)); +} + +constexpr bool HasAggCallFeature(const AggCallFeature features, const AggCallFeature feature) { + return (static_cast(features) & static_cast(feature)); +} + struct AggFuncDefinition { std::string_view canonical_name; - bool distinct = false; - bool star = false; + AggCallFeature call_features = AggCallFeature::None; AggTypeSupport supports_type = nullptr; AggFactory factory = nullptr; }; diff --git a/include/executor/executor.h b/include/executor/executor.h index 5ce4591..ee0f812 100644 --- a/include/executor/executor.h +++ b/include/executor/executor.h @@ -6,7 +6,7 @@ #include #include "model/batch.h" -#include "support/error.h" +#include "common/error.h" #include "tl/expected.hpp" using ExecuteExpected = tl::expected; diff --git a/include/io/file.h b/include/io/file.h index 5394e9f..430eb48 100644 --- a/include/io/file.h +++ b/include/io/file.h @@ -15,7 +15,7 @@ struct FileMetadata { std::filesystem::file_time_type last_write_time; }; -std::optional TryGetFileMetadata(const std::filesystem::path& path); +std::optional GetFileMetadata(const std::filesystem::path& path); bool FileExists(const std::filesystem::path& path); void EnsureParentDirectory(const std::filesystem::path& path); diff --git a/include/io/stream.h b/include/io/stream.h index 04b11ba..a1b2358 100644 --- a/include/io/stream.h +++ b/include/io/stream.h @@ -11,7 +11,7 @@ #include #include -#include "support/error.h" +#include "common/error.h" void ReadBytes(std::istream& in, char* dst, size_t size); void WriteBytes(std::ostream& out, std::string_view bytes); diff --git a/include/model/batch.h b/include/model/batch.h index 1495aa5..7428a8d 100644 --- a/include/model/batch.h +++ b/include/model/batch.h @@ -1,6 +1,9 @@ #pragma once +#include +#include #include +#include #include #include "model/column.h" @@ -24,16 +27,14 @@ class Batch { size_t RowsCount() const; void Reserve(size_t n) const; + void AppendValueFromString(size_t column_index, const std::string& value) const; + void ReadColumnFrom(size_t column_index, std::istream& in, uint32_t row_count, uint64_t size) const; const Column& ColumnAt(size_t i) const; - Column& ColumnAt(size_t i); - - const std::vector>& GetColumns() const { return columns_; } - std::vector>& GetColumns() { return columns_; } void Validate() const; private: Schema schema_; - std::vector> columns_; + std::vector> columns_; }; diff --git a/include/model/column.h b/include/model/column.h index 3a15866..f7e131e 100644 --- a/include/model/column.h +++ b/include/model/column.h @@ -4,7 +4,6 @@ #include #include #include -#include #include "model/schema.h" @@ -21,16 +20,10 @@ class Column { ColumnType Type() const { return type_; } virtual size_t Size() const = 0; - virtual void Reserve(size_t n) = 0; - virtual void Clear() = 0; - - virtual void AppendFromString(const std::string& value) = 0; - virtual uint64_t EstimateSizeFromString(std::string_view value) const = 0; virtual std::string ValueAsString(size_t row) const = 0; virtual std::unique_ptr Clone() const = 0; virtual void WriteTo(std::ostream& out) const = 0; - virtual void ReadFrom(std::istream& in, uint32_t row_count, uint64_t size) = 0; protected: static void CheckRowIndex(const char* module, size_t row, size_t size); @@ -39,4 +32,17 @@ class Column { ColumnType type_; }; -std::unique_ptr CreateColumn(ColumnType type); +class MutableColumn : public Column { + public: + using Column::Column; + ~MutableColumn() override = default; + + public: + virtual void Reserve(size_t n) = 0; + virtual void Clear() = 0; + virtual void AppendFromString(const std::string& value) = 0; + virtual void ReadFrom(std::istream& in, uint32_t row_count, uint64_t size) = 0; + virtual std::unique_ptr CloneMutable() const = 0; +}; + +std::unique_ptr CreateColumn(ColumnType type); diff --git a/include/model/column_int128.h b/include/model/column_int128.h index 282fe15..a900325 100644 --- a/include/model/column_int128.h +++ b/include/model/column_int128.h @@ -1,7 +1,7 @@ #pragma once #include "model/fixed_column.h" -#include "support/int128.h" +#include "common/int128.h" class Int128Column final : public FixedColumn { public: diff --git a/include/model/column_string.h b/include/model/column_string.h index 0d6542a..dc6236a 100644 --- a/include/model/column_string.h +++ b/include/model/column_string.h @@ -1,13 +1,33 @@ #pragma once -#include "model/variant_column.h" +#include +#include -class StringColumn final : public VariantColumn { +#include "model/column.h" + +class StringColumn final : public MutableColumn { public: - using VariantColumn::VariantColumn; + StringColumn(); + StringColumn(const StringColumn&) = default; + StringColumn(StringColumn&&) noexcept = default; + StringColumn& operator=(const StringColumn&) = default; + StringColumn& operator=(StringColumn&&) noexcept = default; + ~StringColumn() override = default; static const char* ModuleName() { return "column_string"; } + size_t Size() const override; + void Reserve(size_t n) override; + void Clear() override; + void AppendFromString(const std::string& value) override; std::string ValueAsString(size_t row) const override; + std::unique_ptr Clone() const override; + std::unique_ptr CloneMutable() const override; + + void WriteTo(std::ostream& out) const override; + void ReadFrom(std::istream& in, uint32_t row_count, uint64_t size) override; + + private: + std::vector values_; }; diff --git a/include/model/fixed_column.h b/include/model/fixed_column.h index 6a82881..affe489 100644 --- a/include/model/fixed_column.h +++ b/include/model/fixed_column.h @@ -4,12 +4,12 @@ #include "io/stream.h" #include "model/column.h" -#include "support/error.h" +#include "common/error.h" -template -class FixedColumn : public Column { +template +class FixedColumn : public MutableColumn { public: - FixedColumn() : Column(TypeValue) {} + FixedColumn() : MutableColumn(TypeValue) {} FixedColumn(const FixedColumn&) = default; FixedColumn(FixedColumn&&) noexcept = default; FixedColumn& operator=(const FixedColumn&) = default; @@ -21,13 +21,12 @@ class FixedColumn : public Column { void Reserve(const size_t n) override { values_.reserve(n); } void Clear() override { values_.clear(); } - uint64_t EstimateSizeFromString(const std::string_view value) const override { - (void)value; - return sizeof(T); + std::unique_ptr Clone() const override { + return std::make_unique(static_cast(*this)); } - std::unique_ptr Clone() const override { - return std::make_unique(static_cast(*this)); + std::unique_ptr CloneMutable() const override { + return std::make_unique(static_cast(*this)); } void WriteTo(std::ostream& out) const override { @@ -37,7 +36,7 @@ class FixedColumn : public Column { void ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) override { const uint64_t expected = static_cast(row_count) * sizeof(T); if (size != expected) { - throw Error::Mismatch(Derived::ModuleName(), "column chunk size mismatch"); + throw Error::InconsistentData(ColumnImpl::ModuleName(), "column chunk size mismatch"); } values_.resize(row_count); ReadBytes(in, reinterpret_cast(values_.data()), values_.size() * sizeof(T)); @@ -47,7 +46,7 @@ class FixedColumn : public Column { void AppendValue(const T value) { values_.push_back(value); } T ValueAt(const size_t row) const { - CheckRowIndex(Derived::ModuleName(), row, values_.size()); + CheckRowIndex(ColumnImpl::ModuleName(), row, values_.size()); return values_[row]; } diff --git a/include/model/schema.h b/include/model/schema.h index 85f3c40..f37530d 100644 --- a/include/model/schema.h +++ b/include/model/schema.h @@ -2,6 +2,7 @@ #include #include +#include #include enum class ColumnType : uint8_t { @@ -18,7 +19,10 @@ enum class ColumnType : uint8_t { struct ColumnSchema { std::string name; - ColumnType type = ColumnType::String; + ColumnType type; + + ColumnSchema(std::string column_name, const ColumnType column_type) + : name(std::move(column_name)), type(column_type) {} bool operator==(const ColumnSchema& other) const = default; }; diff --git a/include/model/variant_column.h b/include/model/variant_column.h deleted file mode 100644 index 7db4599..0000000 --- a/include/model/variant_column.h +++ /dev/null @@ -1,92 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "io/stream.h" -#include "model/column.h" -#include "support/error.h" - -template -class VariantColumn : public Column { - public: - VariantColumn() : Column(TypeValue) {} - VariantColumn(const VariantColumn&) = default; - VariantColumn(VariantColumn&&) noexcept = default; - VariantColumn& operator=(const VariantColumn&) = default; - VariantColumn& operator=(VariantColumn&&) noexcept = default; - ~VariantColumn() override = default; - - public: - size_t Size() const override { return values_.size(); } - void Reserve(const size_t n) override { values_.reserve(n); } - void Clear() override { values_.clear(); } - - uint64_t EstimateSizeFromString(const std::string_view value) const override { return value.size(); } - - std::unique_ptr Clone() const override { - return std::make_unique(static_cast(*this)); - } - - void WriteTo(std::ostream& out) const override { - size_t total_size = 0; - - for (const auto& value : values_) { - if (value.size() > std::numeric_limits::max()) { - throw Error::Overflow(Derived::ModuleName(), "value exceeds supported size"); - } - total_size += sizeof(uint32_t) + value.size(); - } - - std::string buffer(total_size, '\0'); - char* dst = buffer.data(); - - for (const auto& value : values_) { - const uint32_t length = value.size(); - - std::memcpy(dst, &length, sizeof(length)); - dst += sizeof(length); - - if (!value.empty()) { - std::memcpy(dst, value.data(), value.size()); - dst += value.size(); - } - } - - WriteBytes(out, buffer); - } - - void ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) override { - values_.clear(); - values_.reserve(row_count); - - uint64_t consumed = 0; - - for (uint32_t row_index = 0; row_index < row_count; ++row_index) { - const uint32_t length = ReadStream(in); - consumed += sizeof(uint32_t); - - std::string value(length, '\0'); - ReadBytes(in, value.data(), value.size()); - - consumed += length; - values_.push_back(std::move(value)); - } - - if (consumed != size) { - throw Error::Mismatch(Derived::ModuleName(), "column chunk size mismatch"); - } - } - - protected: - void AppendValue(std::string value) { values_.push_back(std::move(value)); } - - const std::string& ValueAt(const size_t row) const { - CheckRowIndex(Derived::ModuleName(), row, values_.size()); - return values_[row]; - } - - protected: - std::vector values_; -}; diff --git a/include/sql_parser/tokenizer.h b/include/sql_parser/tokenizer.h index 66adcb8..04401d7 100644 --- a/include/sql_parser/tokenizer.h +++ b/include/sql_parser/tokenizer.h @@ -6,7 +6,7 @@ #include #include -#include "support/error.h" +#include "common/error.h" #include "tl/expected.hpp" enum class Tokens { @@ -91,217 +91,57 @@ class IOperatorToken : public Command { ~IOperatorToken() override = default; }; -class TSelectToken : public Command { +template +class TypedToken : public BaseToken { public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TCreateToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; + using BaseToken::BaseToken; -class TFromToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; + Tokens GetType() const noexcept override { return TokenType; } }; -class TLimitToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; +using TSelectToken = TypedToken; +using TCreateToken = TypedToken; +using TFromToken = TypedToken; +using TLimitToken = TypedToken; +using TOrderToken = TypedToken; +using TWhereToken = TypedToken; +using TGroupToken = TypedToken; -class TOrderToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; +using TSumToken = TypedToken; +using TMinToken = TypedToken; +using TMaxToken = TypedToken; +using TCountToken = TypedToken; +using TDistinctToken = TypedToken; +using TAvgToken = TypedToken; +using TLengthToken = TypedToken; +using TPlusToken = TypedToken; +using TMinusToken = TypedToken; -class TWhereToken : public Command { +class TNameToken : public TypedToken { public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TGroupToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TSumToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TMinToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TMaxToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TCountToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TDistinctToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TAvgToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TLengthToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TPlusToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TMinusToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TNameToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; + using TypedToken::TypedToken; std::string_view GetName() const noexcept; }; -class TNumericLiteralToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TStringLiteralToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TAsToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TAndToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TByToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TOpenBracketToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TCommaToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TCloseBracketToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TAsteriskToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TDotToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TSemicolonToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TNotEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TLessToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TLessOrEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TGreaterToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TGreaterOrEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TEndOfInputToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; +using TNumericLiteralToken = TypedToken; +using TStringLiteralToken = TypedToken; +using TAsToken = TypedToken; +using TAndToken = TypedToken; +using TByToken = TypedToken; +using TOpenBracketToken = TypedToken; +using TCommaToken = TypedToken; +using TCloseBracketToken = TypedToken; +using TAsteriskToken = TypedToken; +using TDotToken = TypedToken; +using TSemicolonToken = TypedToken; +using TEqualToken = TypedToken; +using TNotEqualToken = TypedToken; +using TLessToken = TypedToken; +using TLessOrEqualToken = TypedToken; +using TGreaterToken = TypedToken; +using TGreaterOrEqualToken = TypedToken; +using TEndOfInputToken = TypedToken; class Tokenizer { public: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7ccaa3d..7075555 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,9 +14,9 @@ add_library(columnar_engine_core io/batch.cpp io/file.cpp io/stream.cpp - support/error.cpp - support/ascii.cpp - support/parsing.cpp + common/error.cpp + common/ascii.cpp + common/parsing.cpp ) target_include_directories(columnar_engine_core PUBLIC ${COLUMNAR_INCLUDE_DIRS}) diff --git a/src/app/main.cpp b/src/app/main.cpp index 9dcbc94..a37ae79 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -9,7 +9,7 @@ #include "io/file.h" #include "model/schema.h" #include "model/schema_csv.h" -#include "support/error.h" +#include "common/error.h" void ConfigureInferSchemaCommand(argparse::ArgumentParser& command) { command.add_description("Infer a schema from a CSV file."); diff --git a/src/support/ascii.cpp b/src/common/ascii.cpp similarity index 95% rename from src/support/ascii.cpp rename to src/common/ascii.cpp index 779323f..8267ff6 100644 --- a/src/support/ascii.cpp +++ b/src/common/ascii.cpp @@ -1,4 +1,4 @@ -#include "support/ascii.h" +#include "common/ascii.h" #include diff --git a/src/support/error.cpp b/src/common/error.cpp similarity index 65% rename from src/support/error.cpp rename to src/common/error.cpp index 0867b49..4c2d869 100644 --- a/src/support/error.cpp +++ b/src/common/error.cpp @@ -1,4 +1,4 @@ -#include "support/error.h" +#include "common/error.h" #include @@ -6,50 +6,47 @@ std::string Error::BuildWhat(const std::string_view module, const std::string_vi return std::string(module) + ": " + std::string(message); } -Error::Error(const Code code, std::string module, std::string module_instance, const Type type, std::string message) +Error::Error(const Code code, std::string module, std::string module_instance, std::string message) : code_(code), module_(std::move(module)), module_instance_(std::move(module_instance)), - type_(type), message_(std::move(message)), what_(BuildWhat(module_, message_)) {} Error Error::InvalidArgument(std::string module, std::string message, std::string module_instance) { - return Error(Code::InvalidArgument, std::move(module), std::move(module_instance), Type::Validation, - std::move(message)); + return Error(Code::InvalidArgument, std::move(module), std::move(module_instance), std::move(message)); } -Error Error::InvalidData(std::string module, std::string message, std::string module_instance) { - return Error(Code::InvalidData, std::move(module), std::move(module_instance), Type::Data, std::move(message)); +Error Error::MalformedData(std::string module, std::string message, std::string module_instance) { + return Error(Code::MalformedData, std::move(module), std::move(module_instance), std::move(message)); } Error Error::NotFound(std::string module, std::string message, std::string module_instance) { - return Error(Code::NotFound, std::move(module), std::move(module_instance), Type::State, std::move(message)); + return Error(Code::NotFound, std::move(module), std::move(module_instance), std::move(message)); } Error Error::OutOfRange(std::string module, std::string message, std::string module_instance) { - return Error(Code::OutOfRange, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); + return Error(Code::OutOfRange, std::move(module), std::move(module_instance), std::move(message)); } -Error Error::Mismatch(std::string module, std::string message, std::string module_instance) { - return Error(Code::Mismatch, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); +Error Error::InconsistentData(std::string module, std::string message, std::string module_instance) { + return Error(Code::InconsistentData, std::move(module), std::move(module_instance), std::move(message)); } Error Error::Overflow(std::string module, std::string message, std::string module_instance) { - return Error(Code::Overflow, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); + return Error(Code::Overflow, std::move(module), std::move(module_instance), std::move(message)); } Error Error::InvalidState(std::string module, std::string message, std::string module_instance) { - return Error(Code::InvalidState, std::move(module), std::move(module_instance), Type::State, std::move(message)); + return Error(Code::InvalidState, std::move(module), std::move(module_instance), std::move(message)); } Error Error::Unsupported(std::string module, std::string message, std::string module_instance) { - return Error(Code::Unsupported, std::move(module), std::move(module_instance), Type::Validation, - std::move(message)); + return Error(Code::Unsupported, std::move(module), std::move(module_instance), std::move(message)); } Error Error::Io(std::string module, std::string message, std::string module_instance) { - return Error(Code::IoFailure, std::move(module), std::move(module_instance), Type::Io, std::move(message)); + return Error(Code::IoFailure, std::move(module), std::move(module_instance), std::move(message)); } Error Error::PathIo(std::string module, const std::filesystem::path& path, const std::string_view action) { @@ -60,8 +57,6 @@ const char* Error::what() const noexcept { return what_.c_str(); } Error::Code Error::GetCode() const noexcept { return code_; } -Error::Type Error::GetType() const noexcept { return type_; } - const std::string& Error::GetModule() const noexcept { return module_; } const std::string& Error::GetModuleInstance() const noexcept { return module_instance_; } @@ -72,14 +67,14 @@ std::string_view Error::CodeToString(const Code code) noexcept { switch (code) { case Code::InvalidArgument: return "invalid_argument"; - case Code::InvalidData: - return "invalid_data"; + case Code::MalformedData: + return "malformed_data"; case Code::NotFound: return "not_found"; case Code::OutOfRange: return "out_of_range"; - case Code::Mismatch: - return "mismatch"; + case Code::InconsistentData: + return "inconsistent_data"; case Code::Overflow: return "overflow"; case Code::InvalidState: @@ -91,17 +86,3 @@ std::string_view Error::CodeToString(const Code code) noexcept { } return "unknown"; } - -std::string_view Error::TypeToString(const Type type) noexcept { - switch (type) { - case Type::Validation: - return "validation"; - case Type::Data: - return "data"; - case Type::State: - return "state"; - case Type::Io: - return "io"; - } - return "unknown"; -} diff --git a/src/support/parsing.cpp b/src/common/parsing.cpp similarity index 60% rename from src/support/parsing.cpp rename to src/common/parsing.cpp index 5737ef2..00c9362 100644 --- a/src/support/parsing.cpp +++ b/src/common/parsing.cpp @@ -1,64 +1,86 @@ -#include "support/parsing.h" +#include "common/parsing.h" #include #include +#include #include #include +#include #include -#include "support/ascii.h" -#include "support/error.h" +#include "common/ascii.h" +#include "common/error.h" template -T ParseInteger(const std::string& value, const std::string_view type_name) { +bool TryParseInteger(const std::string_view value, T& result) { if (value.empty()) { - throw Error::InvalidData("parsing", "empty value for " + std::string(type_name)); + return false; } - T result = 0; const char* begin = value.data(); const char* end = begin + value.size(); + const auto [ptr, ec] = std::from_chars(begin, end, result); + + return ec == std::errc() && ptr == end; +} + +template +T ParseInteger(const std::string& value, const std::string_view type_name) { + if (value.empty()) { + throw Error::MalformedData("parsing", "empty value for " + std::string(type_name)); + } - if (const auto [ptr, ec] = std::from_chars(begin, end, result); ec != std::errc() || ptr != end) { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); + T result = 0; + if (!TryParseInteger(std::string_view(value), result)) { + throw Error::MalformedData("parsing", "invalid " + std::string(type_name) + " value"); } return result; } -static unsigned ParseUnsignedPart(const std::string_view input, const size_t offset, const size_t length, - const std::string_view type_name) { - unsigned value = 0; +static bool TryParseUnsignedPart(const std::string_view input, const size_t offset, const size_t length, + unsigned& value) { const char* begin = input.data() + offset; const char* end = begin + length; - - if (const auto [ptr, ec] = std::from_chars(begin, end, value); ec != std::errc() || ptr != end) { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); - } - - return value; + const auto [ptr, ec] = std::from_chars(begin, end, value); + return ec == std::errc() && ptr == end; } -static std::chrono::sys_days ParseDatePoint(const std::string_view value, const std::string_view type_name) { +static bool TryParseDatePoint(const std::string_view value, std::chrono::sys_days& result) { if (value.size() != 10 || value[4] != '-' || value[7] != '-') { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); + return false; } - const int year = static_cast(ParseUnsignedPart(value, 0, 4, type_name)); - const unsigned month = ParseUnsignedPart(value, 5, 2, type_name); - const unsigned day = ParseUnsignedPart(value, 8, 2, type_name); + unsigned year = 0; + unsigned month = 0; + unsigned day = 0; + + if (!TryParseUnsignedPart(value, 0, 4, year) || !TryParseUnsignedPart(value, 5, 2, month) || + !TryParseUnsignedPart(value, 8, 2, day)) { + return false; + } const std::chrono::year_month_day ymd{ - std::chrono::year{year}, + std::chrono::year{static_cast(year)}, std::chrono::month{month}, std::chrono::day{day}, }; if (!ymd.ok()) { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); + return false; } - return std::chrono::sys_days{ymd}; + result = std::chrono::sys_days{ymd}; + + return true; +} + +static std::chrono::sys_days ParseDatePoint(const std::string_view value, const std::string_view type_name) { + std::chrono::sys_days result{}; + if (!TryParseDatePoint(value, result)) { + throw Error::MalformedData("parsing", "invalid " + std::string(type_name) + " value"); + } + return result; } static std::string FormatDateParts(const std::chrono::year_month_day ymd) { @@ -68,7 +90,7 @@ static std::string FormatDateParts(const std::chrono::year_month_day ymd) { return out.str(); } -bool ParseBoolean(const std::string& value) { +std::optional TryParseBoolean(const std::string_view value) { const std::string lowered = ToLowerAscii(value); if (lowered == "true" || lowered == "1") { return true; @@ -76,18 +98,50 @@ bool ParseBoolean(const std::string& value) { if (lowered == "false" || lowered == "0") { return false; } - throw Error::InvalidData("parsing", "invalid bool value"); + return std::nullopt; +} + +bool ParseBoolean(const std::string& value) { + const std::optional result = TryParseBoolean(value); + if (!result.has_value()) { + throw Error::MalformedData("parsing", "invalid bool value"); + } + return *result; +} + +std::optional TryParseInt16(const std::string_view value) { + int16_t result = 0; + if (!TryParseInteger(value, result)) { + return std::nullopt; + } + return result; } int16_t ParseInt16(const std::string& value) { return ParseInteger(value, "int16"); } +std::optional TryParseInt32(const std::string_view value) { + int32_t result = 0; + if (!TryParseInteger(value, result)) { + return std::nullopt; + } + return result; +} + int32_t ParseInt32(const std::string& value) { return ParseInteger(value, "int32"); } +std::optional TryParseInt64(const std::string_view value) { + int64_t result = 0; + if (!TryParseInteger(value, result)) { + return std::nullopt; + } + return result; +} + int64_t ParseInt64(const std::string& value) { return ParseInteger(value, "int64"); } -Int128 ParseInt128(const std::string& value) { +std::optional TryParseInt128(const std::string_view value) { if (value.empty()) { - throw Error::InvalidData("parsing", "empty value for int128"); + return std::nullopt; } size_t pos = 0; @@ -99,7 +153,7 @@ Int128 ParseInt128(const std::string& value) { } if (pos == value.size()) { - throw Error::InvalidData("parsing", "invalid int128 value"); + return std::nullopt; } const UInt128 positive_limit = ~UInt128{0} >> 1; @@ -111,13 +165,14 @@ Int128 ParseInt128(const std::string& value) { for (; pos < value.size(); ++pos) { const char ch = value[pos]; if (ch < '0' || ch > '9') { - throw Error::InvalidData("parsing", "invalid int128 value"); + return std::nullopt; } const auto digit = static_cast(ch - '0'); if (magnitude > (limit - digit) / 10) { - throw Error::InvalidData("parsing", "invalid int128 value"); + return std::nullopt; } + magnitude = magnitude * 10 + digit; } @@ -131,6 +186,35 @@ Int128 ParseInt128(const std::string& value) { return -static_cast(magnitude); } +Int128 ParseInt128(const std::string& value) { + if (value.empty()) { + throw Error::MalformedData("parsing", "empty value for int128"); + } + + const std::optional result = TryParseInt128(value); + if (!result.has_value()) { + throw Error::MalformedData("parsing", "invalid int128 value"); + } + + return *result; +} + +std::optional TryParseDate(const std::string_view value) { + std::chrono::sys_days day_point{}; + if (!TryParseDatePoint(value, day_point)) { + return std::nullopt; + } + + const auto days_since_epoch = day_point.time_since_epoch().count(); + + if (days_since_epoch < std::numeric_limits::min() || + days_since_epoch > std::numeric_limits::max()) { + return std::nullopt; + } + + return days_since_epoch; +} + int32_t ParseDate(const std::string& value) { const auto day_point = ParseDatePoint(value, "date"); const auto days_since_epoch = day_point.time_since_epoch().count(); @@ -143,41 +227,53 @@ int32_t ParseDate(const std::string& value) { return days_since_epoch; } -int64_t ParseTimestamp(const std::string& value) { +std::optional TryParseTimestamp(const std::string_view value) { if (value.size() < 19) { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; } - const auto day_point = ParseDatePoint(std::string_view(value).substr(0, 10), "timestamp"); + std::chrono::sys_days day_point{}; + if (!TryParseDatePoint(value.substr(0, 10), day_point)) { + return std::nullopt; + } if (value[10] != ' ' && value[10] != 'T') { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; } if (value[13] != ':' || value[16] != ':') { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; } - const unsigned hour = ParseUnsignedPart(value, 11, 2, "timestamp"); - const unsigned minute = ParseUnsignedPart(value, 14, 2, "timestamp"); - const unsigned second = ParseUnsignedPart(value, 17, 2, "timestamp"); + unsigned hour = 0; + unsigned minute = 0; + unsigned second = 0; + + if (!TryParseUnsignedPart(value, 11, 2, hour) || !TryParseUnsignedPart(value, 14, 2, minute) || + !TryParseUnsignedPart(value, 17, 2, second)) { + return std::nullopt; + } if (hour > 23 || minute > 59 || second > 59) { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; } int64_t microseconds = 0; if (value.size() > 19) { if (value[19] != '.') { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; } const size_t digits = value.size() - 20; if (digits == 0 || digits > 6) { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; + } + + unsigned fractional = 0; + if (!TryParseUnsignedPart(value, 20, digits, fractional)) { + return std::nullopt; } - const auto fractional = ParseUnsignedPart(value, 20, digits, "timestamp"); microseconds = static_cast(fractional); for (size_t i = digits; i < 6; ++i) { microseconds *= 10; @@ -190,13 +286,29 @@ int64_t ParseTimestamp(const std::string& value) { return std::chrono::duration_cast(timestamp.time_since_epoch()).count(); } -char ParseCharacter(const std::string& value) { +int64_t ParseTimestamp(const std::string& value) { + const std::optional result = TryParseTimestamp(value); + if (!result.has_value()) { + throw Error::MalformedData("parsing", "invalid timestamp value"); + } + return *result; +} + +std::optional TryParseCharacter(const std::string_view value) { if (value.size() != 1) { - throw Error::InvalidData("parsing", "invalid char value"); + return std::nullopt; } return value[0]; } +char ParseCharacter(const std::string& value) { + const std::optional result = TryParseCharacter(value); + if (!result.has_value()) { + throw Error::MalformedData("parsing", "invalid char value"); + } + return *result; +} + std::string BooleanToString(const bool value) { return value ? "true" : "false"; } std::string Int128ToString(const Int128 value) { @@ -235,9 +347,9 @@ std::string DateToString(const int32_t value) { } std::string TimestampToString(const int64_t value) { - const std::chrono::sys_time timestamp{std::chrono::microseconds{value}}; + const std::chrono::sys_time timestamp{std::chrono::microseconds{value}}; const auto day_point = std::chrono::floor(timestamp); - const std::chrono::hh_mm_ss tod{timestamp - day_point}; + const std::chrono::hh_mm_ss tod{timestamp - day_point}; std::ostringstream out; out << FormatDateParts(std::chrono::year_month_day{day_point}) << ' ' << std::setfill('0') << std::setw(2) diff --git a/src/convert/csv_columnar.cpp b/src/convert/csv_columnar.cpp index f69e27f..1751337 100644 --- a/src/convert/csv_columnar.cpp +++ b/src/convert/csv_columnar.cpp @@ -2,10 +2,10 @@ #include +#include "common/error.h" #include "io/columnar_batch.h" #include "io/csv_batch.h" #include "model/schema_csv.h" -#include "support/error.h" void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, const std::filesystem::path& output_path, size_t max_rows_per_group) { diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index 231be1c..34dbdc7 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -4,12 +4,12 @@ #include #include +#include "common/ascii.h" +#include "common/error.h" +#include "common/int128.h" +#include "common/parsing.h" #include "executor/aggregate_function.h" #include "executor/aggregate_state.h" -#include "support/ascii.h" -#include "support/error.h" -#include "support/int128.h" -#include "support/parsing.h" static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view candidate, const std::string_view current, const bool is_min) { @@ -91,10 +91,7 @@ static std::string FormatAverage(const Int128 sum, const uint64_t count) { class CountAS final : public AggState { public: - void ConsumeValue(const std::string_view value) override { - (void)value; - ConsumeRow(); - } + void ConsumeValue(const std::string_view) override { ConsumeRow(); } void ConsumeRow() override { ++count_; } std::string Finalize() const override { return std::to_string(count_); } @@ -181,10 +178,7 @@ class DistinctAS final : public AggState { std::unordered_set seen_; }; -static bool SupportsAnyType(const ColumnType type) { - (void)type; - return true; -} +static bool SupportsAnyType(const ColumnType) { return true; } static bool SupportsNumericType(const ColumnType type) { switch (type) { @@ -203,10 +197,7 @@ static bool SupportsNumericType(const ColumnType type) { return false; } -static std::unique_ptr CreateCountState(const PlannedAgg& aggregate) { - (void)aggregate; - return std::make_unique(); -} +static std::unique_ptr CreateCountState(const PlannedAgg&) { return std::make_unique(); } static std::unique_ptr CreateSumState(const PlannedAgg& aggregate) { return std::make_unique(aggregate.input_type); @@ -243,40 +234,31 @@ AggRegistrar::AggRegistrar(const AggFuncDefinition& def) { AggRegistry::Instance [[maybe_unused]] static AggRegistrar register_count({ .canonical_name = "COUNT", - .distinct = true, - .star = true, + .call_features = AggCallFeature::Distinct | AggCallFeature::Star, .supports_type = &SupportsAnyType, .factory = &CreateCountState, }); [[maybe_unused]] static AggRegistrar register_sum({ .canonical_name = "SUM", - .distinct = false, - .star = false, .supports_type = &SupportsNumericType, .factory = &CreateSumState, }); [[maybe_unused]] static AggRegistrar register_avg({ .canonical_name = "AVG", - .distinct = false, - .star = false, .supports_type = &SupportsNumericType, .factory = &CreateAvgState, }); [[maybe_unused]] static AggRegistrar register_min({ .canonical_name = "MIN", - .distinct = false, - .star = false, .supports_type = &SupportsAnyType, .factory = &CreateMinState, }); [[maybe_unused]] static AggRegistrar register_max({ .canonical_name = "MAX", - .distinct = false, - .star = false, .supports_type = &SupportsAnyType, .factory = &CreateMaxState, }); @@ -300,17 +282,20 @@ std::unique_ptr CreateAggState(const PlannedAgg& aggregate) { if (aggregate.function == nullptr || aggregate.function->factory == nullptr) { throw Error::Unsupported("executor", "aggregate is not registered"); } - if (aggregate.distinct && !aggregate.function->distinct) { + if (aggregate.distinct && !HasAggCallFeature(aggregate.function->call_features, AggCallFeature::Distinct)) { throw Error::Unsupported("executor", "DISTINCT is not supported for aggregate '" + std::string(aggregate.function->canonical_name) + "'"); } - if (aggregate.argument_kind == AggArgumentKind::Star && !aggregate.function->star) { + if (aggregate.argument_kind == AggArgumentKind::Star && + !HasAggCallFeature(aggregate.function->call_features, AggCallFeature::Star)) { throw Error::Unsupported( "executor", "aggregate '" + std::string(aggregate.function->canonical_name) + "' does not support '*'"); } + std::unique_ptr state = aggregate.function->factory(aggregate); if (aggregate.distinct) { state = std::make_unique(std::move(state)); } + return state; } diff --git a/src/executor/executor.cpp b/src/executor/executor.cpp index 49e44d7..97a21e6 100644 --- a/src/executor/executor.cpp +++ b/src/executor/executor.cpp @@ -6,7 +6,7 @@ #include "executor/operator.h" #include "executor/query_parser.h" #include "executor/query_planner.h" -#include "support/ascii.h" +#include "common/ascii.h" void Executor::RegisterTable(const std::string& name, std::filesystem::path path) { tables_[ToLowerAscii(name)] = std::move(path); diff --git a/src/executor/operator_utils.cpp b/src/executor/operator_utils.cpp index 4cafce0..9c7278d 100644 --- a/src/executor/operator_utils.cpp +++ b/src/executor/operator_utils.cpp @@ -2,9 +2,9 @@ #include -#include "support/ascii.h" -#include "support/error.h" -#include "support/parsing.h" +#include "common/ascii.h" +#include "common/error.h" +#include "common/parsing.h" template static bool EvaluateComparison(const T left, const T right, const ComparisonKind kind) { diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index 9ff44a9..68dbdd4 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -3,19 +3,19 @@ #include #include +#include "common/error.h" #include "executor/aggregate_state.h" #include "executor/operator.h" #include "executor/operator_utils.h" #include "io/columnar_batch.h" #include "io/file.h" -#include "support/error.h" class ScanOperator final : public Operator { public: ScanOperator(std::filesystem::path path, std::vector projection_indexes) : path_(std::move(path)), input_(path_), metadata_(ColumnarBatchReader(path_).GetMetadata()) { if (metadata_.schema.columns.empty()) { - throw Error::InvalidData("operators", "columnar schema is empty", path_.string()); + throw Error::MalformedData("operators", "columnar schema is empty", path_.string()); } projection_indexes_ = std::move(projection_indexes); @@ -41,7 +41,7 @@ class ScanOperator final : public Operator { for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { const size_t source_index = projection_indexes_[projected_index]; const auto& chunk = row_group.columns[source_index]; - batch.ColumnAt(projected_index).ReadFrom(input_.StreamAt(chunk.offset), row_group.row_count, chunk.size); + batch.ReadColumnFrom(projected_index, input_.StreamAt(chunk.offset), row_group.row_count, chunk.size); } return batch; @@ -76,7 +76,7 @@ class FilterOperator final : public Operator { } for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { - filtered.ColumnAt(column_index).AppendFromString(batch->ColumnAt(column_index).ValueAsString(row)); + filtered.AppendValueFromString(column_index, batch->ColumnAt(column_index).ValueAsString(row)); } } @@ -101,11 +101,77 @@ class FilterOperator final : public Operator { PlannedFilter filter_; }; +class AggInput { + public: + virtual ~AggInput() = default; + + virtual void Consume(const Batch& batch, size_t row, AggState& state) const = 0; +}; + +class ColumnAggInput final : public AggInput { + public: + explicit ColumnAggInput(const size_t column_index) : column_index_(column_index) {} + + void Consume(const Batch& batch, const size_t row, AggState& state) const override { + const std::string materialized = batch.ColumnAt(column_index_).ValueAsString(row); + state.ConsumeValue(materialized); + } + + private: + size_t column_index_ = 0; +}; + +class RowAggInput final : public AggInput { + public: + void Consume(const Batch&, const size_t, AggState& state) const override { state.ConsumeRow(); } +}; + +static std::unique_ptr CreateAggInput(const PlannedAgg& aggregate) { + if (aggregate.argument_kind == AggArgumentKind::Column) { + return std::make_unique(aggregate.column_index); + } + return std::make_unique(); +} + struct AggBinding { PlannedAgg aggregate; + std::unique_ptr input; std::unique_ptr state; }; +struct MaterializedGroupKey { + std::string id; + std::vector values; +}; + +class GroupKeyEncoder { + public: + explicit GroupKeyEncoder(std::vector group_keys) : group_keys_(std::move(group_keys)) {} + + MaterializedGroupKey Encode(const Batch& batch, const size_t row) const { + MaterializedGroupKey key; + key.values.reserve(group_keys_.size()); + + for (const auto& group_key : group_keys_) { + std::string value = batch.ColumnAt(group_key.column_index).ValueAsString(row); + AppendValue(key.id, value); + key.values.push_back(std::move(value)); + } + + return key; + } + + private: + static void AppendValue(std::string& id, const std::string_view value) { + id += std::to_string(value.size()); + id.push_back(':'); + id += value; + } + + private: + std::vector group_keys_; +}; + class AggOperator final : public Operator { public: AggOperator(std::unique_ptr child, const std::vector& aggregates) : child_(std::move(child)) { @@ -113,6 +179,7 @@ class AggOperator final : public Operator { for (const auto& aggregate : aggregates) { bindings_.push_back(AggBinding{ .aggregate = aggregate, + .input = CreateAggInput(aggregate), .state = CreateAggState(aggregate), }); } @@ -128,12 +195,7 @@ class AggOperator final : public Operator { while (auto batch = child_->Next()) { for (size_t row = 0; row < batch->RowsCount(); ++row) { for (const auto& binding : bindings_) { - if (binding.aggregate.argument_kind == AggArgumentKind::Column) { - std::string materialized = batch->ColumnAt(binding.aggregate.column_index).ValueAsString(row); - binding.state->ConsumeValue(materialized); - } else { - binding.state->ConsumeRow(); - } + binding.input->Consume(*batch, row, *binding.state); } } } @@ -142,16 +204,14 @@ class AggOperator final : public Operator { schema.columns.reserve(bindings_.size()); for (const auto& binding : bindings_) { - schema.columns.push_back(ColumnSchema{ - .name = binding.aggregate.output_name, - .type = AggregateOutputType(binding.aggregate), - }); + schema.columns.push_back( + ColumnSchema(binding.aggregate.output_name, AggregateOutputType(binding.aggregate))); } Batch result(schema); for (size_t i = 0; i < bindings_.size(); ++i) { - result.ColumnAt(i).AppendFromString(bindings_[i].state->Finalize()); + result.AppendValueFromString(i, bindings_[i].state->Finalize()); } return result; @@ -166,13 +226,16 @@ class AggOperator final : public Operator { class GroupAggOperator final : public Operator { public: GroupAggOperator(std::unique_ptr child, std::vector group_keys, - std::vector aggregates, std::vector select_items, - std::vector order_by) - : child_(std::move(child)), - group_keys_(std::move(group_keys)), - aggregates_(std::move(aggregates)), - select_items_(std::move(select_items)), - order_by_(std::move(order_by)) {} + const std::vector& aggregates, std::vector select_items) + : child_(std::move(child)), group_key_encoder_(std::move(group_keys)), select_items_(std::move(select_items)) { + bindings_.reserve(aggregates.size()); + for (const auto& aggregate : aggregates) { + bindings_.push_back(GroupAggBinding{ + .aggregate = aggregate, + .input = CreateAggInput(aggregate), + }); + } + } public: std::optional Next() override { @@ -183,26 +246,15 @@ class GroupAggOperator final : public Operator { while (auto batch = child_->Next()) { for (size_t row = 0; row < batch->RowsCount(); ++row) { - std::vector key_values; - key_values.reserve(group_keys_.size()); - - std::string group_id; - for (const auto& group_key : group_keys_) { - std::string value = batch->ColumnAt(group_key.column_index).ValueAsString(row); - group_id += std::to_string(value.size()); - group_id.push_back(':'); - group_id += value; - group_id.push_back('|'); - key_values.push_back(std::move(value)); - } + MaterializedGroupKey key = group_key_encoder_.Encode(*batch, row); size_t group_index = 0; - const auto it = group_indexes_.find(group_id); + const auto it = group_indexes_.find(key.id); if (it == group_indexes_.end()) { group_index = groups_.size(); - group_indexes_.emplace(std::move(group_id), group_index); + group_indexes_.emplace(std::move(key.id), group_index); groups_.push_back(GroupState{ - .key_values = std::move(key_values), + .key_values = std::move(key.values), .states = CreateStates(), }); } else { @@ -210,14 +262,8 @@ class GroupAggOperator final : public Operator { } GroupState& group = groups_[group_index]; - for (size_t i = 0; i < aggregates_.size(); ++i) { - const PlannedAgg& aggregate = aggregates_[i]; - if (aggregate.argument_kind == AggArgumentKind::Column) { - std::string materialized = batch->ColumnAt(aggregate.column_index).ValueAsString(row); - group.states[i]->ConsumeValue(materialized); - } else { - group.states[i]->ConsumeRow(); - } + for (size_t i = 0; i < bindings_.size(); ++i) { + bindings_[i].input->Consume(*batch, row, *group.states[i]); } } } @@ -225,10 +271,7 @@ class GroupAggOperator final : public Operator { Schema schema; schema.columns.reserve(select_items_.size()); for (const auto& item : select_items_) { - schema.columns.push_back(ColumnSchema{ - .name = item.output_name, - .type = item.output_type, - }); + schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); } std::vector> rows; @@ -237,34 +280,10 @@ class GroupAggOperator final : public Operator { rows.push_back(MaterializeRow(group)); } - if (!order_by_.empty()) { - std::ranges::stable_sort( - rows, [&](const std::vector& lhs, const std::vector& rhs) { - for (const auto& order : order_by_) { - const int comparison = CompareValues(order.value_type, lhs[order.result_column_index], - rhs[order.result_column_index]); - if (comparison != 0) { - return order.descending ? comparison > 0 : comparison < 0; - } - } - - for (size_t i = 0; i < select_items_.size(); ++i) { - if (select_items_[i].kind != SelectItemKind::GroupKey) { - continue; - } - const int comparison = CompareValues(select_items_[i].output_type, lhs[i], rhs[i]); - if (comparison != 0) { - return comparison > 0; - } - } - return false; - }); - } - Batch result(schema, rows.size()); for (const auto& row : rows) { for (size_t i = 0; i < row.size(); ++i) { - result.ColumnAt(i).AppendFromString(row[i]); + result.AppendValueFromString(i, row[i]); } } @@ -277,11 +296,16 @@ class GroupAggOperator final : public Operator { std::vector> states; }; + struct GroupAggBinding { + PlannedAgg aggregate; + std::unique_ptr input; + }; + std::vector> CreateStates() const { std::vector> states; - states.reserve(aggregates_.size()); - for (const auto& aggregate : aggregates_) { - states.push_back(CreateAggState(aggregate)); + states.reserve(bindings_.size()); + for (const auto& binding : bindings_) { + states.push_back(CreateAggState(binding.aggregate)); } return states; } @@ -303,15 +327,93 @@ class GroupAggOperator final : public Operator { private: std::unique_ptr child_; - std::vector group_keys_; - std::vector aggregates_; + GroupKeyEncoder group_key_encoder_; + std::vector bindings_; std::vector select_items_; - std::vector order_by_; std::unordered_map group_indexes_; std::vector groups_; bool returned_ = false; }; +class OrderByOperator final : public Operator { + public: + OrderByOperator(std::unique_ptr child, std::vector order_by, + std::vector select_items) + : child_(std::move(child)), order_by_(std::move(order_by)), select_items_(std::move(select_items)) {} + + public: + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + returned_ = true; + + std::optional schema; + std::vector> rows; + + while (auto batch = child_->Next()) { + if (!schema.has_value()) { + schema = batch->GetSchema(); + } + + rows.reserve(rows.size() + batch->RowsCount()); + for (size_t row = 0; row < batch->RowsCount(); ++row) { + std::vector values; + values.reserve(batch->ColumnsCount()); + for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { + values.push_back(batch->ColumnAt(column_index).ValueAsString(row)); + } + rows.push_back(std::move(values)); + } + } + + if (!schema.has_value()) { + return std::nullopt; + } + + SortRows(rows); + + Batch result(*schema, rows.size()); + for (const auto& row : rows) { + for (size_t i = 0; i < row.size(); ++i) { + result.AppendValueFromString(i, row[i]); + } + } + + return result; + } + + private: + void SortRows(std::vector>& rows) const { + std::ranges::stable_sort(rows, [&](const std::vector& lhs, const std::vector& rhs) { + for (const auto& order : order_by_) { + const int comparison = + CompareValues(order.value_type, lhs[order.result_column_index], rhs[order.result_column_index]); + if (comparison != 0) { + return order.descending ? comparison > 0 : comparison < 0; + } + } + + for (size_t i = 0; i < select_items_.size(); ++i) { + if (select_items_[i].kind != SelectItemKind::GroupKey) { + continue; + } + const int comparison = CompareValues(select_items_[i].output_type, lhs[i], rhs[i]); + if (comparison != 0) { + return comparison > 0; + } + } + return false; + }); + } + + private: + std::unique_ptr child_; + std::vector order_by_; + std::vector select_items_; + bool returned_ = false; +}; + class LimitOperator final : public Operator { public: LimitOperator(std::unique_ptr child, const size_t limit) : child_(std::move(child)), remaining_(limit) {} @@ -331,7 +433,7 @@ class LimitOperator final : public Operator { Batch limited(batch->GetSchema(), remaining_); for (size_t row = 0; row < remaining_; ++row) { for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { - limited.ColumnAt(column_index).AppendFromString(batch->ColumnAt(column_index).ValueAsString(row)); + limited.AppendValueFromString(column_index, batch->ColumnAt(column_index).ValueAsString(row)); } } remaining_ = 0; @@ -354,10 +456,13 @@ std::unique_ptr BuildPlan(const PlannedQuery& planned) { } if (!planned.group_keys.empty()) { root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, - planned.select_items, planned.order_by); + planned.select_items); } else { root = std::make_unique(std::move(root), planned.aggregates); } + if (!planned.order_by.empty()) { + root = std::make_unique(std::move(root), planned.order_by, planned.select_items); + } if (planned.limit.has_value()) { root = std::make_unique(std::move(root), *planned.limit); } diff --git a/src/executor/planner_utils.cpp b/src/executor/planner_utils.cpp index 0012cf2..a2358c5 100644 --- a/src/executor/planner_utils.cpp +++ b/src/executor/planner_utils.cpp @@ -1,7 +1,7 @@ #include "executor/planner_utils.h" -#include "support/ascii.h" -#include "support/error.h" +#include "common/ascii.h" +#include "common/error.h" size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { const std::string needle = ToLowerAscii(column_name); @@ -21,14 +21,12 @@ void ValidateColumnQualifier(const Query& query, const ColumnRef& column) { if (column.qualifier.empty()) { return; } - if (SameQualifier(column.qualifier, query.table_name)) { return; } if (!query.table_alias.empty() && SameQualifier(column.qualifier, query.table_alias)) { return; } - throw Error::InvalidArgument("query_planner", "unknown table qualifier '" + column.qualifier + "'"); } @@ -67,7 +65,9 @@ std::string NormalizeLiteral(const QueryLiteral& literal, const ColumnType type) } } -bool SameColumnName(const std::string_view lhs, const std::string_view rhs) { return ToLowerAscii(lhs) == ToLowerAscii(rhs); } +bool SameColumnName(const std::string_view lhs, const std::string_view rhs) { + return ToLowerAscii(lhs) == ToLowerAscii(rhs); +} bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs) { if (!SameColumnName(lhs.name, rhs.name)) { @@ -83,11 +83,9 @@ bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { if (lhs.kind != rhs.kind) { return false; } - if (lhs.kind == SelectItemKind::GroupKey) { return SameColumnRef(lhs.column, rhs.column); } - return ToLowerAscii(lhs.aggregate.output_name) == ToLowerAscii(rhs.aggregate.output_name); } diff --git a/src/executor/query_parser.cpp b/src/executor/query_parser.cpp index f139ba7..fa5a38f 100644 --- a/src/executor/query_parser.cpp +++ b/src/executor/query_parser.cpp @@ -1,12 +1,13 @@ +#include "executor/query_parser.h" + #include #include #include #include -#include "executor/query_parser.h" +#include "common/ascii.h" +#include "common/error.h" #include "sql_parser/tokenizer.h" -#include "support/ascii.h" -#include "support/error.h" class TokenCursor { public: @@ -190,9 +191,11 @@ static std::optional ParseOptionalLimit(TokenCursor& cursor) { const Token& limit_token = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after LIMIT"); size_t limit = 0; + const std::string_view text = limit_token.GetText(); const char* begin = text.data(); const char* end = begin + text.size(); + if (const auto [ptr, ec] = std::from_chars(begin, end, limit); ec != std::errc() || ptr != end) { throw Error::InvalidArgument("query_parser", "invalid LIMIT value"); } @@ -216,8 +219,8 @@ static AggSpec ParseAggregate(TokenCursor& cursor) { } cursor.Consume(Tokens::CloseBracket, "expected ')' after aggregate arguments"); - spec.output_name = FormatAggregateCall(spec.function_name, FormatColumnRef(spec.column), spec.distinct, - spec.argument_kind); + spec.output_name = + FormatAggregateCall(spec.function_name, FormatColumnRef(spec.column), spec.distinct, spec.argument_kind); return spec; } diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index 4aa5bde..261e245 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -5,8 +5,8 @@ #include "executor/operator_utils.h" #include "executor/planner_utils.h" #include "io/columnar_batch.h" -#include "support/ascii.h" -#include "support/error.h" +#include "common/ascii.h" +#include "common/error.h" static std::optional TryFindColumnIndex(const Schema& schema, const std::string_view column_name) { for (size_t i = 0; i < schema.columns.size(); ++i) { @@ -161,11 +161,12 @@ PlannedQuery PlanQuery(const Query& query_ast, const std::unordered_map #include -#include "io/batch.h" -#include "support/error.h" +#include "common/error.h" static void ValidateOptionalLimit(const std::optional& value, const std::string_view name) { if (value && *value == 0) { @@ -13,10 +14,10 @@ static void ValidateOptionalLimit(const std::optional& value, const std: bool BatchSizing::WouldExceed(const size_t next_rows, const size_t column_count, const uint64_t next_bytes) const { ValidateOptionalLimit(max_rows, "max rows"); ValidateOptionalLimit(max_values, "max values"); + if (max_bytes && *max_bytes == 0) { throw Error::InvalidArgument("batch_io", "max bytes must be > 0"); } - if (max_rows && next_rows > *max_rows) { return true; } @@ -28,5 +29,6 @@ bool BatchSizing::WouldExceed(const size_t next_rows, const size_t column_count, if (max_bytes && next_bytes > *max_bytes) { return true; } + return false; } diff --git a/src/io/columnar_batch.cpp b/src/io/columnar_batch.cpp index 98ee983..7457c56 100644 --- a/src/io/columnar_batch.cpp +++ b/src/io/columnar_batch.cpp @@ -7,7 +7,7 @@ #include "io/columnar_batch.h" #include "io/stream.h" -#include "support/error.h" +#include "common/error.h" static constexpr std::string_view ColumnarMagic = "CLMN"; @@ -27,7 +27,7 @@ static void FlushWrite(const std::filesystem::path& path, std::ofstream& out) { } ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::path& path) { - const auto file_metadata = TryGetFileMetadata(path); + const auto file_metadata = GetFileMetadata(path); if (!file_metadata || !file_metadata->is_regular) { throw Error::NotFound("batch_io", "columnar file not found", path.string()); @@ -37,7 +37,7 @@ ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::pa constexpr uint64_t FooterSize = sizeof(uint64_t) + ColumnarMagic.size(); if (file_size < FooterSize) { - throw Error::InvalidData("batch_io", "columnar file is too small", path.string()); + throw Error::MalformedData("batch_io", "columnar file is too small", path.string()); } InputFile file(path); @@ -45,11 +45,11 @@ ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::pa const std::string magic_read = file.ReadStringAt(file_size - ColumnarMagic.size(), ColumnarMagic.size()); if (magic_read != ColumnarMagic) { - throw Error::InvalidData("batch_io", "invalid columnar magic", path.string()); + throw Error::MalformedData("batch_io", "invalid columnar magic", path.string()); } if (metadata_size > file_size - FooterSize) { - throw Error::InvalidData("batch_io", "metadata size exceeds file size", path.string()); + throw Error::MalformedData("batch_io", "metadata size exceeds file size", path.string()); } return ReadMetadata(file.StreamAt(file_size - FooterSize - metadata_size)); @@ -58,7 +58,7 @@ ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::pa ColumnarBatchReader::ColumnarBatchReader(const std::filesystem::path& path) : path_(path), input_(path), metadata_(ReadFileMetadata(path)) { if (metadata_.schema.columns.empty()) { - throw Error::InvalidData("batch_io", "columnar schema is empty", path.string()); + throw Error::MalformedData("batch_io", "columnar schema is empty", path.string()); } } @@ -70,15 +70,13 @@ std::optional ColumnarBatchReader::ReadNext() { const auto& [row_count, row_group_columns] = metadata_.row_groups[next_group_++]; Batch batch(metadata_.schema, row_count); - const auto& columns = batch.GetColumns(); - - if (row_group_columns.size() != columns.size()) { - throw Error::Mismatch("batch_io", "row group column count mismatch", path_.string()); + if (row_group_columns.size() != batch.ColumnsCount()) { + throw Error::InconsistentData("batch_io", "row group column count mismatch", path_.string()); } - for (size_t col = 0; col < columns.size(); ++col) { + for (size_t col = 0; col < batch.ColumnsCount(); ++col) { const auto& [offset, size] = row_group_columns[col]; - columns[col]->ReadFrom(input_.StreamAt(offset), row_count, size); + batch.ReadColumnFrom(col, input_.StreamAt(offset), row_count, size); } return batch; @@ -99,7 +97,7 @@ void ColumnarBatchWriter::Write(const Batch& batch) { batch.Validate(); if (batch.GetSchema() != metadata_.schema) { - throw Error::Mismatch("batch_io", "batch schema mismatch", path_.string()); + throw Error::InconsistentData("batch_io", "batch schema mismatch", path_.string()); } const size_t row_count = batch.RowsCount(); @@ -115,9 +113,9 @@ void ColumnarBatchWriter::Write(const Batch& batch) { group.row_count = row_count; group.columns.reserve(batch.ColumnsCount()); - for (const auto& column : batch.GetColumns()) { + for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { const uint64_t offset = TellWrite(path_, out_); - column->WriteTo(out_); + batch.ColumnAt(column_index).WriteTo(out_); const uint64_t end = TellWrite(path_, out_); group.columns.push_back(ColumnChunkMetadata{offset, end - offset}); } diff --git a/src/io/csv.cpp b/src/io/csv.cpp index 0116cbb..f9e2c3a 100644 --- a/src/io/csv.cpp +++ b/src/io/csv.cpp @@ -4,7 +4,7 @@ #include #include "io/file.h" -#include "support/error.h" +#include "common/error.h" class CsvCharReader { public: @@ -81,7 +81,7 @@ bool CsvReader::ReadRow(std::vector& row) const { return false; } if (in_quotes) { - throw Error::InvalidData("csv", "unexpected EOF inside quoted field"); + throw Error::MalformedData("csv", "unexpected EOF inside quoted field"); } return true; } diff --git a/src/io/csv_batch.cpp b/src/io/csv_batch.cpp index 5f1f934..2ec38d6 100644 --- a/src/io/csv_batch.cpp +++ b/src/io/csv_batch.cpp @@ -1,11 +1,12 @@ #include "io/csv_batch.h" #include +#include #include -#include #include -#include "support/error.h" +#include "common/error.h" +#include "common/int128.h" static uint64_t AddChecked(const uint64_t current, const uint64_t add) { if (add > std::numeric_limits::max() - current) { @@ -14,11 +15,32 @@ static uint64_t AddChecked(const uint64_t current, const uint64_t add) { return current + add; } -static uint64_t EstimateRowBytes(const std::vector>& columns, - const std::vector& row) { +static uint64_t EstimateValueBytes(const ColumnType type, const std::string_view value) { + switch (type) { + case ColumnType::Boolean: + return sizeof(uint8_t); + case ColumnType::Int16: + return sizeof(int16_t); + case ColumnType::Int32: + case ColumnType::Date: + return sizeof(int32_t); + case ColumnType::Int64: + case ColumnType::Timestamp: + return sizeof(int64_t); + case ColumnType::Int128: + return sizeof(Int128); + case ColumnType::Character: + return sizeof(char); + case ColumnType::String: + return value.size(); + } + throw Error::Unsupported("batch_io", "unsupported column type"); +} + +static uint64_t EstimateRowBytes(const Schema& schema, const std::vector& row) { uint64_t bytes = 0; - for (size_t i = 0; i < columns.size(); ++i) { - bytes = AddChecked(bytes, columns[i]->EstimateSizeFromString(row[i])); + for (size_t i = 0; i < schema.columns.size(); ++i) { + bytes = AddChecked(bytes, EstimateValueBytes(schema.columns[i].type, row[i])); } return bytes; } @@ -35,7 +57,7 @@ static void ValidateSizing(const BatchSizing& sizing) { } } -static void ReserveForSizing(const Batch& batch, const BatchSizing& sizing, const size_t column_count) { +static void ReserveForSizing(Batch& batch, const BatchSizing& sizing, const size_t column_count) { if (sizing.max_rows) { batch.Reserve(*sizing.max_rows); return; @@ -62,7 +84,6 @@ std::optional CsvBatchReader::ReadNext() { const size_t column_count = schema_.columns.size(); ReserveForSizing(batch, sizing_, column_count); - const auto& columns = batch.GetColumns(); std::vector row; row.reserve(column_count); @@ -82,14 +103,14 @@ std::optional CsvBatchReader::ReadNext() { } if (row.size() != column_count) { - throw Error::Mismatch("batch_io", "data csv column count mismatch"); + throw Error::InconsistentData("batch_io", "data csv column count mismatch"); } const size_t next_rows = rows + 1; uint64_t next_bytes = bytes; if (sizing_.max_bytes) { - next_bytes = AddChecked(bytes, EstimateRowBytes(columns, row)); + next_bytes = AddChecked(bytes, EstimateRowBytes(schema_, row)); } if (rows > 0 && sizing_.WouldExceed(next_rows, column_count, next_bytes)) { @@ -98,7 +119,7 @@ std::optional CsvBatchReader::ReadNext() { } for (size_t col = 0; col < column_count; ++col) { - columns[col]->AppendFromString(row[col]); + batch.AppendValueFromString(col, row[col]); } rows = next_rows; @@ -122,7 +143,7 @@ CsvBatchWriter::CsvBatchWriter(const std::filesystem::path& path, Schema schema) void CsvBatchWriter::Write(const Batch& batch) { batch.Validate(); if (batch.GetSchema() != schema_) { - throw Error::Mismatch("batch_io", "batch schema mismatch"); + throw Error::InconsistentData("batch_io", "batch schema mismatch"); } const size_t column_count = schema_.columns.size(); diff --git a/src/io/file.cpp b/src/io/file.cpp index 67822c1..a24f34a 100644 --- a/src/io/file.cpp +++ b/src/io/file.cpp @@ -3,9 +3,9 @@ #include "io/file.h" #include "io/stream.h" -#include "support/error.h" +#include "common/error.h" -std::optional TryGetFileMetadata(const std::filesystem::path& path) { +std::optional GetFileMetadata(const std::filesystem::path& path) { std::error_code ec; if (!std::filesystem::exists(path, ec)) { if (ec) { diff --git a/src/io/stream.cpp b/src/io/stream.cpp index 41d85f2..b01fb4a 100644 --- a/src/io/stream.cpp +++ b/src/io/stream.cpp @@ -1,6 +1,6 @@ #include "io/stream.h" -#include "support/error.h" +#include "common/error.h" void ReadBytes(std::istream& in, char* dst, const size_t size) { if (size == 0) { diff --git a/src/model/batch.cpp b/src/model/batch.cpp index 09fe5ca..e282fd0 100644 --- a/src/model/batch.cpp +++ b/src/model/batch.cpp @@ -2,13 +2,14 @@ #include -#include "support/error.h" +#include "common/error.h" -static std::vector> CloneColumns(const std::vector>& columns) { - std::vector> clones; +static std::vector> CloneColumns( + const std::vector>& columns) { + std::vector> clones; clones.reserve(columns.size()); for (const auto& column : columns) { - clones.push_back(column->Clone()); + clones.push_back(column->CloneMutable()); } return clones; } @@ -51,14 +52,22 @@ void Batch::Reserve(const size_t n) const { } } -const Column& Batch::ColumnAt(const size_t i) const { - if (i >= columns_.size()) { +void Batch::AppendValueFromString(const size_t column_index, const std::string& value) const { + if (column_index >= columns_.size()) { throw Error::OutOfRange("batch", "column index out of range"); } - return *columns_[i]; + columns_[column_index]->AppendFromString(value); +} + +void Batch::ReadColumnFrom(const size_t column_index, std::istream& in, const uint32_t row_count, + const uint64_t size) const { + if (column_index >= columns_.size()) { + throw Error::OutOfRange("batch", "column index out of range"); + } + columns_[column_index]->ReadFrom(in, row_count, size); } -Column& Batch::ColumnAt(const size_t i) { +const Column& Batch::ColumnAt(const size_t i) const { if (i >= columns_.size()) { throw Error::OutOfRange("batch", "column index out of range"); } @@ -67,7 +76,7 @@ Column& Batch::ColumnAt(const size_t i) { void Batch::Validate() const { if (schema_.columns.size() != columns_.size()) { - throw Error::Mismatch("batch", "column count mismatch"); + throw Error::InconsistentData("batch", "column count mismatch"); } if (columns_.empty()) { @@ -77,10 +86,10 @@ void Batch::Validate() const { const size_t rows = columns_.front()->Size(); for (size_t i = 0; i < columns_.size(); ++i) { if (columns_[i]->Type() != schema_.columns[i].type) { - throw Error::Mismatch("batch", "column type mismatch"); + throw Error::InconsistentData("batch", "column type mismatch"); } if (columns_[i]->Size() != rows) { - throw Error::Mismatch("batch", "column row count mismatch"); + throw Error::InconsistentData("batch", "column row count mismatch"); } } } diff --git a/src/model/column.cpp b/src/model/column.cpp index ab11742..2218767 100644 --- a/src/model/column.cpp +++ b/src/model/column.cpp @@ -11,9 +11,9 @@ #include "model/column_int64.h" #include "model/column_string.h" #include "model/column_timestamp.h" -#include "support/error.h" +#include "common/error.h" -std::unique_ptr CreateColumn(const ColumnType type) { +std::unique_ptr CreateColumn(const ColumnType type) { switch (type) { case ColumnType::Int64: return std::make_unique(); diff --git a/src/model/column_boolean.cpp b/src/model/column_boolean.cpp index c58ee59..6146b25 100644 --- a/src/model/column_boolean.cpp +++ b/src/model/column_boolean.cpp @@ -1,6 +1,6 @@ #include "model/column_boolean.h" -#include "support/parsing.h" +#include "common/parsing.h" void BooleanColumn::AppendFromString(const std::string& value) { AppendValue(ParseBoolean(value) ? 1 : 0); } diff --git a/src/model/column_character.cpp b/src/model/column_character.cpp index e7312e4..447a5e6 100644 --- a/src/model/column_character.cpp +++ b/src/model/column_character.cpp @@ -1,6 +1,6 @@ #include "model/column_character.h" -#include "support/parsing.h" +#include "common/parsing.h" void CharacterColumn::AppendFromString(const std::string& value) { AppendValue(ParseCharacter(value)); } diff --git a/src/model/column_date.cpp b/src/model/column_date.cpp index e9cc250..3269575 100644 --- a/src/model/column_date.cpp +++ b/src/model/column_date.cpp @@ -1,6 +1,6 @@ #include "model/column_date.h" -#include "support/parsing.h" +#include "common/parsing.h" void DateColumn::AppendFromString(const std::string& value) { AppendValue(ParseDate(value)); } diff --git a/src/model/column_int128.cpp b/src/model/column_int128.cpp index 2102e6f..752a213 100644 --- a/src/model/column_int128.cpp +++ b/src/model/column_int128.cpp @@ -1,6 +1,6 @@ #include "model/column_int128.h" -#include "support/parsing.h" +#include "common/parsing.h" void Int128Column::AppendFromString(const std::string& value) { AppendValue(ParseInt128(value)); } diff --git a/src/model/column_int16.cpp b/src/model/column_int16.cpp index 95c6cd5..7a01fc5 100644 --- a/src/model/column_int16.cpp +++ b/src/model/column_int16.cpp @@ -1,6 +1,6 @@ #include "model/column_int16.h" -#include "support/parsing.h" +#include "common/parsing.h" void Int16Column::AppendFromString(const std::string& value) { AppendValue(ParseInt16(value)); } diff --git a/src/model/column_int32.cpp b/src/model/column_int32.cpp index d0d3639..447708f 100644 --- a/src/model/column_int32.cpp +++ b/src/model/column_int32.cpp @@ -1,6 +1,6 @@ #include "model/column_int32.h" -#include "support/parsing.h" +#include "common/parsing.h" void Int32Column::AppendFromString(const std::string& value) { AppendValue(ParseInt32(value)); } diff --git a/src/model/column_int64.cpp b/src/model/column_int64.cpp index df9be61..911a196 100644 --- a/src/model/column_int64.cpp +++ b/src/model/column_int64.cpp @@ -1,6 +1,6 @@ #include "model/column_int64.h" -#include "support/parsing.h" +#include "common/parsing.h" void Int64Column::AppendFromString(const std::string& value) { AppendValue(ParseInt64(value)); } diff --git a/src/model/column_string.cpp b/src/model/column_string.cpp index 1a94bc2..807798a 100644 --- a/src/model/column_string.cpp +++ b/src/model/column_string.cpp @@ -1,5 +1,77 @@ #include "model/column_string.h" -void StringColumn::AppendFromString(const std::string& value) { AppendValue(value); } +#include +#include +#include -std::string StringColumn::ValueAsString(const size_t row) const { return ValueAt(row); } +#include "io/stream.h" +#include "common/error.h" + +StringColumn::StringColumn() : MutableColumn(ColumnType::String) {} + +size_t StringColumn::Size() const { return values_.size(); } + +void StringColumn::Reserve(const size_t n) { values_.reserve(n); } + +void StringColumn::Clear() { values_.clear(); } + +void StringColumn::AppendFromString(const std::string& value) { values_.push_back(value); } + +std::string StringColumn::ValueAsString(const size_t row) const { + CheckRowIndex(ModuleName(), row, values_.size()); + return values_[row]; +} + +std::unique_ptr StringColumn::Clone() const { return std::make_unique(*this); } + +std::unique_ptr StringColumn::CloneMutable() const { return std::make_unique(*this); } + +void StringColumn::WriteTo(std::ostream& out) const { + size_t total_size = 0; + + for (const auto& value : values_) { + if (value.size() > std::numeric_limits::max()) { + throw Error::Overflow(ModuleName(), "value exceeds supported size"); + } + total_size += sizeof(uint32_t) + value.size(); + } + + std::string buffer(total_size, '\0'); + char* dst = buffer.data(); + + for (const auto& value : values_) { + const uint32_t length = value.size(); + + std::memcpy(dst, &length, sizeof(length)); + dst += sizeof(length); + + if (!value.empty()) { + std::memcpy(dst, value.data(), value.size()); + dst += value.size(); + } + } + + WriteBytes(out, buffer); +} + +void StringColumn::ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) { + values_.clear(); + values_.reserve(row_count); + + uint64_t consumed = 0; + + for (uint32_t row_index = 0; row_index < row_count; ++row_index) { + const uint32_t length = ReadStream(in); + consumed += sizeof(uint32_t); + + std::string value(length, '\0'); + ReadBytes(in, value.data(), value.size()); + + consumed += length; + values_.push_back(std::move(value)); + } + + if (consumed != size) { + throw Error::InconsistentData(ModuleName(), "column chunk size mismatch"); + } +} diff --git a/src/model/column_timestamp.cpp b/src/model/column_timestamp.cpp index c69a98a..d320957 100644 --- a/src/model/column_timestamp.cpp +++ b/src/model/column_timestamp.cpp @@ -1,6 +1,6 @@ #include "model/column_timestamp.h" -#include "support/parsing.h" +#include "common/parsing.h" void TimestampColumn::AppendFromString(const std::string& value) { AppendValue(ParseTimestamp(value)); } diff --git a/src/model/metadata.cpp b/src/model/metadata.cpp index 17e1209..ed90c0d 100644 --- a/src/model/metadata.cpp +++ b/src/model/metadata.cpp @@ -3,7 +3,7 @@ #include #include -#include "support/error.h" +#include "common/error.h" #include "io/stream.h" static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { @@ -20,7 +20,7 @@ static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { return static_cast(type_byte); } - throw Error::InvalidData("columnar", "unknown column type in metadata"); + throw Error::MalformedData("columnar", "unknown column type in metadata"); } ColumnarMetadata ReadMetadata(std::istream& in) { @@ -49,7 +49,7 @@ ColumnarMetadata ReadMetadata(std::istream& in) { const uint32_t columns_in_group = ReadStream(in); if (columns_in_group != column_count) { - throw Error::Mismatch("columnar", "row group column count mismatch"); + throw Error::InconsistentData("columnar", "row group column count mismatch"); } group.columns.reserve(columns_in_group); diff --git a/src/model/schema_csv.cpp b/src/model/schema_csv.cpp index 7be6762..42119ff 100644 --- a/src/model/schema_csv.cpp +++ b/src/model/schema_csv.cpp @@ -5,41 +5,29 @@ #include #include "io/csv.h" -#include "support/error.h" -#include "support/parsing.h" +#include "common/error.h" +#include "common/parsing.h" static bool CanParseAs(const ColumnType type, const std::string& value) { - try { - switch (type) { - case ColumnType::Boolean: - static_cast(ParseBoolean(value)); - return true; - case ColumnType::Int16: - static_cast(ParseInt16(value)); - return true; - case ColumnType::Int32: - static_cast(ParseInt32(value)); - return true; - case ColumnType::Int64: - static_cast(ParseInt64(value)); - return true; - case ColumnType::Int128: - static_cast(ParseInt128(value)); - return true; - case ColumnType::Date: - static_cast(ParseDate(value)); - return true; - case ColumnType::Timestamp: - static_cast(ParseTimestamp(value)); - return true; - case ColumnType::Character: - static_cast(ParseCharacter(value)); - return true; - case ColumnType::String: - return true; - } - } catch (const Error&) { - return false; + switch (type) { + case ColumnType::Boolean: + return TryParseBoolean(value).has_value(); + case ColumnType::Int16: + return TryParseInt16(value).has_value(); + case ColumnType::Int32: + return TryParseInt32(value).has_value(); + case ColumnType::Int64: + return TryParseInt64(value).has_value(); + case ColumnType::Int128: + return TryParseInt128(value).has_value(); + case ColumnType::Date: + return TryParseDate(value).has_value(); + case ColumnType::Timestamp: + return TryParseTimestamp(value).has_value(); + case ColumnType::Character: + return TryParseCharacter(value).has_value(); + case ColumnType::String: + return true; } throw Error::Unsupported("schema", "unknown column type"); @@ -81,7 +69,7 @@ Schema ReadSchemaCsv(const std::filesystem::path& path) { row.pop_back(); } if (row.size() != 2) { - throw Error::InvalidData("schema", "schema csv must have 2 columns", path.string()); + throw Error::MalformedData("schema", "schema csv must have 2 columns", path.string()); } schema.columns.push_back(ColumnSchema{ row[0], @@ -90,7 +78,7 @@ Schema ReadSchemaCsv(const std::filesystem::path& path) { } if (schema.columns.empty()) { - throw Error::InvalidData("schema", "schema csv is empty", path.string()); + throw Error::MalformedData("schema", "schema csv is empty", path.string()); } return schema; @@ -106,7 +94,7 @@ Schema InferSchemaCsv(const std::filesystem::path& path) { if (values_by_column.empty()) { values_by_column.resize(row.size()); } else if (row.size() != values_by_column.size()) { - throw Error::InvalidData("schema", "csv rows have inconsistent column count", path.string()); + throw Error::MalformedData("schema", "csv rows have inconsistent column count", path.string()); } for (size_t i = 0; i < row.size(); ++i) { @@ -115,7 +103,7 @@ Schema InferSchemaCsv(const std::filesystem::path& path) { } if (values_by_column.empty()) { - throw Error::InvalidData("schema", "csv is empty", path.string()); + throw Error::MalformedData("schema", "csv is empty", path.string()); } Schema schema; diff --git a/src/sql_parser/tokenizer_factory.cpp b/src/sql_parser/tokenizer_factory.cpp index dd99921..43e0dba 100644 --- a/src/sql_parser/tokenizer_factory.cpp +++ b/src/sql_parser/tokenizer_factory.cpp @@ -4,7 +4,7 @@ #include #include -#include "support/ascii.h" +#include "common/ascii.h" #include "sql_parser/tokenizer.h" static const std::unordered_map kKeywords = { @@ -120,11 +120,11 @@ TokenPtr MakeToken(const Tokens type, std::string text, const size_t offset) { } Expected MakeUnexpectedCharacter(const char ch, const size_t offset) { - return tl::unexpected(Error::InvalidData("tokenizer_factory", "unexpected character '" + DescribeCharacter(ch) + + return tl::unexpected(Error::MalformedData("tokenizer_factory", "unexpected character '" + DescribeCharacter(ch) + "' at position " + std::to_string(offset))); } Expected MakeUnterminatedString(const size_t offset) { return tl::unexpected( - Error::InvalidData("tokenizer_factory", "unterminated string literal at position " + std::to_string(offset))); + Error::MalformedData("tokenizer_factory", "unterminated string literal at position " + std::to_string(offset))); } diff --git a/src/sql_parser/tokenizer_tokens.cpp b/src/sql_parser/tokenizer_tokens.cpp index 0d07844..ee9279d 100644 --- a/src/sql_parser/tokenizer_tokens.cpp +++ b/src/sql_parser/tokenizer_tokens.cpp @@ -88,46 +88,4 @@ void Command::AddArg(TokenPtr arg) { args_.push_back(std::move(arg)); } const std::vector& Command::GetArgs() const noexcept { return args_; } -// alert: не типобезопасная вещь, лучше переписать -#define COLUMNAR_DEFINE_TOKEN_TYPE(token_class, token_value) \ - Tokens token_class::GetType() const noexcept { return Tokens::token_value; } - -COLUMNAR_DEFINE_TOKEN_TYPE(TSelectToken, Select) -COLUMNAR_DEFINE_TOKEN_TYPE(TCreateToken, Create) -COLUMNAR_DEFINE_TOKEN_TYPE(TFromToken, From) -COLUMNAR_DEFINE_TOKEN_TYPE(TLimitToken, Limit) -COLUMNAR_DEFINE_TOKEN_TYPE(TOrderToken, Order) -COLUMNAR_DEFINE_TOKEN_TYPE(TWhereToken, Where) -COLUMNAR_DEFINE_TOKEN_TYPE(TGroupToken, Group) -COLUMNAR_DEFINE_TOKEN_TYPE(TSumToken, Sum) -COLUMNAR_DEFINE_TOKEN_TYPE(TMinToken, Min) -COLUMNAR_DEFINE_TOKEN_TYPE(TMaxToken, Max) -COLUMNAR_DEFINE_TOKEN_TYPE(TCountToken, Count) -COLUMNAR_DEFINE_TOKEN_TYPE(TDistinctToken, Distinct) -COLUMNAR_DEFINE_TOKEN_TYPE(TAvgToken, Avg) -COLUMNAR_DEFINE_TOKEN_TYPE(TLengthToken, Length) -COLUMNAR_DEFINE_TOKEN_TYPE(TPlusToken, Plus) -COLUMNAR_DEFINE_TOKEN_TYPE(TMinusToken, Minus) -COLUMNAR_DEFINE_TOKEN_TYPE(TNameToken, NameToken) -COLUMNAR_DEFINE_TOKEN_TYPE(TNumericLiteralToken, NumericLiteral) -COLUMNAR_DEFINE_TOKEN_TYPE(TStringLiteralToken, StringLiteral) -COLUMNAR_DEFINE_TOKEN_TYPE(TAsToken, As) -COLUMNAR_DEFINE_TOKEN_TYPE(TAndToken, And) -COLUMNAR_DEFINE_TOKEN_TYPE(TByToken, By) -COLUMNAR_DEFINE_TOKEN_TYPE(TOpenBracketToken, OpenBracket) -COLUMNAR_DEFINE_TOKEN_TYPE(TCommaToken, Comma) -COLUMNAR_DEFINE_TOKEN_TYPE(TCloseBracketToken, CloseBracket) -COLUMNAR_DEFINE_TOKEN_TYPE(TAsteriskToken, Asterisk) -COLUMNAR_DEFINE_TOKEN_TYPE(TDotToken, Dot) -COLUMNAR_DEFINE_TOKEN_TYPE(TSemicolonToken, Semicolon) -COLUMNAR_DEFINE_TOKEN_TYPE(TEqualToken, Equal) -COLUMNAR_DEFINE_TOKEN_TYPE(TNotEqualToken, NotEqual) -COLUMNAR_DEFINE_TOKEN_TYPE(TLessToken, Less) -COLUMNAR_DEFINE_TOKEN_TYPE(TLessOrEqualToken, LessOrEqual) -COLUMNAR_DEFINE_TOKEN_TYPE(TGreaterToken, Greater) -COLUMNAR_DEFINE_TOKEN_TYPE(TGreaterOrEqualToken, GreaterOrEqual) -COLUMNAR_DEFINE_TOKEN_TYPE(TEndOfInputToken, EndOfInput) - -#undef COLUMNAR_DEFINE_TOKEN_TYPE - std::string_view TNameToken::GetName() const noexcept { return GetText(); } diff --git a/tests/test_batch.cpp b/tests/test_batch.cpp index 78964e4..eb5f93d 100644 --- a/tests/test_batch.cpp +++ b/tests/test_batch.cpp @@ -7,7 +7,7 @@ #include "io/columnar_batch.h" #include "io/csv.h" #include "io/csv_batch.h" -#include "support/error.h" +#include "common/error.h" #include "testing/temp_file.h" static_assert(std::is_copy_constructible_v); @@ -89,10 +89,10 @@ TEST(batch, write_batch_csv_writes_single_batch) { }; Batch batch(schema); - batch.ColumnAt(0).AppendFromString("1"); - batch.ColumnAt(1).AppendFromString("alpha"); - batch.ColumnAt(0).AppendFromString("2"); - batch.ColumnAt(1).AppendFromString("be,ta"); + batch.AppendValueFromString(0, "1"); + batch.AppendValueFromString(1, "alpha"); + batch.AppendValueFromString(0, "2"); + batch.AppendValueFromString(1, "be,ta"); const TempFile data_out("batch_write_csv"); WriteBatchCsv(data_out.Path(), batch); @@ -150,9 +150,9 @@ TEST(batch, validate_detects_row_count_mismatch) { }; Batch batch(schema); - batch.ColumnAt(0).AppendFromString("1"); - batch.ColumnAt(0).AppendFromString("2"); - batch.ColumnAt(1).AppendFromString("alpha"); + batch.AppendValueFromString(0, "1"); + batch.AppendValueFromString(0, "2"); + batch.AppendValueFromString(1, "alpha"); EXPECT_THROW(batch.Validate(), Error); } @@ -165,12 +165,12 @@ TEST(batch, copy_is_deep) { }; Batch original(schema); - original.ColumnAt(0).AppendFromString("1"); - original.ColumnAt(1).AppendFromString("alpha"); + original.AppendValueFromString(0, "1"); + original.AppendValueFromString(1, "alpha"); Batch copied = original; - copied.ColumnAt(0).AppendFromString("2"); - copied.ColumnAt(1).AppendFromString("beta"); + copied.AppendValueFromString(0, "2"); + copied.AppendValueFromString(1, "beta"); EXPECT_EQ(original.RowsCount(), 1u); EXPECT_EQ(original.ColumnAt(0).ValueAsString(0), "1"); diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp index 5dd88ed..172754b 100644 --- a/tests/test_columnar.cpp +++ b/tests/test_columnar.cpp @@ -9,8 +9,8 @@ #include "io/file.h" #include "model/schema.h" #include "model/schema_csv.h" -#include "support/error.h" -#include "support/parsing.h" +#include "common/error.h" +#include "common/parsing.h" #include "testing/temp_file.h" TEST(columnar, csv_to_columnar_and_back) { @@ -109,7 +109,7 @@ TEST(columnar, metadata_offsets_and_sizes) { } } - const auto file_info = TryGetFileMetadata(columnar_file.Path()); + const auto file_info = GetFileMetadata(columnar_file.Path()); ASSERT_TRUE(file_info.has_value()); EXPECT_TRUE(file_info->is_regular); EXPECT_LT(expected_offset, file_info->size); diff --git a/tests/test_columns.cpp b/tests/test_columns.cpp index 8c32a40..56a895f 100644 --- a/tests/test_columns.cpp +++ b/tests/test_columns.cpp @@ -5,9 +5,9 @@ #include "model/column.h" #include "model/column_int64.h" #include "model/column_string.h" -#include "support/error.h" +#include "common/error.h" #include "gtest/gtest.h" -#include "support/parsing.h" +#include "common/parsing.h" static void ExpectColumnRoundtrip(const ColumnType type, const std::vector& values, const std::vector& expected_values = {}) { diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index 2ce8be1..cec5830 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -69,6 +69,43 @@ TEST(executor, groups_rows_and_orders_by_aggregate_descending) { EXPECT_EQ(actual_rows, expected_rows); } +TEST(executor, encodes_multicolumn_group_keys_with_length_prefix) { + const TempFile schema_file("executor_schema"); + const TempFile data_file("executor_data"); + const TempFile columnar_file("executor_columnar"); + + WriteRows(schema_file.Path(), { + {"KeyLeft", "string"}, + {"KeyRight", "string"}, + {"Value", "int64"}, + }); + WriteRows(data_file.Path(), { + {"ab", "c", "1"}, + {"a", "bc", "2"}, + {"ab", "c", "3"}, + {"a|", "bc", "4"}, + {"a|", "bc", "5"}, + }); + ConvertCsvToColumnar(schema_file.Path(), data_file.Path(), columnar_file.Path(), 2); + + Executor executor; + executor.RegisterTable("events", columnar_file.Path()); + + auto result = executor.Execute("SELECT KeyLeft, KeyRight, COUNT(*) FROM events GROUP BY KeyLeft, KeyRight;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); + + EXPECT_EQ(BatchColumnNames(result.value()), (std::vector{"KeyLeft", "KeyRight", "COUNT(*)"})); + auto actual_rows = BatchRows(result.value()); + auto expected_rows = std::vector>{ + {"a", "bc", "1"}, + {"a|", "bc", "2"}, + {"ab", "c", "2"}, + }; + std::ranges::sort(actual_rows); + std::ranges::sort(expected_rows); + EXPECT_EQ(actual_rows, expected_rows); +} + TEST(executor, supports_from_with_as_alias_and_qualified_columns) { const Batch batch = BuildHitsTable( "SELECT h.AdvEngineID, COUNT(*) FROM hits AS h WHERE h.AdvEngineID <> 0 GROUP BY h.AdvEngineID ORDER BY " diff --git a/tests/test_fileio.cpp b/tests/test_fileio.cpp index c68acbc..d45683e 100644 --- a/tests/test_fileio.cpp +++ b/tests/test_fileio.cpp @@ -5,7 +5,7 @@ #include "gtest/gtest.h" #include "io/file.h" -#include "support/error.h" +#include "common/error.h" #include "testing/temp_file.h" TEST(fileio, write_and_read_roundtrip) { @@ -41,7 +41,7 @@ TEST(fileio, metadata_for_file) { WriteFileBytes(temp.Path(), payload); - const auto metadata = TryGetFileMetadata(temp.Path()); + const auto metadata = GetFileMetadata(temp.Path()); ASSERT_TRUE(metadata.has_value()); EXPECT_TRUE(metadata->is_regular); EXPECT_FALSE(metadata->is_directory); @@ -50,7 +50,7 @@ TEST(fileio, metadata_for_file) { TEST(fileio, metadata_for_missing_file) { const auto missing = UniqueTempPath("fileio_missing"); - const auto metadata = TryGetFileMetadata(missing); + const auto metadata = GetFileMetadata(missing); EXPECT_FALSE(metadata.has_value()); } diff --git a/tests/test_tokenizer.cpp b/tests/test_tokenizer.cpp index ca11a74..0e9d919 100644 --- a/tests/test_tokenizer.cpp +++ b/tests/test_tokenizer.cpp @@ -109,9 +109,9 @@ TEST(sql_tokenizer, tokenizer_returns_tokens_one_by_one) { TEST(sql_tokenizer, rejects_invalid_input) { const auto unterminated = TokenizeSql("SELECT 'oops"); ASSERT_FALSE(unterminated.has_value()); - EXPECT_EQ(unterminated.error().GetCode(), Error::Code::InvalidData); + EXPECT_EQ(unterminated.error().GetCode(), Error::Code::MalformedData); const auto invalid_character = TokenizeSql("SELECT @oops"); ASSERT_FALSE(invalid_character.has_value()); - EXPECT_EQ(invalid_character.error().GetCode(), Error::Code::InvalidData); + EXPECT_EQ(invalid_character.error().GetCode(), Error::Code::MalformedData); } diff --git a/include/tl/expected.hpp b/third-party/tl/expected.hpp similarity index 100% rename from include/tl/expected.hpp rename to third-party/tl/expected.hpp From 3ad4b483ef926eac66dc2c71c734bd43d7ffae73 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sun, 17 May 2026 15:37:29 +0300 Subject: [PATCH 25/29] review: 99% done --- include/common/error.h | 3 - include/executor/aggregate_state.h | 1 - include/executor/executor.h | 3 +- include/executor/operator.h | 1 - include/executor/operator_utils.h | 3 +- include/io/batch.h | 2 - include/io/columnar_batch.h | 3 - include/io/csv.h | 4 - include/io/csv_batch.h | 2 - include/io/file.h | 1 - include/model/batch.h | 1 - include/model/column.h | 2 - include/model/fixed_column.h | 4 +- include/testing/executor_test_utils.h | 5 +- include/testing/temp_file.h | 2 - src/app/main.cpp | 10 ++ src/common/ascii.cpp | 4 + src/common/error.cpp | 1 + src/common/parsing.cpp | 36 +++-- src/convert/csv_columnar.cpp | 2 +- src/executor/aggregate_states.cpp | 17 ++- src/executor/operator_utils.cpp | 41 +++--- src/executor/operators.cpp | 200 +++++++++++++++++++------- src/executor/planner_utils.cpp | 18 ++- src/executor/query_parser.cpp | 25 ++-- src/executor/query_planner.cpp | 52 ++++--- src/io/batch.cpp | 7 +- src/io/columnar_batch.cpp | 35 +++-- src/io/csv.cpp | 13 +- src/io/csv_batch.cpp | 23 +-- src/io/file.cpp | 37 ++--- src/io/stream.cpp | 4 +- src/model/batch.cpp | 15 +- src/model/column.cpp | 3 +- src/model/metadata.cpp | 14 +- src/model/schema_csv.cpp | 17 ++- src/sql_parser/tokenizer.cpp | 6 + src/sql_parser/tokenizer_factory.cpp | 15 +- src/sql_parser/tokenizer_tokens.cpp | 1 + tests/test_executor.cpp | 11 ++ 40 files changed, 425 insertions(+), 219 deletions(-) diff --git a/include/common/error.h b/include/common/error.h index 73783d2..6dbfe34 100644 --- a/include/common/error.h +++ b/include/common/error.h @@ -20,7 +20,6 @@ class Error final : public std::exception { IoFailure = 9, }; - public: Error(Code code, std::string module, std::string module_instance, std::string message); Error(const Error&) = default; Error(Error&&) noexcept = default; @@ -28,7 +27,6 @@ class Error final : public std::exception { Error& operator=(Error&&) noexcept = default; ~Error() override = default; - public: static Error InvalidArgument(std::string module, std::string message, std::string module_instance = {}); static Error MalformedData(std::string module, std::string message, std::string module_instance = {}); static Error NotFound(std::string module, std::string message, std::string module_instance = {}); @@ -51,7 +49,6 @@ class Error final : public std::exception { private: static std::string BuildWhat(std::string_view module, std::string_view message); - private: Code code_; std::string module_; std::string module_instance_; diff --git a/include/executor/aggregate_state.h b/include/executor/aggregate_state.h index 4149b73..c4d22a9 100644 --- a/include/executor/aggregate_state.h +++ b/include/executor/aggregate_state.h @@ -15,7 +15,6 @@ class AggState { AggState& operator=(AggState&&) noexcept = default; virtual ~AggState() = default; - public: virtual void ConsumeValue(std::string_view value) = 0; virtual void ConsumeRow() = 0; virtual std::string Finalize() const = 0; diff --git a/include/executor/executor.h b/include/executor/executor.h index ee0f812..1c54192 100644 --- a/include/executor/executor.h +++ b/include/executor/executor.h @@ -5,8 +5,8 @@ #include #include -#include "model/batch.h" #include "common/error.h" +#include "model/batch.h" #include "tl/expected.hpp" using ExecuteExpected = tl::expected; @@ -15,7 +15,6 @@ class Executor { public: Executor() = default; - public: void RegisterTable(const std::string& name, std::filesystem::path path); ExecuteExpected Execute(std::string_view query) const; diff --git a/include/executor/operator.h b/include/executor/operator.h index 60ae1c3..4906cf1 100644 --- a/include/executor/operator.h +++ b/include/executor/operator.h @@ -15,7 +15,6 @@ class Operator { Operator& operator=(Operator&&) noexcept = default; virtual ~Operator() = default; - public: virtual std::optional Next() = 0; }; diff --git a/include/executor/operator_utils.h b/include/executor/operator_utils.h index 44f9d18..d247c5d 100644 --- a/include/executor/operator_utils.h +++ b/include/executor/operator_utils.h @@ -1,9 +1,10 @@ #pragma once +#include #include #include "executor/query_plan.h" -int CompareValues(ColumnType type, std::string_view lhs, std::string_view rhs); +std::strong_ordering CompareValues(ColumnType type, std::string_view lhs, std::string_view rhs); bool MatchesComparison(ColumnType type, std::string_view lhs, std::string_view rhs, ComparisonKind kind); ColumnType AggregateOutputType(const PlannedAgg& aggregate); diff --git a/include/io/batch.h b/include/io/batch.h index ff8b47b..70e8703 100644 --- a/include/io/batch.h +++ b/include/io/batch.h @@ -21,7 +21,6 @@ class BatchReader { BatchReader& operator=(BatchReader&&) noexcept = default; virtual ~BatchReader() = default; - public: virtual std::optional ReadNext() = 0; }; @@ -34,7 +33,6 @@ class BatchWriter { BatchWriter& operator=(BatchWriter&&) noexcept = default; virtual ~BatchWriter() = default; - public: virtual void Write(const Batch& batch) = 0; virtual void Flush() = 0; }; diff --git a/include/io/columnar_batch.h b/include/io/columnar_batch.h index 50771da..9dfebb0 100644 --- a/include/io/columnar_batch.h +++ b/include/io/columnar_batch.h @@ -16,7 +16,6 @@ class ColumnarBatchReader final : public BatchReader { ColumnarBatchReader& operator=(ColumnarBatchReader&&) noexcept = default; ~ColumnarBatchReader() override = default; - public: std::optional ReadNext() override; const Schema& GetSchema() const { return metadata_.schema; } @@ -25,7 +24,6 @@ class ColumnarBatchReader final : public BatchReader { private: static ColumnarMetadata ReadFileMetadata(const std::filesystem::path& path); - private: std::filesystem::path path_; InputFile input_; @@ -42,7 +40,6 @@ class ColumnarBatchWriter final : public BatchWriter { ColumnarBatchWriter& operator=(ColumnarBatchWriter&&) noexcept = default; ~ColumnarBatchWriter() override = default; - public: void Write(const Batch& batch) override; void Flush() override; diff --git a/include/io/csv.h b/include/io/csv.h index 1601a20..f55cc43 100644 --- a/include/io/csv.h +++ b/include/io/csv.h @@ -16,13 +16,11 @@ class CsvReader { CsvReader& operator=(CsvReader&& other) noexcept; ~CsvReader() = default; - public: bool ReadRow(std::vector& row) const; private: void RebindAfterMove(bool uses_owned_stream, std::istream* source_stream) noexcept; - private: std::ifstream owned_in_; std::istream* in_ = nullptr; }; @@ -37,14 +35,12 @@ class CsvWriter { CsvWriter& operator=(CsvWriter&& other) noexcept; ~CsvWriter() = default; - public: void WriteRow(const std::vector& row) const; void Flush() const; private: void RebindAfterMove(bool uses_owned_stream, std::ostream* source_stream) noexcept; - private: std::ofstream owned_out_; std::ostream* out_ = nullptr; }; diff --git a/include/io/csv_batch.h b/include/io/csv_batch.h index 6514738..97dbab2 100644 --- a/include/io/csv_batch.h +++ b/include/io/csv_batch.h @@ -16,7 +16,6 @@ class CsvBatchReader final : public BatchReader { CsvBatchReader& operator=(CsvBatchReader&&) noexcept = default; ~CsvBatchReader() override = default; - public: std::optional ReadNext() override; const Schema& GetSchema() const { return schema_; } @@ -39,7 +38,6 @@ class CsvBatchWriter final : public BatchWriter { CsvBatchWriter& operator=(CsvBatchWriter&&) noexcept = default; ~CsvBatchWriter() override = default; - public: void Write(const Batch& batch) override; void Flush() override; diff --git a/include/io/file.h b/include/io/file.h index 430eb48..7ab909d 100644 --- a/include/io/file.h +++ b/include/io/file.h @@ -32,7 +32,6 @@ class InputFile { InputFile& operator=(InputFile&&) noexcept = default; ~InputFile() = default; - public: void ReadAt(char* dst, size_t size, uint64_t offset); std::string ReadStringAt(uint64_t offset, size_t size); std::istream& StreamAt(uint64_t offset); diff --git a/include/model/batch.h b/include/model/batch.h index 7428a8d..6d8703b 100644 --- a/include/model/batch.h +++ b/include/model/batch.h @@ -20,7 +20,6 @@ class Batch { Batch& operator=(Batch&& other) noexcept = default; ~Batch() = default; - public: const Schema& GetSchema() const { return schema_; } size_t ColumnsCount() const; diff --git a/include/model/column.h b/include/model/column.h index f7e131e..118c236 100644 --- a/include/model/column.h +++ b/include/model/column.h @@ -16,7 +16,6 @@ class Column { Column& operator=(Column&&) noexcept = default; virtual ~Column() = default; - public: ColumnType Type() const { return type_; } virtual size_t Size() const = 0; @@ -37,7 +36,6 @@ class MutableColumn : public Column { using Column::Column; ~MutableColumn() override = default; - public: virtual void Reserve(size_t n) = 0; virtual void Clear() = 0; virtual void AppendFromString(const std::string& value) = 0; diff --git a/include/model/fixed_column.h b/include/model/fixed_column.h index affe489..63bc657 100644 --- a/include/model/fixed_column.h +++ b/include/model/fixed_column.h @@ -2,9 +2,9 @@ #include +#include "common/error.h" #include "io/stream.h" #include "model/column.h" -#include "common/error.h" template class FixedColumn : public MutableColumn { @@ -16,7 +16,6 @@ class FixedColumn : public MutableColumn { FixedColumn& operator=(FixedColumn&&) noexcept = default; ~FixedColumn() override = default; - public: size_t Size() const override { return values_.size(); } void Reserve(const size_t n) override { values_.reserve(n); } void Clear() override { values_.clear(); } @@ -50,6 +49,5 @@ class FixedColumn : public MutableColumn { return values_[row]; } - protected: std::vector values_; }; diff --git a/include/testing/executor_test_utils.h b/include/testing/executor_test_utils.h index 7a800f8..2aa6ae2 100644 --- a/include/testing/executor_test_utils.h +++ b/include/testing/executor_test_utils.h @@ -3,12 +3,9 @@ #include #include -#include "gtest/gtest.h" #include "io/csv_batch.h" inline std::vector SingleRowValues(const Batch& batch) { - EXPECT_EQ(batch.RowsCount(), 1u); - std::vector> rows; rows.reserve(1); @@ -24,7 +21,9 @@ inline std::vector SingleRowValues(const Batch& batch) { inline std::vector> BatchRows(const Batch& batch) { std::vector> rows; rows.reserve(batch.RowsCount()); + AppendBatchRows(batch, rows); + return rows; } diff --git a/include/testing/temp_file.h b/include/testing/temp_file.h index d729551..2eb9307 100644 --- a/include/testing/temp_file.h +++ b/include/testing/temp_file.h @@ -33,7 +33,6 @@ class TempFile { } ~TempFile() { Remove(); } - public: const std::filesystem::path& Path() const { return path_; } private: @@ -45,6 +44,5 @@ class TempFile { std::filesystem::remove(path_, ec); } - private: std::filesystem::path path_; }; diff --git a/src/app/main.cpp b/src/app/main.cpp index a37ae79..dfc3d62 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -44,27 +44,34 @@ void ConfigureRunQueryCommand(argparse::ArgumentParser& command) { int RunInferSchema(const argparse::ArgumentParser& command) { const auto input_path = std::filesystem::path(command.get("--input")); const auto output_path = std::filesystem::path(command.get("--output")); + EnsureParentDirectory(output_path); WriteSchemaCsv(output_path, InferSchemaCsv(input_path)); + return 0; } int RunConvert(const argparse::ArgumentParser& command) { const auto output_path = std::filesystem::path(command.get("--output")); + EnsureParentDirectory(output_path); ConvertCsvToColumnar(std::filesystem::path(command.get("--schema")), std::filesystem::path(command.get("--input")), output_path, command.get("--row-group-size")); + return 0; } int RunRoundtrip(const argparse::ArgumentParser& command) { const auto schema_output_path = std::filesystem::path(command.get("--schema-output")); const auto csv_output_path = std::filesystem::path(command.get("--csv-output")); + EnsureParentDirectory(schema_output_path); EnsureParentDirectory(csv_output_path); + ConvertColumnarToCsv(std::filesystem::path(command.get("--input")), schema_output_path, csv_output_path); + return 0; } @@ -118,12 +125,15 @@ int main(const int argc, char** argv) { if (program.is_subcommand_used("infer-schema")) { return RunInferSchema(infer_schema_command); } + if (program.is_subcommand_used("convert")) { return RunConvert(convert_command); } + if (program.is_subcommand_used("roundtrip")) { return RunRoundtrip(roundtrip_command); } + if (program.is_subcommand_used("run-query")) { return RunQuery(run_query_command); } diff --git a/src/common/ascii.cpp b/src/common/ascii.cpp index 8267ff6..8a2ae21 100644 --- a/src/common/ascii.cpp +++ b/src/common/ascii.cpp @@ -5,17 +5,21 @@ std::string ToLowerAscii(const std::string_view input) { std::string output; output.reserve(input.size()); + for (const unsigned char ch : input) { output.push_back(static_cast(std::tolower(ch))); } + return output; } std::string ToUpperAscii(const std::string_view input) { std::string output; output.reserve(input.size()); + for (const unsigned char ch : input) { output.push_back(static_cast(std::toupper(ch))); } + return output; } diff --git a/src/common/error.cpp b/src/common/error.cpp index 4c2d869..c4dcbd3 100644 --- a/src/common/error.cpp +++ b/src/common/error.cpp @@ -84,5 +84,6 @@ std::string_view Error::CodeToString(const Code code) noexcept { case Code::IoFailure: return "io_failure"; } + return "unknown"; } diff --git a/src/common/parsing.cpp b/src/common/parsing.cpp index 00c9362..3a83c72 100644 --- a/src/common/parsing.cpp +++ b/src/common/parsing.cpp @@ -27,12 +27,12 @@ bool TryParseInteger(const std::string_view value, T& result) { template T ParseInteger(const std::string& value, const std::string_view type_name) { if (value.empty()) { - throw Error::MalformedData("parsing", "empty value for " + std::string(type_name)); + throw Error::MalformedData("common", "empty value for " + std::string(type_name)); } T result = 0; if (!TryParseInteger(std::string_view(value), result)) { - throw Error::MalformedData("parsing", "invalid " + std::string(type_name) + " value"); + throw Error::MalformedData("common", "invalid " + std::string(type_name) + " value"); } return result; @@ -78,7 +78,7 @@ static bool TryParseDatePoint(const std::string_view value, std::chrono::sys_day static std::chrono::sys_days ParseDatePoint(const std::string_view value, const std::string_view type_name) { std::chrono::sys_days result{}; if (!TryParseDatePoint(value, result)) { - throw Error::MalformedData("parsing", "invalid " + std::string(type_name) + " value"); + throw Error::MalformedData("common", "invalid " + std::string(type_name) + " value"); } return result; } @@ -104,7 +104,7 @@ std::optional TryParseBoolean(const std::string_view value) { bool ParseBoolean(const std::string& value) { const std::optional result = TryParseBoolean(value); if (!result.has_value()) { - throw Error::MalformedData("parsing", "invalid bool value"); + throw Error::MalformedData("common", "invalid bool value"); } return *result; } @@ -179,6 +179,7 @@ std::optional TryParseInt128(const std::string_view value) { if (!negative) { return static_cast(magnitude); } + if (magnitude == negative_limit) { return -static_cast(positive_limit) - 1; } @@ -188,12 +189,12 @@ std::optional TryParseInt128(const std::string_view value) { Int128 ParseInt128(const std::string& value) { if (value.empty()) { - throw Error::MalformedData("parsing", "empty value for int128"); + throw Error::MalformedData("common", "empty value for int128"); } const std::optional result = TryParseInt128(value); if (!result.has_value()) { - throw Error::MalformedData("parsing", "invalid int128 value"); + throw Error::MalformedData("common", "invalid int128 value"); } return *result; @@ -221,7 +222,7 @@ int32_t ParseDate(const std::string& value) { if (days_since_epoch < std::numeric_limits::min() || days_since_epoch > std::numeric_limits::max()) { - throw Error::Overflow("parsing", "date value exceeds supported range"); + throw Error::Overflow("common", "date value exceeds supported range"); } return days_since_epoch; @@ -240,6 +241,7 @@ std::optional TryParseTimestamp(const std::string_view value) { if (value[10] != ' ' && value[10] != 'T') { return std::nullopt; } + if (value[13] != ':' || value[16] != ':') { return std::nullopt; } @@ -289,7 +291,7 @@ std::optional TryParseTimestamp(const std::string_view value) { int64_t ParseTimestamp(const std::string& value) { const std::optional result = TryParseTimestamp(value); if (!result.has_value()) { - throw Error::MalformedData("parsing", "invalid timestamp value"); + throw Error::MalformedData("common", "invalid timestamp value"); } return *result; } @@ -304,7 +306,7 @@ std::optional TryParseCharacter(const std::string_view value) { char ParseCharacter(const std::string& value) { const std::optional result = TryParseCharacter(value); if (!result.has_value()) { - throw Error::MalformedData("parsing", "invalid char value"); + throw Error::MalformedData("common", "invalid char value"); } return *result; } @@ -332,6 +334,7 @@ std::string Int128ToString(const Int128 value) { out.push_back(static_cast('0' + magnitude % 10)); magnitude /= 10; } + if (negative) { out.push_back('-'); } @@ -372,34 +375,44 @@ std::string TimestampToString(const int64_t value) { ColumnType ParseColumnType(const std::string_view input) { const std::string lowered = ToLowerAscii(input); + if (lowered == "bool") { return ColumnType::Boolean; } + if (lowered == "int16") { return ColumnType::Int16; } + if (lowered == "int32") { return ColumnType::Int32; } + if (lowered == "int64") { return ColumnType::Int64; } + if (lowered == "int128") { return ColumnType::Int128; } + if (lowered == "string") { return ColumnType::String; } + if (lowered == "date") { return ColumnType::Date; } + if (lowered == "timestamp") { return ColumnType::Timestamp; } + if (lowered == "char") { return ColumnType::Character; } - throw Error::Unsupported("parsing", "unknown column type"); + + throw Error::Unsupported("common", "unknown column type"); } std::string ColumnTypeToString(const ColumnType type) { @@ -423,5 +436,6 @@ std::string ColumnTypeToString(const ColumnType type) { case ColumnType::Character: return "char"; } - throw Error::Unsupported("parsing", "unknown column type"); + + throw Error::Unsupported("common", "unknown column type"); } diff --git a/src/convert/csv_columnar.cpp b/src/convert/csv_columnar.cpp index 1751337..eff1ace 100644 --- a/src/convert/csv_columnar.cpp +++ b/src/convert/csv_columnar.cpp @@ -10,7 +10,7 @@ void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, const std::filesystem::path& output_path, size_t max_rows_per_group) { if (max_rows_per_group == 0) { - throw Error::InvalidArgument("columnar", "row group size must be > 0"); + throw Error::InvalidArgument("convert", "row group size must be > 0"); } const Schema schema = ReadSchemaCsv(schema_path); diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index 34dbdc7..b8e59b9 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -17,26 +17,31 @@ static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view case ColumnType::Boolean: { const bool left = ParseBoolean(std::string(candidate)); const bool right = ParseBoolean(std::string(current)); + return is_min ? left < right : left > right; } case ColumnType::Int16: { const int16_t left = ParseInt16(std::string(candidate)); const int16_t right = ParseInt16(std::string(current)); + return is_min ? left < right : left > right; } case ColumnType::Int32: { const int32_t left = ParseInt32(std::string(candidate)); const int32_t right = ParseInt32(std::string(current)); + return is_min ? left < right : left > right; } case ColumnType::Int64: { const int64_t left = ParseInt64(std::string(candidate)); const int64_t right = ParseInt64(std::string(current)); + return is_min ? left < right : left > right; } case ColumnType::Int128: { const Int128 left = ParseInt128(std::string(candidate)); const Int128 right = ParseInt128(std::string(current)); + return is_min ? left < right : left > right; } case ColumnType::String: @@ -44,19 +49,23 @@ static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view case ColumnType::Date: { const int32_t left = ParseDate(std::string(candidate)); const int32_t right = ParseDate(std::string(current)); + return is_min ? left < right : left > right; } case ColumnType::Timestamp: { const int64_t left = ParseTimestamp(std::string(candidate)); const int64_t right = ParseTimestamp(std::string(current)); + return is_min ? left < right : left > right; } case ColumnType::Character: { const char left = ParseCharacter(std::string(candidate)); const char right = ParseCharacter(std::string(current)); + return is_min ? left < right : left > right; } } + throw Error::Unsupported("executor", "unsupported column type in comparison"); } @@ -79,6 +88,7 @@ static Int128 ParseToInt128(const ColumnType type, const std::string_view value) case ColumnType::Timestamp: break; } + throw Error::Unsupported("executor", "unsupported numeric conversion"); } @@ -104,7 +114,6 @@ class SumAS final : public AggState { public: explicit SumAS(const ColumnType type) : type_(type) {} - public: void ConsumeValue(const std::string_view value) override { sum_ += ParseToInt128(type_, value); } void ConsumeRow() override { throw Error::InvalidState("executor", "SUM requires a value"); } @@ -119,7 +128,6 @@ class AvgAS final : public AggState { public: explicit AvgAS(const ColumnType type) : type_(type) {} - public: void ConsumeValue(const std::string_view value) override { sum_ += ParseToInt128(type_, value); ++count_; @@ -138,7 +146,6 @@ class ExtremumAS final : public AggState { public: ExtremumAS(const ColumnType type, const bool is_min) : type_(type), is_min_(is_min) {} - public: void ConsumeValue(const std::string_view value) override { if (!extremum_.has_value()) { extremum_ = std::string(value); @@ -163,7 +170,6 @@ class DistinctAS final : public AggState { public: explicit DistinctAS(std::unique_ptr nested) : nested_(std::move(nested)) {} - public: void ConsumeValue(const std::string_view value) override { if (seen_.insert(std::string(value)).second) { nested_->ConsumeValue(value); @@ -194,6 +200,7 @@ static bool SupportsNumericType(const ColumnType type) { case ColumnType::Timestamp: return false; } + return false; } @@ -282,10 +289,12 @@ std::unique_ptr CreateAggState(const PlannedAgg& aggregate) { if (aggregate.function == nullptr || aggregate.function->factory == nullptr) { throw Error::Unsupported("executor", "aggregate is not registered"); } + if (aggregate.distinct && !HasAggCallFeature(aggregate.function->call_features, AggCallFeature::Distinct)) { throw Error::Unsupported("executor", "DISTINCT is not supported for aggregate '" + std::string(aggregate.function->canonical_name) + "'"); } + if (aggregate.argument_kind == AggArgumentKind::Star && !HasAggCallFeature(aggregate.function->call_features, AggCallFeature::Star)) { throw Error::Unsupported( diff --git a/src/executor/operator_utils.cpp b/src/executor/operator_utils.cpp index 9c7278d..57a88b8 100644 --- a/src/executor/operator_utils.cpp +++ b/src/executor/operator_utils.cpp @@ -1,46 +1,46 @@ #include "executor/operator_utils.h" +#include #include #include "common/ascii.h" #include "common/error.h" #include "common/parsing.h" -template -static bool EvaluateComparison(const T left, const T right, const ComparisonKind kind) { +static bool EvaluateOrdering(const std::strong_ordering ordering, const ComparisonKind kind) { if (kind == ComparisonKind::Equal) { - return left == right; + return ordering == 0; } + if (kind == ComparisonKind::NotEqual) { - return left != right; + return ordering != 0; } + if (kind == ComparisonKind::Less) { - return left < right; + return ordering < 0; } + if (kind == ComparisonKind::LessOrEqual) { - return left <= right; + return ordering <= 0; } + if (kind == ComparisonKind::Greater) { - return left > right; + return ordering > 0; } + if (kind == ComparisonKind::GreaterOrEqual) { - return left >= right; + return ordering >= 0; } + throw Error::Unsupported("executor", "unsupported comparison kind"); } template -static int CompareValuesImpl(const T left, const T right) { - if (left < right) { - return -1; - } - if (left > right) { - return 1; - } - return 0; +static std::strong_ordering CompareValuesImpl(const T left, const T right) { + return left <=> right; } -int CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { +std::strong_ordering CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { switch (type) { case ColumnType::Boolean: return CompareValuesImpl(ParseBoolean(std::string(lhs)), ParseBoolean(std::string(rhs))); @@ -61,24 +61,29 @@ int CompareValues(const ColumnType type, const std::string_view lhs, const std:: case ColumnType::Character: return CompareValuesImpl(ParseCharacter(std::string(lhs)), ParseCharacter(std::string(rhs))); } + throw Error::Unsupported("executor", "unsupported column type in comparison"); } bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, const ComparisonKind kind) { - return EvaluateComparison(CompareValues(type, lhs, rhs), 0, kind); + return EvaluateOrdering(CompareValues(type, lhs, rhs), kind); } ColumnType AggregateOutputType(const PlannedAgg& aggregate) { const std::string name = ToUpperAscii(aggregate.function->canonical_name); + if (name == "COUNT") { return ColumnType::Int64; } + if (name == "SUM" || name == "AVG") { return ColumnType::Int128; } + if (name == "MIN" || name == "MAX") { return aggregate.input_type; } + throw Error::Unsupported("executor", "unsupported aggregate result type"); } diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index 68dbdd4..b9cd3f9 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -15,7 +16,7 @@ class ScanOperator final : public Operator { ScanOperator(std::filesystem::path path, std::vector projection_indexes) : path_(std::move(path)), input_(path_), metadata_(ColumnarBatchReader(path_).GetMetadata()) { if (metadata_.schema.columns.empty()) { - throw Error::MalformedData("operators", "columnar schema is empty", path_.string()); + throw Error::MalformedData("executor", "columnar schema is empty", path_.string()); } projection_indexes_ = std::move(projection_indexes); @@ -23,13 +24,13 @@ class ScanOperator final : public Operator { for (const size_t source_index : projection_indexes_) { if (source_index >= metadata_.schema.columns.size()) { - throw Error::OutOfRange("operators", "projection index out of range", path_.string()); + throw Error::OutOfRange("executor", "projection index out of range", path_.string()); } + projected_schema_.columns.push_back(metadata_.schema.columns[source_index]); } } - public: std::optional Next() override { if (next_group_ >= metadata_.row_groups.size()) { return std::nullopt; @@ -41,6 +42,7 @@ class ScanOperator final : public Operator { for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { const size_t source_index = projection_indexes_[projected_index]; const auto& chunk = row_group.columns[source_index]; + batch.ReadColumnFrom(projected_index, input_.StreamAt(chunk.offset), row_group.row_count, chunk.size); } @@ -62,7 +64,6 @@ class FilterOperator final : public Operator { FilterOperator(std::unique_ptr child, PlannedFilter filter) : child_(std::move(child)), filter_(std::move(filter)) {} - public: std::optional Next() override { while (auto batch = child_->Next()) { Batch filtered(batch->GetSchema()); @@ -96,7 +97,6 @@ class FilterOperator final : public Operator { return batch.ColumnAt(value.column_index).ValueAsString(row); } - private: std::unique_ptr child_; PlannedFilter filter_; }; @@ -165,10 +165,10 @@ class GroupKeyEncoder { static void AppendValue(std::string& id, const std::string_view value) { id += std::to_string(value.size()); id.push_back(':'); + id += value; } - private: std::vector group_keys_; }; @@ -176,6 +176,7 @@ class AggOperator final : public Operator { public: AggOperator(std::unique_ptr child, const std::vector& aggregates) : child_(std::move(child)) { bindings_.reserve(aggregates.size()); + for (const auto& aggregate : aggregates) { bindings_.push_back(AggBinding{ .aggregate = aggregate, @@ -185,11 +186,11 @@ class AggOperator final : public Operator { } } - public: std::optional Next() override { if (returned_) { return std::nullopt; } + returned_ = true; while (auto batch = child_->Next()) { @@ -229,6 +230,7 @@ class GroupAggOperator final : public Operator { const std::vector& aggregates, std::vector select_items) : child_(std::move(child)), group_key_encoder_(std::move(group_keys)), select_items_(std::move(select_items)) { bindings_.reserve(aggregates.size()); + for (const auto& aggregate : aggregates) { bindings_.push_back(GroupAggBinding{ .aggregate = aggregate, @@ -237,11 +239,11 @@ class GroupAggOperator final : public Operator { } } - public: std::optional Next() override { if (returned_) { return std::nullopt; } + returned_ = true; while (auto batch = child_->Next()) { @@ -253,6 +255,7 @@ class GroupAggOperator final : public Operator { if (it == group_indexes_.end()) { group_index = groups_.size(); group_indexes_.emplace(std::move(key.id), group_index); + groups_.push_back(GroupState{ .key_values = std::move(key.values), .states = CreateStates(), @@ -270,17 +273,20 @@ class GroupAggOperator final : public Operator { Schema schema; schema.columns.reserve(select_items_.size()); + for (const auto& item : select_items_) { schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); } std::vector> rows; rows.reserve(groups_.size()); + for (const auto& group : groups_) { rows.push_back(MaterializeRow(group)); } Batch result(schema, rows.size()); + for (const auto& row : rows) { for (size_t i = 0; i < row.size(); ++i) { result.AppendValueFromString(i, row[i]); @@ -304,9 +310,11 @@ class GroupAggOperator final : public Operator { std::vector> CreateStates() const { std::vector> states; states.reserve(bindings_.size()); + for (const auto& binding : bindings_) { states.push_back(CreateAggState(binding.aggregate)); } + return states; } @@ -325,7 +333,6 @@ class GroupAggOperator final : public Operator { return row; } - private: std::unique_ptr child_; GroupKeyEncoder group_key_encoder_; std::vector bindings_; @@ -335,21 +342,83 @@ class GroupAggOperator final : public Operator { bool returned_ = false; }; +struct SortedRow { + std::vector values; + size_t ordinal = 0; +}; + +class RowOrdering { + public: + RowOrdering(std::vector order_by, std::vector select_items) + : order_by_(std::move(order_by)), select_items_(std::move(select_items)) {} + + bool operator()(const SortedRow& lhs, const SortedRow& rhs) const { + for (const auto& order : order_by_) { + const std::strong_ordering comparison = CompareValues( + order.value_type, lhs.values[order.result_column_index], rhs.values[order.result_column_index]); + if (comparison != 0) { + return order.descending ? comparison > 0 : comparison < 0; + } + } + + for (size_t i = 0; i < select_items_.size(); ++i) { + if (select_items_[i].kind != SelectItemKind::GroupKey) { + continue; + } + const std::strong_ordering comparison = + CompareValues(select_items_[i].output_type, lhs.values[i], rhs.values[i]); + if (comparison != 0) { + return comparison > 0; + } + } + + return lhs.ordinal < rhs.ordinal; + } + + private: + std::vector order_by_; + std::vector select_items_; +}; + +static std::vector MaterializeBatchRow(const Batch& batch, const size_t row) { + std::vector values; + values.reserve(batch.ColumnsCount()); + + for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { + values.push_back(batch.ColumnAt(column_index).ValueAsString(row)); + } + + return values; +} + +static Batch BuildBatchFromSortedRows(const Schema& schema, const std::vector& rows) { + Batch result(schema, rows.size()); + + for (const auto& row : rows) { + for (size_t i = 0; i < row.values.size(); ++i) { + result.AppendValueFromString(i, row.values[i]); + } + } + + return result; +} + class OrderByOperator final : public Operator { public: OrderByOperator(std::unique_ptr child, std::vector order_by, std::vector select_items) - : child_(std::move(child)), order_by_(std::move(order_by)), select_items_(std::move(select_items)) {} + : child_(std::move(child)), ordering_(std::move(order_by), std::move(select_items)) {} - public: std::optional Next() override { if (returned_) { return std::nullopt; } + returned_ = true; std::optional schema; - std::vector> rows; + std::vector rows; + size_t ordinal = 0; while (auto batch = child_->Next()) { if (!schema.has_value()) { @@ -357,13 +426,12 @@ class OrderByOperator final : public Operator { } rows.reserve(rows.size() + batch->RowsCount()); + for (size_t row = 0; row < batch->RowsCount(); ++row) { - std::vector values; - values.reserve(batch->ColumnsCount()); - for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { - values.push_back(batch->ColumnAt(column_index).ValueAsString(row)); - } - rows.push_back(std::move(values)); + rows.push_back(SortedRow{ + .values = MaterializeBatchRow(*batch, row), + .ordinal = ordinal++, + }); } } @@ -373,44 +441,69 @@ class OrderByOperator final : public Operator { SortRows(rows); - Batch result(*schema, rows.size()); - for (const auto& row : rows) { - for (size_t i = 0; i < row.size(); ++i) { - result.AppendValueFromString(i, row[i]); - } - } - - return result; + return BuildBatchFromSortedRows(*schema, rows); } private: - void SortRows(std::vector>& rows) const { - std::ranges::stable_sort(rows, [&](const std::vector& lhs, const std::vector& rhs) { - for (const auto& order : order_by_) { - const int comparison = - CompareValues(order.value_type, lhs[order.result_column_index], rhs[order.result_column_index]); - if (comparison != 0) { - return order.descending ? comparison > 0 : comparison < 0; - } + void SortRows(std::vector& rows) const { std::ranges::sort(rows, ordering_); } + + std::unique_ptr child_; + RowOrdering ordering_; + bool returned_ = false; +}; + +class TopKOperator final : public Operator { + public: + TopKOperator(std::unique_ptr child, std::vector order_by, + std::vector select_items, const size_t limit) + : child_(std::move(child)), ordering_(std::move(order_by), std::move(select_items)), limit_(limit) {} + + std::optional Next() override { + if (returned_ || limit_ == 0) { + return std::nullopt; + } + + returned_ = true; + + std::optional schema; + std::vector top_rows; + size_t ordinal = 0; + + while (auto batch = child_->Next()) { + if (!schema.has_value()) { + schema = batch->GetSchema(); } - for (size_t i = 0; i < select_items_.size(); ++i) { - if (select_items_[i].kind != SelectItemKind::GroupKey) { - continue; - } - const int comparison = CompareValues(select_items_[i].output_type, lhs[i], rhs[i]); - if (comparison != 0) { - return comparison > 0; + for (size_t row = 0; row < batch->RowsCount(); ++row) { + SortedRow candidate{ + .values = MaterializeBatchRow(*batch, row), + .ordinal = ordinal++, + }; + + if (top_rows.size() < limit_) { + top_rows.push_back(std::move(candidate)); + std::ranges::push_heap(top_rows, ordering_); + } else if (ordering_(candidate, top_rows.front())) { + std::ranges::pop_heap(top_rows, ordering_); + top_rows.back() = std::move(candidate); + std::ranges::push_heap(top_rows, ordering_); } } - return false; - }); + } + + if (!schema.has_value()) { + return std::nullopt; + } + + std::ranges::sort(top_rows, ordering_); + + return BuildBatchFromSortedRows(*schema, top_rows); } private: std::unique_ptr child_; - std::vector order_by_; - std::vector select_items_; + RowOrdering ordering_; + size_t limit_; bool returned_ = false; }; @@ -418,7 +511,6 @@ class LimitOperator final : public Operator { public: LimitOperator(std::unique_ptr child, const size_t limit) : child_(std::move(child)), remaining_(limit) {} - public: std::optional Next() override { if (remaining_ == 0) { return std::nullopt; @@ -427,20 +519,25 @@ class LimitOperator final : public Operator { while (auto batch = child_->Next()) { if (batch->RowsCount() <= remaining_) { remaining_ -= batch->RowsCount(); + return batch; } Batch limited(batch->GetSchema(), remaining_); + for (size_t row = 0; row < remaining_; ++row) { for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { limited.AppendValueFromString(column_index, batch->ColumnAt(column_index).ValueAsString(row)); } } + remaining_ = 0; + return limited; } remaining_ = 0; + return std::nullopt; } @@ -451,20 +548,25 @@ class LimitOperator final : public Operator { std::unique_ptr BuildPlan(const PlannedQuery& planned) { std::unique_ptr root = std::make_unique(planned.table_path, planned.projection_indexes); + if (planned.filter.has_value()) { root = std::make_unique(std::move(root), *planned.filter); } + if (!planned.group_keys.empty()) { root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, planned.select_items); } else { root = std::make_unique(std::move(root), planned.aggregates); } - if (!planned.order_by.empty()) { + + if (!planned.order_by.empty() && planned.limit.has_value()) { + root = std::make_unique(std::move(root), planned.order_by, planned.select_items, *planned.limit); + } else if (!planned.order_by.empty()) { root = std::make_unique(std::move(root), planned.order_by, planned.select_items); - } - if (planned.limit.has_value()) { + } else if (planned.limit.has_value()) { root = std::make_unique(std::move(root), *planned.limit); } + return root; } diff --git a/src/executor/planner_utils.cpp b/src/executor/planner_utils.cpp index a2358c5..060c851 100644 --- a/src/executor/planner_utils.cpp +++ b/src/executor/planner_utils.cpp @@ -5,12 +5,14 @@ size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { const std::string needle = ToLowerAscii(column_name); + for (size_t i = 0; i < schema.columns.size(); ++i) { if (ToLowerAscii(schema.columns[i].name) == needle) { return i; } } - throw Error::InvalidArgument("query_planner", "unknown column '" + std::string(column_name) + "'"); + + throw Error::InvalidArgument("executor", "unknown column '" + std::string(column_name) + "'"); } static bool SameQualifier(const std::string_view lhs, const std::string_view rhs) { @@ -21,18 +23,21 @@ void ValidateColumnQualifier(const Query& query, const ColumnRef& column) { if (column.qualifier.empty()) { return; } + if (SameQualifier(column.qualifier, query.table_name)) { return; } + if (!query.table_alias.empty() && SameQualifier(column.qualifier, query.table_alias)) { return; } - throw Error::InvalidArgument("query_planner", "unknown table qualifier '" + column.qualifier + "'"); + + throw Error::InvalidArgument("executor", "unknown table qualifier '" + column.qualifier + "'"); } static std::string UnescapeSqlString(const std::string_view text) { if (text.size() < 2 || text.front() != '\'' || text.back() != '\'') { - throw Error::InvalidArgument("query_planner", "invalid string literal"); + throw Error::InvalidArgument("executor", "invalid string literal"); } std::string result; @@ -44,6 +49,7 @@ static std::string UnescapeSqlString(const std::string_view text) { ++i; continue; } + result.push_back(text[i]); } @@ -61,7 +67,7 @@ std::string NormalizeLiteral(const QueryLiteral& literal, const ColumnType type) } return literal.text; default: - throw Error::InvalidArgument("query_planner", "unsupported literal type"); + throw Error::InvalidArgument("executor", "unsupported literal type"); } } @@ -73,9 +79,11 @@ bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs) { if (!SameColumnName(lhs.name, rhs.name)) { return false; } + if (lhs.qualifier.empty() || rhs.qualifier.empty()) { return true; } + return SameQualifier(lhs.qualifier, rhs.qualifier); } @@ -83,9 +91,11 @@ bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { if (lhs.kind != rhs.kind) { return false; } + if (lhs.kind == SelectItemKind::GroupKey) { return SameColumnRef(lhs.column, rhs.column); } + return ToLowerAscii(lhs.aggregate.output_name) == ToLowerAscii(rhs.aggregate.output_name); } diff --git a/src/executor/query_parser.cpp b/src/executor/query_parser.cpp index fa5a38f..11233b6 100644 --- a/src/executor/query_parser.cpp +++ b/src/executor/query_parser.cpp @@ -13,10 +13,9 @@ class TokenCursor { public: explicit TokenCursor(std::vector tokens) : tokens_(std::move(tokens)) {} - public: const Token& Peek(const size_t lookahead = 0) const { if (pos_ + lookahead >= tokens_.size()) { - throw Error::InvalidArgument("query_parser", "unexpected end of query"); + throw Error::InvalidArgument("executor", "unexpected end of query"); } return *tokens_[pos_ + lookahead]; } @@ -27,7 +26,7 @@ class TokenCursor { const Token& Consume(const Tokens type, const std::string_view message) { if (!Match(type)) { - throw Error::InvalidArgument("query_parser", std::string(message)); + throw Error::InvalidArgument("executor", std::string(message)); } return *tokens_[pos_++]; } @@ -42,7 +41,7 @@ class TokenCursor { const Token& ConsumeFunctionToken(const std::string_view message) { if (pos_ >= tokens_.size()) { - throw Error::InvalidArgument("query_parser", std::string(message)); + throw Error::InvalidArgument("executor", std::string(message)); } const Tokens type = tokens_[pos_]->GetType(); @@ -61,7 +60,7 @@ class TokenCursor { case Tokens::Greater: case Tokens::GreaterOrEqual: case Tokens::EndOfInput: - throw Error::InvalidArgument("query_parser", std::string(message)); + throw Error::InvalidArgument("executor", std::string(message)); default: break; } @@ -89,7 +88,7 @@ static ComparisonKind ComparisonKindFromToken(const Tokens type) { case Tokens::GreaterOrEqual: return ComparisonKind::GreaterOrEqual; default: - throw Error::InvalidArgument("query_parser", "expected comparison operator in WHERE"); + throw Error::InvalidArgument("executor", "expected comparison operator in WHERE"); } } @@ -164,7 +163,7 @@ static QueryValue ParseFilterValue(TokenCursor& cursor, const std::string_view m .kind = LiteralKind::Identifier, }); default: - throw Error::InvalidArgument("query_parser", std::string(message)); + throw Error::InvalidArgument("executor", std::string(message)); } } @@ -172,6 +171,7 @@ static std::string FormatAggregateCall(const std::string_view function_name, con const bool distinct, const AggArgumentKind argument_kind) { std::string output(function_name); output += '('; + if (argument_kind == AggArgumentKind::Star) { output += '*'; } else { @@ -180,7 +180,9 @@ static std::string FormatAggregateCall(const std::string_view function_name, con } output += column_name; } + output += ')'; + return output; } @@ -197,7 +199,7 @@ static std::optional ParseOptionalLimit(TokenCursor& cursor) { const char* end = begin + text.size(); if (const auto [ptr, ec] = std::from_chars(begin, end, limit); ec != std::errc() || ptr != end) { - throw Error::InvalidArgument("query_parser", "invalid LIMIT value"); + throw Error::InvalidArgument("executor", "invalid LIMIT value"); } return limit; @@ -248,19 +250,23 @@ static std::string ParseSelectItemAlias(TokenCursor& cursor) { if (cursor.TryConsume(Tokens::As)) { return std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); } + if (cursor.Match(Tokens::NameToken)) { return std::string(cursor.Consume(Tokens::NameToken, "expected alias").GetText()); } + return {}; } static SelectItemSpec ParseSelectItem(TokenCursor& cursor) { SelectItemSpec item = ParseSelectExpression(cursor); + if (const std::string alias = ParseSelectItemAlias(cursor); !alias.empty()) { item.output_name = alias; } else if (item.kind == SelectItemKind::Aggregate) { item.output_name = item.aggregate.output_name; } + return item; } @@ -303,6 +309,7 @@ Query ParseQuery(const std::string_view query) { cursor.Consume(Tokens::From, "expected FROM"); query_ast.table_name = std::string(cursor.Consume(Tokens::NameToken, "expected table name").GetText()); + if (cursor.TryConsume(Tokens::As)) { query_ast.table_alias = std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); } else if (cursor.Match(Tokens::NameToken)) { @@ -325,6 +332,7 @@ Query ParseQuery(const std::string_view query) { if (cursor.TryConsume(Tokens::Group)) { cursor.Consume(Tokens::By, "expected BY after GROUP"); query_ast.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); + while (cursor.TryConsume(Tokens::Comma)) { query_ast.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); } @@ -333,6 +341,7 @@ Query ParseQuery(const std::string_view query) { if (cursor.TryConsume(Tokens::Order)) { cursor.Consume(Tokens::By, "expected BY after ORDER"); query_ast.order_by.push_back(ParseOrderByItem(cursor)); + while (cursor.TryConsume(Tokens::Comma)) { query_ast.order_by.push_back(ParseOrderByItem(cursor)); } diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index 261e245..e5f7724 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -2,11 +2,11 @@ #include +#include "common/ascii.h" +#include "common/error.h" #include "executor/operator_utils.h" #include "executor/planner_utils.h" #include "io/columnar_batch.h" -#include "common/ascii.h" -#include "common/error.h" static std::optional TryFindColumnIndex(const Schema& schema, const std::string_view column_name) { for (size_t i = 0; i < schema.columns.size(); ++i) { @@ -22,7 +22,7 @@ PlannedQuery PlanQuery(const Query& query_ast, const std::unordered_mapsecond; @@ -30,12 +30,14 @@ PlannedQuery PlanQuery(const Query& query_ast, const std::unordered_map size_t { + const auto ensure_projected_column = [&](const size_t source_index) -> size_t { const auto it = std::ranges::find(planned.projection_indexes, source_index); if (it != planned.projection_indexes.end()) { return static_cast(std::distance(planned.projection_indexes.begin(), it)); } + planned.projection_indexes.push_back(source_index); + return planned.projection_indexes.size() - 1; }; @@ -47,13 +49,15 @@ PlannedQuery PlanQuery(const Query& query_ast, const std::unordered_map PlannedFilterValue { if (value.kind == QueryValueKind::Column) { ValidateColumnQualifier(query_ast, value.column); + const size_t source_index = FindColumnIndex(schema, value.column.name); + return PlannedFilterValue{ .value = PlannedValue{ .kind = PlannedValueKind::Column, .column_type = schema.columns[source_index].type, - .column_index = register_projection(source_index), + .column_index = ensure_projected_column(source_index), .literal_text = {}, }, .literal = std::nullopt, @@ -68,7 +72,7 @@ PlannedQuery PlanQuery(const Query& query_ast, const std::unordered_mapcolumn_type, .output_name = item.output_name, }); + continue; } @@ -162,13 +172,13 @@ PlannedQuery PlanQuery(const Query& query_ast, const std::unordered_map(std::distance(query_ast.select_items.begin(), it)); + planned.order_by.push_back(PlannedOrderBy{ .result_column_index = result_column_index, .descending = order_item.descending, diff --git a/src/io/batch.cpp b/src/io/batch.cpp index d2fcb79..cb2cf7a 100644 --- a/src/io/batch.cpp +++ b/src/io/batch.cpp @@ -7,7 +7,7 @@ static void ValidateOptionalLimit(const std::optional& value, const std::string_view name) { if (value && *value == 0) { - throw Error::InvalidArgument("batch_io", std::string(name) + " must be > 0"); + throw Error::InvalidArgument("io", std::string(name) + " must be > 0"); } } @@ -16,16 +16,19 @@ bool BatchSizing::WouldExceed(const size_t next_rows, const size_t column_count, ValidateOptionalLimit(max_values, "max values"); if (max_bytes && *max_bytes == 0) { - throw Error::InvalidArgument("batch_io", "max bytes must be > 0"); + throw Error::InvalidArgument("io", "max bytes must be > 0"); } + if (max_rows && next_rows > *max_rows) { return true; } + if (max_values && column_count > 0) { if (next_rows > *max_values / column_count) { return true; } } + if (max_bytes && next_bytes > *max_bytes) { return true; } diff --git a/src/io/columnar_batch.cpp b/src/io/columnar_batch.cpp index 7457c56..9b34359 100644 --- a/src/io/columnar_batch.cpp +++ b/src/io/columnar_batch.cpp @@ -1,3 +1,5 @@ +#include "io/columnar_batch.h" + #include #include #include @@ -5,16 +7,15 @@ #include #include -#include "io/columnar_batch.h" -#include "io/stream.h" #include "common/error.h" +#include "io/stream.h" static constexpr std::string_view ColumnarMagic = "CLMN"; static uint64_t TellWrite(const std::filesystem::path& path, std::ofstream& out) { const auto pos = out.tellp(); if (pos == -1) { - throw Error::PathIo("batch_io", path, "tell file"); + throw Error::PathIo("io", path, "tell file"); } return pos; } @@ -22,7 +23,7 @@ static uint64_t TellWrite(const std::filesystem::path& path, std::ofstream& out) static void FlushWrite(const std::filesystem::path& path, std::ofstream& out) { out.flush(); if (!out) { - throw Error::PathIo("batch_io", path, "write file"); + throw Error::PathIo("io", path, "write file"); } } @@ -30,14 +31,14 @@ ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::pa const auto file_metadata = GetFileMetadata(path); if (!file_metadata || !file_metadata->is_regular) { - throw Error::NotFound("batch_io", "columnar file not found", path.string()); + throw Error::NotFound("io", "columnar file not found", path.string()); } const uint64_t file_size = file_metadata->size; constexpr uint64_t FooterSize = sizeof(uint64_t) + ColumnarMagic.size(); if (file_size < FooterSize) { - throw Error::MalformedData("batch_io", "columnar file is too small", path.string()); + throw Error::MalformedData("io", "columnar file is too small", path.string()); } InputFile file(path); @@ -45,11 +46,11 @@ ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::pa const std::string magic_read = file.ReadStringAt(file_size - ColumnarMagic.size(), ColumnarMagic.size()); if (magic_read != ColumnarMagic) { - throw Error::MalformedData("batch_io", "invalid columnar magic", path.string()); + throw Error::MalformedData("io", "invalid columnar magic", path.string()); } if (metadata_size > file_size - FooterSize) { - throw Error::MalformedData("batch_io", "metadata size exceeds file size", path.string()); + throw Error::MalformedData("io", "metadata size exceeds file size", path.string()); } return ReadMetadata(file.StreamAt(file_size - FooterSize - metadata_size)); @@ -58,7 +59,7 @@ ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::pa ColumnarBatchReader::ColumnarBatchReader(const std::filesystem::path& path) : path_(path), input_(path), metadata_(ReadFileMetadata(path)) { if (metadata_.schema.columns.empty()) { - throw Error::MalformedData("batch_io", "columnar schema is empty", path.string()); + throw Error::MalformedData("io", "columnar schema is empty", path.string()); } } @@ -71,7 +72,7 @@ std::optional ColumnarBatchReader::ReadNext() { Batch batch(metadata_.schema, row_count); if (row_group_columns.size() != batch.ColumnsCount()) { - throw Error::InconsistentData("batch_io", "row group column count mismatch", path_.string()); + throw Error::InconsistentData("io", "row group column count mismatch", path_.string()); } for (size_t col = 0; col < batch.ColumnsCount(); ++col) { @@ -85,19 +86,19 @@ std::optional ColumnarBatchReader::ReadNext() { ColumnarBatchWriter::ColumnarBatchWriter(const std::filesystem::path& path, Schema schema) : path_(path), out_(OpenOutputFile(path)) { if (schema.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns", path.string()); + throw Error::InvalidArgument("io", "schema has no columns", path.string()); } metadata_.schema = std::move(schema); } void ColumnarBatchWriter::Write(const Batch& batch) { if (finalized_) { - throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); + throw Error::InvalidState("io", "writer already finalized", path_.string()); } batch.Validate(); if (batch.GetSchema() != metadata_.schema) { - throw Error::InconsistentData("batch_io", "batch schema mismatch", path_.string()); + throw Error::InconsistentData("io", "batch schema mismatch", path_.string()); } const size_t row_count = batch.RowsCount(); @@ -105,8 +106,9 @@ void ColumnarBatchWriter::Write(const Batch& batch) { if (row_count == 0) { return; } + if (row_count > std::numeric_limits::max()) { - throw Error::Overflow("batch_io", "row group exceeds supported size", path_.string()); + throw Error::Overflow("io", "row group exceeds supported size", path_.string()); } RowGroupMetadata group; @@ -116,7 +118,9 @@ void ColumnarBatchWriter::Write(const Batch& batch) { for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { const uint64_t offset = TellWrite(path_, out_); batch.ColumnAt(column_index).WriteTo(out_); + const uint64_t end = TellWrite(path_, out_); + group.columns.push_back(ColumnChunkMetadata{offset, end - offset}); } @@ -127,7 +131,7 @@ void ColumnarBatchWriter::Flush() { FlushWrite(path_, out_); } void ColumnarBatchWriter::Finalize() && { if (finalized_) { - throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); + throw Error::InvalidState("io", "writer already finalized", path_.string()); } const uint64_t metadata_start = TellWrite(path_, out_); @@ -139,5 +143,6 @@ void ColumnarBatchWriter::Finalize() && { WriteBytes(out_, ColumnarMagic); FlushWrite(path_, out_); + finalized_ = true; } diff --git a/src/io/csv.cpp b/src/io/csv.cpp index f9e2c3a..66819d6 100644 --- a/src/io/csv.cpp +++ b/src/io/csv.cpp @@ -3,14 +3,13 @@ #include #include -#include "io/file.h" #include "common/error.h" +#include "io/file.h" class CsvCharReader { public: explicit CsvCharReader(std::istream& in) : buffer_(in.rdbuf()) {} - public: std::optional Read() const { const Traits::int_type next = buffer_->sbumpc(); if (Traits::eq_int_type(next, Traits::eof())) { @@ -76,12 +75,13 @@ bool CsvReader::ReadRow(std::vector& row) const { while (true) { const std::optional next = input.Read(); + if (!next.has_value()) { if (!saw_data) { return false; } if (in_quotes) { - throw Error::MalformedData("csv", "unexpected EOF inside quoted field"); + throw Error::MalformedData("io", "unexpected EOF inside quoted field"); } return true; } @@ -178,18 +178,21 @@ void CsvWriter::WriteRow(const std::vector& row) const { *out_ << ch; } } + *out_ << '"'; } + *out_ << '\n'; + if (!*out_) { - throw Error::Io("csv", "failed to write csv row"); + throw Error::Io("io", "failed to write csv row"); } } void CsvWriter::Flush() const { out_->flush(); if (!*out_) { - throw Error::Io("csv", "failed to write csv"); + throw Error::Io("io", "failed to write csv"); } } diff --git a/src/io/csv_batch.cpp b/src/io/csv_batch.cpp index 2ec38d6..6e8d526 100644 --- a/src/io/csv_batch.cpp +++ b/src/io/csv_batch.cpp @@ -10,7 +10,7 @@ static uint64_t AddChecked(const uint64_t current, const uint64_t add) { if (add > std::numeric_limits::max() - current) { - throw Error::Overflow("batch_io", "batch byte size overflow"); + throw Error::Overflow("io", "batch byte size overflow"); } return current + add; } @@ -34,7 +34,7 @@ static uint64_t EstimateValueBytes(const ColumnType type, const std::string_view case ColumnType::String: return value.size(); } - throw Error::Unsupported("batch_io", "unsupported column type"); + throw Error::Unsupported("io", "unsupported column type"); } static uint64_t EstimateRowBytes(const Schema& schema, const std::vector& row) { @@ -47,13 +47,15 @@ static uint64_t EstimateRowBytes(const Schema& schema, const std::vector 0"); + throw Error::InvalidArgument("io", "max rows must be > 0"); } + if (sizing.max_values && *sizing.max_values == 0) { - throw Error::InvalidArgument("batch_io", "max values must be > 0"); + throw Error::InvalidArgument("io", "max values must be > 0"); } + if (sizing.max_bytes && *sizing.max_bytes == 0) { - throw Error::InvalidArgument("batch_io", "max bytes must be > 0"); + throw Error::InvalidArgument("io", "max bytes must be > 0"); } } @@ -62,6 +64,7 @@ static void ReserveForSizing(Batch& batch, const BatchSizing& sizing, const size batch.Reserve(*sizing.max_rows); return; } + if (sizing.max_values && column_count > 0) { batch.Reserve(std::max(1, *sizing.max_values / column_count)); } @@ -70,7 +73,7 @@ static void ReserveForSizing(Batch& batch, const BatchSizing& sizing, const size CsvBatchReader::CsvBatchReader(const std::filesystem::path& path, Schema schema, BatchSizing sizing) : csv_reader_(path), schema_(std::move(schema)), sizing_(std::move(sizing)) { if (schema_.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns"); + throw Error::InvalidArgument("io", "schema has no columns"); } ValidateSizing(sizing_); } @@ -103,7 +106,7 @@ std::optional CsvBatchReader::ReadNext() { } if (row.size() != column_count) { - throw Error::InconsistentData("batch_io", "data csv column count mismatch"); + throw Error::InconsistentData("io", "data csv column count mismatch"); } const size_t next_rows = rows + 1; @@ -136,14 +139,14 @@ std::optional CsvBatchReader::ReadNext() { CsvBatchWriter::CsvBatchWriter(const std::filesystem::path& path, Schema schema) : csv_writer_(path), schema_(std::move(schema)) { if (schema_.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns"); + throw Error::InvalidArgument("io", "schema has no columns"); } } void CsvBatchWriter::Write(const Batch& batch) { batch.Validate(); if (batch.GetSchema() != schema_) { - throw Error::InconsistentData("batch_io", "batch schema mismatch"); + throw Error::InconsistentData("io", "batch schema mismatch"); } const size_t column_count = schema_.columns.size(); @@ -155,6 +158,7 @@ void CsvBatchWriter::Write(const Batch& batch) { for (size_t col = 0; col < column_count; ++col) { row[col] = batch.ColumnAt(col).ValueAsString(row_index); } + csv_writer_.WriteRow(row); } } @@ -179,6 +183,7 @@ void AppendBatchRows(const Batch& batch, std::vector>& void WriteBatchCsv(const std::filesystem::path& path, const Batch& batch) { CsvBatchWriter writer(path, batch.GetSchema()); + writer.Write(batch); writer.Flush(); } diff --git a/src/io/file.cpp b/src/io/file.cpp index a24f34a..7965ba0 100644 --- a/src/io/file.cpp +++ b/src/io/file.cpp @@ -1,15 +1,16 @@ +#include "io/file.h" + #include #include -#include "io/file.h" -#include "io/stream.h" #include "common/error.h" +#include "io/stream.h" std::optional GetFileMetadata(const std::filesystem::path& path) { std::error_code ec; if (!std::filesystem::exists(path, ec)) { if (ec) { - throw Error::PathIo("fileio", path, "check existence"); + throw Error::PathIo("io", path, "check existence"); } return std::nullopt; } @@ -17,7 +18,7 @@ std::optional GetFileMetadata(const std::filesystem::path& path) { FileMetadata metadata; const auto status = std::filesystem::status(path, ec); if (ec) { - throw Error::PathIo("fileio", path, "read status"); + throw Error::PathIo("io", path, "read status"); } metadata.is_regular = std::filesystem::is_regular_file(status); @@ -25,13 +26,13 @@ std::optional GetFileMetadata(const std::filesystem::path& path) { if (metadata.is_regular) { metadata.size = std::filesystem::file_size(path, ec); if (ec) { - throw Error::PathIo("fileio", path, "read file size"); + throw Error::PathIo("io", path, "read file size"); } } metadata.last_write_time = std::filesystem::last_write_time(path, ec); if (ec) { - throw Error::PathIo("fileio", path, "read last write time"); + throw Error::PathIo("io", path, "read last write time"); } return metadata; @@ -41,8 +42,9 @@ bool FileExists(const std::filesystem::path& path) { std::error_code ec; const bool exists = std::filesystem::exists(path, ec); if (ec) { - throw Error::PathIo("fileio", path, "check existence"); + throw Error::PathIo("io", path, "check existence"); } + return exists; } @@ -55,14 +57,14 @@ void EnsureParentDirectory(const std::filesystem::path& path) { std::error_code ec; std::filesystem::create_directories(parent, ec); if (ec) { - throw Error::PathIo("fileio", path, "create parent directories"); + throw Error::PathIo("io", path, "create parent directories"); } } std::ifstream OpenInputFile(const std::filesystem::path& path) { std::ifstream file(path, std::ios::binary); if (!file.is_open()) { - throw Error::PathIo("fileio", path, "open for read"); + throw Error::PathIo("io", path, "open for read"); } return file; } @@ -70,7 +72,7 @@ std::ifstream OpenInputFile(const std::filesystem::path& path) { void SeekInputFile(std::istream& in, const std::filesystem::path& path, const uint64_t offset) { in.seekg(static_cast(offset)); if (!in) { - throw Error::PathIo("fileio", path, "seek for read"); + throw Error::PathIo("io", path, "seek for read"); } } @@ -81,7 +83,7 @@ void InputFile::ReadAt(char* dst, const size_t size, const uint64_t offset) { try { ReadBytes(in_, dst, size); } catch (const Error&) { - throw Error::PathIo("fileio", path_, "read file"); + throw Error::PathIo("io", path_, "read file"); } } @@ -99,7 +101,7 @@ std::istream& InputFile::StreamAt(const uint64_t offset) { std::ofstream OpenOutputFile(const std::filesystem::path& path, const FileOpenMode mode) { std::ofstream file(path, std::ios::binary | (mode == FileOpenMode::Append ? std::ios::app : std::ios::trunc)); if (!file.is_open()) { - throw Error::PathIo("fileio", path, mode == FileOpenMode::Append ? "open for append" : "open for write"); + throw Error::PathIo("io", path, mode == FileOpenMode::Append ? "open for append" : "open for write"); } return file; } @@ -112,13 +114,14 @@ std::string ReadTextFile(const std::filesystem::path& path) { std::vector ReadFileBytes(const std::filesystem::path& path) { std::ifstream file = OpenInputFile(path); file.seekg(0, std::ios::end); + if (!file) { - throw Error::PathIo("fileio", path, "read file size"); + throw Error::PathIo("io", path, "read file size"); } const auto end_pos = file.tellg(); if (end_pos < 0) { - throw Error::PathIo("fileio", path, "read file size"); + throw Error::PathIo("io", path, "read file size"); } std::vector data(end_pos); @@ -130,7 +133,7 @@ std::vector ReadFileBytes(const std::filesystem::path& path) { file.read(reinterpret_cast(data.data()), data.size()); if (!file) { - throw Error::PathIo("fileio", path, "read file"); + throw Error::PathIo("io", path, "read file"); } return data; @@ -141,13 +144,13 @@ void WriteFileBytes(const std::filesystem::path& path, const std::span(bytes.data()), bytes.size()); if (!file) { - throw Error::PathIo("fileio", path, "write file"); + throw Error::PathIo("io", path, "write file"); } } file.flush(); if (!file) { - throw Error::PathIo("fileio", path, "write file"); + throw Error::PathIo("io", path, "write file"); } } diff --git a/src/io/stream.cpp b/src/io/stream.cpp index b01fb4a..f2c372d 100644 --- a/src/io/stream.cpp +++ b/src/io/stream.cpp @@ -8,7 +8,7 @@ void ReadBytes(std::istream& in, char* dst, const size_t size) { } in.read(dst, size); if (in.gcount() != static_cast(size) || !in) { - throw Error::Io("stream", "failed to read bytes"); + throw Error::Io("io", "failed to read bytes"); } } @@ -18,6 +18,6 @@ void WriteBytes(std::ostream& out, const std::string_view bytes) { } out.write(bytes.data(), bytes.size()); if (!out) { - throw Error::Io("stream", "failed to write bytes"); + throw Error::Io("io", "failed to write bytes"); } } diff --git a/src/model/batch.cpp b/src/model/batch.cpp index e282fd0..b646c3a 100644 --- a/src/model/batch.cpp +++ b/src/model/batch.cpp @@ -8,9 +8,11 @@ static std::vector> CloneColumns( const std::vector>& columns) { std::vector> clones; clones.reserve(columns.size()); + for (const auto& column : columns) { clones.push_back(column->CloneMutable()); } + return clones; } @@ -54,7 +56,7 @@ void Batch::Reserve(const size_t n) const { void Batch::AppendValueFromString(const size_t column_index, const std::string& value) const { if (column_index >= columns_.size()) { - throw Error::OutOfRange("batch", "column index out of range"); + throw Error::OutOfRange("model", "column index out of range"); } columns_[column_index]->AppendFromString(value); } @@ -62,21 +64,21 @@ void Batch::AppendValueFromString(const size_t column_index, const std::string& void Batch::ReadColumnFrom(const size_t column_index, std::istream& in, const uint32_t row_count, const uint64_t size) const { if (column_index >= columns_.size()) { - throw Error::OutOfRange("batch", "column index out of range"); + throw Error::OutOfRange("model", "column index out of range"); } columns_[column_index]->ReadFrom(in, row_count, size); } const Column& Batch::ColumnAt(const size_t i) const { if (i >= columns_.size()) { - throw Error::OutOfRange("batch", "column index out of range"); + throw Error::OutOfRange("model", "column index out of range"); } return *columns_[i]; } void Batch::Validate() const { if (schema_.columns.size() != columns_.size()) { - throw Error::InconsistentData("batch", "column count mismatch"); + throw Error::InconsistentData("model", "column count mismatch"); } if (columns_.empty()) { @@ -86,10 +88,11 @@ void Batch::Validate() const { const size_t rows = columns_.front()->Size(); for (size_t i = 0; i < columns_.size(); ++i) { if (columns_[i]->Type() != schema_.columns[i].type) { - throw Error::InconsistentData("batch", "column type mismatch"); + throw Error::InconsistentData("model", "column type mismatch"); } + if (columns_[i]->Size() != rows) { - throw Error::InconsistentData("batch", "column row count mismatch"); + throw Error::InconsistentData("model", "column row count mismatch"); } } } diff --git a/src/model/column.cpp b/src/model/column.cpp index 2218767..0a14447 100644 --- a/src/model/column.cpp +++ b/src/model/column.cpp @@ -34,7 +34,8 @@ std::unique_ptr CreateColumn(const ColumnType type) { case ColumnType::Character: return std::make_unique(); } - throw Error::Unsupported("column", "unsupported column type"); + + throw Error::Unsupported("model", "unsupported column type"); } Column::Column(const ColumnType type) : type_(type) {} diff --git a/src/model/metadata.cpp b/src/model/metadata.cpp index ed90c0d..66765ab 100644 --- a/src/model/metadata.cpp +++ b/src/model/metadata.cpp @@ -20,7 +20,7 @@ static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { return static_cast(type_byte); } - throw Error::MalformedData("columnar", "unknown column type in metadata"); + throw Error::MalformedData("model", "unknown column type in metadata"); } ColumnarMetadata ReadMetadata(std::istream& in) { @@ -34,6 +34,7 @@ ColumnarMetadata ReadMetadata(std::istream& in) { std::string name(name_size, '\0'); ReadBytes(in, name.data(), name.size()); + const uint8_t type_byte = ReadStream(in); const ColumnType type = ColumnTypeFromByte(type_byte); @@ -49,7 +50,7 @@ ColumnarMetadata ReadMetadata(std::istream& in) { const uint32_t columns_in_group = ReadStream(in); if (columns_in_group != column_count) { - throw Error::InconsistentData("columnar", "row group column count mismatch"); + throw Error::InconsistentData("model", "row group column count mismatch"); } group.columns.reserve(columns_in_group); @@ -58,6 +59,7 @@ ColumnarMetadata ReadMetadata(std::istream& in) { ColumnChunkMetadata chunk; chunk.offset = ReadStream(in); chunk.size = ReadStream(in); + group.columns.push_back(chunk); } @@ -69,22 +71,23 @@ ColumnarMetadata ReadMetadata(std::istream& in) { void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { if (metadata.schema.columns.size() > std::numeric_limits::max()) { - throw Error::Overflow("columnar", "too many columns for metadata"); + throw Error::Overflow("model", "too many columns for metadata"); } WriteStream(out, metadata.schema.columns.size()); for (const auto& [name, type] : metadata.schema.columns) { if (name.size() > std::numeric_limits::max()) { - throw Error::Overflow("columnar", "column name too long"); + throw Error::Overflow("model", "column name too long"); } + WriteStream(out, name.size()); WriteBytes(out, name); WriteStream(out, static_cast(type)); } if (metadata.row_groups.size() > std::numeric_limits::max()) { - throw Error::Overflow("columnar", "too many row groups for metadata"); + throw Error::Overflow("model", "too many row groups for metadata"); } WriteStream(out, metadata.row_groups.size()); @@ -92,6 +95,7 @@ void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { for (const auto& [row_count, columns] : metadata.row_groups) { WriteStream(out, row_count); WriteStream(out, columns.size()); + for (const auto& [offset, size] : columns) { WriteStream(out, offset); WriteStream(out, size); diff --git a/src/model/schema_csv.cpp b/src/model/schema_csv.cpp index 42119ff..b9587c6 100644 --- a/src/model/schema_csv.cpp +++ b/src/model/schema_csv.cpp @@ -4,9 +4,9 @@ #include #include -#include "io/csv.h" #include "common/error.h" #include "common/parsing.h" +#include "io/csv.h" static bool CanParseAs(const ColumnType type, const std::string& value) { switch (type) { @@ -30,7 +30,7 @@ static bool CanParseAs(const ColumnType type, const std::string& value) { return true; } - throw Error::Unsupported("schema", "unknown column type"); + throw Error::Unsupported("model", "unknown column type"); } static ColumnType InferColumnType(const std::vector& values) { @@ -41,12 +41,14 @@ static ColumnType InferColumnType(const std::vector& values) { for (const ColumnType type : InferenceOrder) { bool matches = true; + for (const auto& value : values) { if (!CanParseAs(type, value)) { matches = false; break; } } + if (matches) { return type; } @@ -65,12 +67,15 @@ Schema ReadSchemaCsv(const std::filesystem::path& path) { if (row.size() == 1 && row[0].empty()) { continue; } + while (row.size() > 2 && row.back().empty()) { row.pop_back(); } + if (row.size() != 2) { - throw Error::MalformedData("schema", "schema csv must have 2 columns", path.string()); + throw Error::MalformedData("model", "schema csv must have 2 columns", path.string()); } + schema.columns.push_back(ColumnSchema{ row[0], ParseColumnType(row[1]), @@ -78,7 +83,7 @@ Schema ReadSchemaCsv(const std::filesystem::path& path) { } if (schema.columns.empty()) { - throw Error::MalformedData("schema", "schema csv is empty", path.string()); + throw Error::MalformedData("model", "schema csv is empty", path.string()); } return schema; @@ -94,7 +99,7 @@ Schema InferSchemaCsv(const std::filesystem::path& path) { if (values_by_column.empty()) { values_by_column.resize(row.size()); } else if (row.size() != values_by_column.size()) { - throw Error::MalformedData("schema", "csv rows have inconsistent column count", path.string()); + throw Error::MalformedData("model", "csv rows have inconsistent column count", path.string()); } for (size_t i = 0; i < row.size(); ++i) { @@ -103,7 +108,7 @@ Schema InferSchemaCsv(const std::filesystem::path& path) { } if (values_by_column.empty()) { - throw Error::MalformedData("schema", "csv is empty", path.string()); + throw Error::MalformedData("model", "csv is empty", path.string()); } Schema schema; diff --git a/src/sql_parser/tokenizer.cpp b/src/sql_parser/tokenizer.cpp index c8980b1..6e2cb46 100644 --- a/src/sql_parser/tokenizer.cpp +++ b/src/sql_parser/tokenizer.cpp @@ -129,23 +129,28 @@ Expected Tokenizer::GetNext() { ++pos_; return MakeToken(Tokens::LessOrEqual, "<=", start); } + if (pos_ < query_.size() && query_[pos_] == '>') { ++pos_; return MakeToken(Tokens::NotEqual, "<>", start); } + return MakeToken(Tokens::Less, "<", start); case '>': ++pos_; + if (pos_ < query_.size() && query_[pos_] == '=') { ++pos_; return MakeToken(Tokens::GreaterOrEqual, ">=", start); } + return MakeToken(Tokens::Greater, ">", start); case '!': if (pos_ + 1 < query_.size() && query_[pos_ + 1] == '=') { pos_ += 2; return MakeToken(Tokens::NotEqual, "!=", start); } + return MakeUnexpectedCharacter(ch, start); default: return MakeUnexpectedCharacter(ch, start); @@ -165,6 +170,7 @@ Expected> TokenizeSql(const std::string_view input) { } tokens.push_back(token.value()); + if (token.value()->GetType() == Tokens::EndOfInput) { break; } diff --git a/src/sql_parser/tokenizer_factory.cpp b/src/sql_parser/tokenizer_factory.cpp index 43e0dba..aec0708 100644 --- a/src/sql_parser/tokenizer_factory.cpp +++ b/src/sql_parser/tokenizer_factory.cpp @@ -7,7 +7,7 @@ #include "common/ascii.h" #include "sql_parser/tokenizer.h" -static const std::unordered_map kKeywords = { +static const std::unordered_map Keywords = { {"AS", Tokens::As}, {"AND", Tokens::And}, {"AVG", Tokens::Avg}, {"BY", Tokens::By}, {"COUNT", Tokens::Count}, {"CREATE", Tokens::Create}, {"DISTINCT", Tokens::Distinct}, {"FROM", Tokens::From}, {"GROUP", Tokens::Group}, {"LENGTH", Tokens::Length}, {"LIMIT", Tokens::Limit}, {"MAX", Tokens::Max}, @@ -36,10 +36,12 @@ static std::string DescribeCharacter(const char ch) { Tokens ResolveIdentifierType(const std::string_view text) { const std::string upper = ToUpperAscii(text); - const auto it = kKeywords.find(upper); - if (it != kKeywords.end()) { + const auto it = Keywords.find(upper); + + if (it != Keywords.end()) { return it->second; } + return Tokens::NameToken; } @@ -116,15 +118,16 @@ TokenPtr MakeToken(const Tokens type, std::string text, const size_t offset) { case Tokens::EndOfInput: return std::make_shared(std::move(text), offset); } + return nullptr; } Expected MakeUnexpectedCharacter(const char ch, const size_t offset) { - return tl::unexpected(Error::MalformedData("tokenizer_factory", "unexpected character '" + DescribeCharacter(ch) + - "' at position " + std::to_string(offset))); + return tl::unexpected(Error::MalformedData( + "sql_parser", "unexpected character '" + DescribeCharacter(ch) + "' at position " + std::to_string(offset))); } Expected MakeUnterminatedString(const size_t offset) { return tl::unexpected( - Error::MalformedData("tokenizer_factory", "unterminated string literal at position " + std::to_string(offset))); + Error::MalformedData("sql_parser", "unterminated string literal at position " + std::to_string(offset))); } diff --git a/src/sql_parser/tokenizer_tokens.cpp b/src/sql_parser/tokenizer_tokens.cpp index ee9279d..04b8552 100644 --- a/src/sql_parser/tokenizer_tokens.cpp +++ b/src/sql_parser/tokenizer_tokens.cpp @@ -75,6 +75,7 @@ std::string_view TokenTypeToString(const Tokens type) noexcept { case Tokens::EndOfInput: return "end_of_input"; } + return "unknown"; } diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index cec5830..e0b9241 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -162,6 +162,17 @@ TEST(executor, supports_select_alias_order_by_alias_and_limit) { EXPECT_EQ(actual_rows, expected_rows); } +TEST(executor, supports_order_by_limit_top_k_in_result_order) { + const Batch batch = + BuildHitsTable("SELECT RegionID, COUNT(*) FROM hits GROUP BY RegionID ORDER BY RegionID ASC LIMIT 2;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"RegionID", "COUNT(*)"})); + EXPECT_EQ(BatchRows(batch), (std::vector>{ + {"10", "2"}, + {"20", "2"}, + })); +} + TEST(executor, supports_multiple_aggregates_with_alias_and_limit) { const Batch batch = BuildHitsTable( "SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) " From 3b02d52aacd282931fbbc730d11fee3c3a0a4fed Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sun, 17 May 2026 15:48:52 +0300 Subject: [PATCH 26/29] review: finally done <3 --- include/model/column_boolean.h | 3 - include/model/column_character.h | 3 - include/model/column_date.h | 3 - include/model/column_int128.h | 3 - include/model/column_int16.h | 3 - include/model/column_int32.h | 3 - include/model/column_int64.h | 3 - include/model/column_timestamp.h | 3 - include/model/column_traits.h | 111 ++++++++++++++++++++++++++++++ include/model/fixed_column.h | 7 ++ src/CMakeLists.txt | 8 --- src/executor/aggregate_states.cpp | 59 ++-------------- src/executor/operator_utils.cpp | 28 ++------ src/model/column_boolean.cpp | 7 -- src/model/column_character.cpp | 7 -- src/model/column_date.cpp | 7 -- src/model/column_int128.cpp | 7 -- src/model/column_int16.cpp | 7 -- src/model/column_int32.cpp | 7 -- src/model/column_int64.cpp | 7 -- src/model/column_timestamp.cpp | 7 -- 21 files changed, 129 insertions(+), 164 deletions(-) create mode 100644 include/model/column_traits.h delete mode 100644 src/model/column_boolean.cpp delete mode 100644 src/model/column_character.cpp delete mode 100644 src/model/column_date.cpp delete mode 100644 src/model/column_int128.cpp delete mode 100644 src/model/column_int16.cpp delete mode 100644 src/model/column_int32.cpp delete mode 100644 src/model/column_int64.cpp delete mode 100644 src/model/column_timestamp.cpp diff --git a/include/model/column_boolean.h b/include/model/column_boolean.h index 33614e2..a60e7b3 100644 --- a/include/model/column_boolean.h +++ b/include/model/column_boolean.h @@ -7,7 +7,4 @@ class BooleanColumn final : public FixedColumn +#include +#include + +#include "common/error.h" +#include "common/int128.h" +#include "common/parsing.h" +#include "model/schema.h" + +template +struct ColumnValueTraits; + +template <> +struct ColumnValueTraits { + using Type = uint8_t; + + static Type Parse(const std::string& value) { return ParseBoolean(value) ? 1 : 0; } + static std::string ToString(const Type value) { return BooleanToString(value != 0); } +}; + +template <> +struct ColumnValueTraits { + using Type = int16_t; + + static Type Parse(const std::string& value) { return ParseInt16(value); } + static std::string ToString(const Type value) { return std::to_string(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = int32_t; + + static Type Parse(const std::string& value) { return ParseInt32(value); } + static std::string ToString(const Type value) { return std::to_string(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = int64_t; + + static Type Parse(const std::string& value) { return ParseInt64(value); } + static std::string ToString(const Type value) { return std::to_string(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = Int128; + + static Type Parse(const std::string& value) { return ParseInt128(value); } + static std::string ToString(const Type value) { return Int128ToString(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = int32_t; + + static Type Parse(const std::string& value) { return ParseDate(value); } + static std::string ToString(const Type value) { return DateToString(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = int64_t; + + static Type Parse(const std::string& value) { return ParseTimestamp(value); } + static std::string ToString(const Type value) { return TimestampToString(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = char; + + static Type Parse(const std::string& value) { return ParseCharacter(value); } + static std::string ToString(const Type value) { return std::string(1, value); } +}; + +template <> +struct ColumnValueTraits { + using Type = std::string; + + static Type Parse(const std::string& value) { return value; } + static std::string ToString(const Type& value) { return value; } +}; + +template +decltype(auto) VisitColumnType(const ColumnType type, Visitor&& visitor) { + switch (type) { + case ColumnType::Boolean: + return std::forward(visitor).template operator()(); + case ColumnType::Int16: + return std::forward(visitor).template operator()(); + case ColumnType::Int32: + return std::forward(visitor).template operator()(); + case ColumnType::Int64: + return std::forward(visitor).template operator()(); + case ColumnType::Int128: + return std::forward(visitor).template operator()(); + case ColumnType::String: + return std::forward(visitor).template operator()(); + case ColumnType::Date: + return std::forward(visitor).template operator()(); + case ColumnType::Timestamp: + return std::forward(visitor).template operator()(); + case ColumnType::Character: + return std::forward(visitor).template operator()(); + } + + throw Error::Unsupported("model", "unsupported column type"); +} diff --git a/include/model/fixed_column.h b/include/model/fixed_column.h index 63bc657..df904ef 100644 --- a/include/model/fixed_column.h +++ b/include/model/fixed_column.h @@ -5,6 +5,7 @@ #include "common/error.h" #include "io/stream.h" #include "model/column.h" +#include "model/column_traits.h" template class FixedColumn : public MutableColumn { @@ -19,6 +20,12 @@ class FixedColumn : public MutableColumn { size_t Size() const override { return values_.size(); } void Reserve(const size_t n) override { values_.reserve(n); } void Clear() override { values_.clear(); } + void AppendFromString(const std::string& value) override { + AppendValue(ColumnValueTraits::Parse(value)); + } + std::string ValueAsString(const size_t row) const override { + return ColumnValueTraits::ToString(ValueAt(row)); + } std::unique_ptr Clone() const override { return std::make_unique(static_cast(*this)); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7075555..22adee1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,15 +1,7 @@ add_library(columnar_engine_core model/batch.cpp model/column.cpp - model/column_boolean.cpp - model/column_character.cpp - model/column_date.cpp - model/column_int16.cpp - model/column_int32.cpp - model/column_int128.cpp - model/column_int64.cpp model/column_string.cpp - model/column_timestamp.cpp model/metadata.cpp io/batch.cpp io/file.cpp diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index b8e59b9..2c2b95e 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -10,63 +10,16 @@ #include "common/parsing.h" #include "executor/aggregate_function.h" #include "executor/aggregate_state.h" +#include "model/column_traits.h" static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view candidate, const std::string_view current, const bool is_min) { - switch (type) { - case ColumnType::Boolean: { - const bool left = ParseBoolean(std::string(candidate)); - const bool right = ParseBoolean(std::string(current)); - - return is_min ? left < right : left > right; - } - case ColumnType::Int16: { - const int16_t left = ParseInt16(std::string(candidate)); - const int16_t right = ParseInt16(std::string(current)); - - return is_min ? left < right : left > right; - } - case ColumnType::Int32: { - const int32_t left = ParseInt32(std::string(candidate)); - const int32_t right = ParseInt32(std::string(current)); - - return is_min ? left < right : left > right; - } - case ColumnType::Int64: { - const int64_t left = ParseInt64(std::string(candidate)); - const int64_t right = ParseInt64(std::string(current)); - - return is_min ? left < right : left > right; - } - case ColumnType::Int128: { - const Int128 left = ParseInt128(std::string(candidate)); - const Int128 right = ParseInt128(std::string(current)); - - return is_min ? left < right : left > right; - } - case ColumnType::String: - return is_min ? candidate < current : candidate > current; - case ColumnType::Date: { - const int32_t left = ParseDate(std::string(candidate)); - const int32_t right = ParseDate(std::string(current)); - - return is_min ? left < right : left > right; - } - case ColumnType::Timestamp: { - const int64_t left = ParseTimestamp(std::string(candidate)); - const int64_t right = ParseTimestamp(std::string(current)); - - return is_min ? left < right : left > right; - } - case ColumnType::Character: { - const char left = ParseCharacter(std::string(candidate)); - const char right = ParseCharacter(std::string(current)); - - return is_min ? left < right : left > right; - } - } + return VisitColumnType(type, [&]() { + const auto left = ColumnValueTraits::Parse(std::string(candidate)); + const auto right = ColumnValueTraits::Parse(std::string(current)); - throw Error::Unsupported("executor", "unsupported column type in comparison"); + return is_min ? left < right : left > right; + }); } static Int128 ParseToInt128(const ColumnType type, const std::string_view value) { diff --git a/src/executor/operator_utils.cpp b/src/executor/operator_utils.cpp index 57a88b8..21d0744 100644 --- a/src/executor/operator_utils.cpp +++ b/src/executor/operator_utils.cpp @@ -5,7 +5,7 @@ #include "common/ascii.h" #include "common/error.h" -#include "common/parsing.h" +#include "model/column_traits.h" static bool EvaluateOrdering(const std::strong_ordering ordering, const ComparisonKind kind) { if (kind == ComparisonKind::Equal) { @@ -41,28 +41,10 @@ static std::strong_ordering CompareValuesImpl(const T left, const T right) { } std::strong_ordering CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { - switch (type) { - case ColumnType::Boolean: - return CompareValuesImpl(ParseBoolean(std::string(lhs)), ParseBoolean(std::string(rhs))); - case ColumnType::Int16: - return CompareValuesImpl(ParseInt16(std::string(lhs)), ParseInt16(std::string(rhs))); - case ColumnType::Int32: - return CompareValuesImpl(ParseInt32(std::string(lhs)), ParseInt32(std::string(rhs))); - case ColumnType::Int64: - return CompareValuesImpl(ParseInt64(std::string(lhs)), ParseInt64(std::string(rhs))); - case ColumnType::Int128: - return CompareValuesImpl(ParseInt128(std::string(lhs)), ParseInt128(std::string(rhs))); - case ColumnType::String: - return CompareValuesImpl(std::string(lhs), std::string(rhs)); - case ColumnType::Date: - return CompareValuesImpl(ParseDate(std::string(lhs)), ParseDate(std::string(rhs))); - case ColumnType::Timestamp: - return CompareValuesImpl(ParseTimestamp(std::string(lhs)), ParseTimestamp(std::string(rhs))); - case ColumnType::Character: - return CompareValuesImpl(ParseCharacter(std::string(lhs)), ParseCharacter(std::string(rhs))); - } - - throw Error::Unsupported("executor", "unsupported column type in comparison"); + return VisitColumnType(type, [&]() { + return CompareValuesImpl(ColumnValueTraits::Parse(std::string(lhs)), + ColumnValueTraits::Parse(std::string(rhs))); + }); } bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, diff --git a/src/model/column_boolean.cpp b/src/model/column_boolean.cpp deleted file mode 100644 index 6146b25..0000000 --- a/src/model/column_boolean.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "model/column_boolean.h" - -#include "common/parsing.h" - -void BooleanColumn::AppendFromString(const std::string& value) { AppendValue(ParseBoolean(value) ? 1 : 0); } - -std::string BooleanColumn::ValueAsString(const size_t row) const { return BooleanToString(ValueAt(row) != 0); } diff --git a/src/model/column_character.cpp b/src/model/column_character.cpp deleted file mode 100644 index 447a5e6..0000000 --- a/src/model/column_character.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "model/column_character.h" - -#include "common/parsing.h" - -void CharacterColumn::AppendFromString(const std::string& value) { AppendValue(ParseCharacter(value)); } - -std::string CharacterColumn::ValueAsString(const size_t row) const { return std::string(1, ValueAt(row)); } diff --git a/src/model/column_date.cpp b/src/model/column_date.cpp deleted file mode 100644 index 3269575..0000000 --- a/src/model/column_date.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "model/column_date.h" - -#include "common/parsing.h" - -void DateColumn::AppendFromString(const std::string& value) { AppendValue(ParseDate(value)); } - -std::string DateColumn::ValueAsString(const size_t row) const { return DateToString(ValueAt(row)); } diff --git a/src/model/column_int128.cpp b/src/model/column_int128.cpp deleted file mode 100644 index 752a213..0000000 --- a/src/model/column_int128.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "model/column_int128.h" - -#include "common/parsing.h" - -void Int128Column::AppendFromString(const std::string& value) { AppendValue(ParseInt128(value)); } - -std::string Int128Column::ValueAsString(const size_t row) const { return Int128ToString(ValueAt(row)); } diff --git a/src/model/column_int16.cpp b/src/model/column_int16.cpp deleted file mode 100644 index 7a01fc5..0000000 --- a/src/model/column_int16.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "model/column_int16.h" - -#include "common/parsing.h" - -void Int16Column::AppendFromString(const std::string& value) { AppendValue(ParseInt16(value)); } - -std::string Int16Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_int32.cpp b/src/model/column_int32.cpp deleted file mode 100644 index 447708f..0000000 --- a/src/model/column_int32.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "model/column_int32.h" - -#include "common/parsing.h" - -void Int32Column::AppendFromString(const std::string& value) { AppendValue(ParseInt32(value)); } - -std::string Int32Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_int64.cpp b/src/model/column_int64.cpp deleted file mode 100644 index 911a196..0000000 --- a/src/model/column_int64.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "model/column_int64.h" - -#include "common/parsing.h" - -void Int64Column::AppendFromString(const std::string& value) { AppendValue(ParseInt64(value)); } - -std::string Int64Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_timestamp.cpp b/src/model/column_timestamp.cpp deleted file mode 100644 index d320957..0000000 --- a/src/model/column_timestamp.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "model/column_timestamp.h" - -#include "common/parsing.h" - -void TimestampColumn::AppendFromString(const std::string& value) { AppendValue(ParseTimestamp(value)); } - -std::string TimestampColumn::ValueAsString(const size_t row) const { return TimestampToString(ValueAt(row)); } From 8d4070fc39a889249c341fdf8ed8acc266aff933 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sun, 17 May 2026 16:01:18 +0300 Subject: [PATCH 27/29] upd: update dashboard --- README.md | 34 ++++++++++++------------ benchmarks/results/readme_benchmarks.csv | 34 ++++++++++++------------ 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index be0e72b..b094083 100644 --- a/README.md +++ b/README.md @@ -8,21 +8,21 @@ | Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | | --- | ---: | ---: | ---: | ---: | ---: | --- | -| [0](benchmarks/queries/query_0.sql) | 31.90 | 10.09 | 9.93 | 9.63 | 10.88 | ok | -| [1](benchmarks/queries/query_1.sql) | 47.47 | 29.97 | 29.35 | 28.25 | 32.94 | ok | -| [2](benchmarks/queries/query_2.sql) | 65.71 | 99.09 | 88.86 | 52.07 | 166.55 | ok | -| [3](benchmarks/queries/query_3.sql) | 102.02 | 63.90 | 64.27 | 59.52 | 67.55 | ok | -| [4](benchmarks/queries/query_4.sql) | 53.78 | 72.75 | 57.36 | 55.74 | 120.53 | ok | -| [5](benchmarks/queries/query_5.sql) | 64.92 | 47.45 | 47.18 | 44.87 | 50.55 | ok | -| [6](benchmarks/queries/query_6.sql) | 689.46 | 689.40 | 681.74 | 669.80 | 724.31 | ok | -| [7](benchmarks/queries/query_7.sql) | 32.64 | 28.93 | 28.98 | 28.51 | 29.27 | ok | -| [8](benchmarks/queries/query_8.sql) | 137.21 | 125.35 | 126.21 | 118.63 | 130.37 | ok | -| [9](benchmarks/queries/query_9.sql) | 181.60 | 185.20 | 182.91 | 177.70 | 197.28 | ok | -| [10](benchmarks/queries/query_10.sql) | 48.49 | 36.32 | 36.14 | 35.74 | 37.27 | ok | -| [11](benchmarks/queries/query_11.sql) | 52.50 | 40.40 | 40.20 | 38.94 | 42.24 | ok | -| [12](benchmarks/queries/query_12.sql) | 91.76 | 88.94 | 88.69 | 86.63 | 91.77 | ok | -| [13](benchmarks/queries/query_13.sql) | 100.89 | 97.84 | 96.94 | 94.84 | 102.63 | ok | -| [14](benchmarks/queries/query_14.sql) | 107.95 | 95.51 | 95.43 | 94.33 | 96.84 | ok | -| [15](benchmarks/queries/query_15.sql) | 175.84 | 169.96 | 169.44 | 164.33 | 176.64 | ok | -| [16](benchmarks/queries/query_16.sql) | 259.40 | 267.56 | 267.52 | 266.79 | 268.43 | ok | +| [0](benchmarks/queries/query_0.sql) | 1076.34 | 11.43 | 11.67 | 9.57 | 12.81 | ok | +| [1](benchmarks/queries/query_1.sql) | 41.93 | 27.20 | 27.21 | 27.19 | 27.22 | ok | +| [2](benchmarks/queries/query_2.sql) | 64.18 | 51.76 | 51.70 | 51.57 | 52.08 | ok | +| [3](benchmarks/queries/query_3.sql) | 69.35 | 51.94 | 52.19 | 51.02 | 52.38 | ok | +| [4](benchmarks/queries/query_4.sql) | 44.52 | 43.83 | 43.77 | 43.72 | 44.07 | ok | +| [5](benchmarks/queries/query_5.sql) | 59.39 | 48.10 | 45.38 | 44.72 | 56.93 | ok | +| [6](benchmarks/queries/query_6.sql) | 757.49 | 679.54 | 678.98 | 676.05 | 684.13 | ok | +| [7](benchmarks/queries/query_7.sql) | 28.79 | 28.27 | 28.27 | 28.13 | 28.44 | ok | +| [8](benchmarks/queries/query_8.sql) | 126.09 | 112.90 | 112.57 | 111.87 | 114.58 | ok | +| [9](benchmarks/queries/query_9.sql) | 164.71 | 166.90 | 166.97 | 164.56 | 169.11 | ok | +| [10](benchmarks/queries/query_10.sql) | 51.34 | 37.20 | 37.17 | 37.06 | 37.38 | ok | +| [11](benchmarks/queries/query_11.sql) | 50.58 | 38.50 | 38.49 | 38.39 | 38.65 | ok | +| [12](benchmarks/queries/query_12.sql) | 63.54 | 64.18 | 64.19 | 63.69 | 64.67 | ok | +| [13](benchmarks/queries/query_13.sql) | 96.63 | 83.26 | 76.40 | 76.12 | 104.12 | ok | +| [14](benchmarks/queries/query_14.sql) | 83.27 | 113.13 | 87.14 | 72.80 | 205.45 | ok | +| [15](benchmarks/queries/query_15.sql) | 105.19 | 108.28 | 108.08 | 104.50 | 112.46 | ok | +| [16](benchmarks/queries/query_16.sql) | 204.87 | 201.82 | 200.27 | 199.42 | 207.34 | ok | diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index 2595e32..1f96c84 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,18 +1,18 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,31.90,10.09,9.93,9.63,10.88,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,47.47,29.97,29.35,28.25,32.94,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",65.71,99.09,88.86,52.07,166.55,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,102.02,63.90,64.27,59.52,67.55,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,53.78,72.75,57.36,55.74,120.53,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,64.92,47.45,47.18,44.87,50.55,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",689.46,689.40,681.74,669.80,724.31,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",32.64,28.93,28.98,28.51,29.27,ok, -8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",137.21,125.35,126.21,118.63,130.37,ok, -9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",181.60,185.20,182.91,177.70,197.28,ok, -10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",48.49,36.32,36.14,35.74,37.27,ok, -11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",52.50,40.40,40.20,38.94,42.24,ok, -12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",91.76,88.94,88.69,86.63,91.77,ok, -13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",100.89,97.84,96.94,94.84,102.63,ok, -14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",107.95,95.51,95.43,94.33,96.84,ok, -15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",175.84,169.96,169.44,164.33,176.64,ok, -16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",259.40,267.56,267.52,266.79,268.43,ok, +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,1076.34,11.43,11.67,9.57,12.81,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,41.93,27.20,27.21,27.19,27.22,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",64.18,51.76,51.70,51.57,52.08,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,69.35,51.94,52.19,51.02,52.38,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,44.52,43.83,43.77,43.72,44.07,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,59.39,48.10,45.38,44.72,56.93,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",757.49,679.54,678.98,676.05,684.13,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",28.79,28.27,28.27,28.13,28.44,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",126.09,112.90,112.57,111.87,114.58,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",164.71,166.90,166.97,164.56,169.11,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",51.34,37.20,37.17,37.06,37.38,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",50.58,38.50,38.49,38.39,38.65,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",63.54,64.18,64.19,63.69,64.67,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",96.63,83.26,76.40,76.12,104.12,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",83.27,113.13,87.14,72.80,205.45,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",105.19,108.28,108.08,104.50,112.46,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",204.87,201.82,200.27,199.42,207.34,ok, From eaee82c5475724b0d41dea8bb56102d10117dcb5 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sun, 17 May 2026 16:04:07 +0300 Subject: [PATCH 28/29] fix: added test to test job --- .github/workflows/cmake-single-platform.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cmake-single-platform.yml b/.github/workflows/cmake-single-platform.yml index f414afd..b77e5dd 100644 --- a/.github/workflows/cmake-single-platform.yml +++ b/.github/workflows/cmake-single-platform.yml @@ -35,6 +35,7 @@ jobs: ./tests/test_columnar ./tests/test_columns ./tests/test_csv + ./tests/test_executor ./tests/test_fileio ./tests/test_stream ./tests/test_tokenizer From 107d65a75b0c4b5fb405692b86c8b8816ef31c92 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Sun, 17 May 2026 16:06:48 +0300 Subject: [PATCH 29/29] fix: std::chrono::sys_time issue --- src/common/parsing.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/parsing.cpp b/src/common/parsing.cpp index 3a83c72..302b9dd 100644 --- a/src/common/parsing.cpp +++ b/src/common/parsing.cpp @@ -350,7 +350,7 @@ std::string DateToString(const int32_t value) { } std::string TimestampToString(const int64_t value) { - const std::chrono::sys_time timestamp{std::chrono::microseconds{value}}; + const std::chrono::sys_time timestamp{std::chrono::microseconds{value}}; const auto day_point = std::chrono::floor(timestamp); const std::chrono::hh_mm_ss tod{timestamp - day_point};