diff --git a/.clang-tidy b/.clang-tidy index 3d705b4..26f6e6d 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,5 @@ --- -Checks: '-*,cppcoreguidelines-avoid-goto,cppcoreguidelines-pro-type-const-cast, google-readability-casting, modernize-replace-random-shuffle, readability-braces-around-statements, readability-container-size-empty, readability-redundant-control-flow, readability-redundant-string-init, modernize-use-nullptr, readability-identifier-naming, google-build-using-namespace' +Checks: '-*,cppcoreguidelines-avoid-goto,cppcoreguidelines-pro-type-const-cast, google-readability-casting, modernize-replace-random-shuffle, readability-braces-around-statements, readability-container-size-empty, readability-redundant-control-flow, readability-redundant-string-init, modernize-use-nullptr, readability-identifier-naming, google-build-using-namespace' HeaderFilterRegex: '\.h$' WarningsAsErrors: '*' CheckOptions: @@ -25,17 +25,11 @@ CheckOptions: value: lower_case - key: readability-identifier-naming.ParameterCase value: lower_case - - key: readability-identifier-naming.GlobalConstantPrefix - value: k - key: readability-identifier-naming.GlobalConstantCase value: CamelCase - - key: readability-identifier-naming.StaticConstantPrefix - value: k - key: readability-identifier-naming.StaticConstantCase value: CamelCase - key: readability-identifier-naming.ConstexprVariableCase value: CamelCase - - key: readability-identifier-naming.ConstexprVariablePrefix - value: k - key: google-runtime-int.TypeSuffix value: _t \ No newline at end of file diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4d761dd --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +trim_trailing_whitespace = false diff --git a/.github/workflows/cmake-single-platform.yml b/.github/workflows/cmake-single-platform.yml index 5826612..b77e5dd 100644 --- a/.github/workflows/cmake-single-platform.yml +++ b/.github/workflows/cmake-single-platform.yml @@ -35,10 +35,16 @@ jobs: ./tests/test_columnar ./tests/test_columns ./tests/test_csv + ./tests/test_executor ./tests/test_fileio ./tests/test_stream ./tests/test_tokenizer + - name: Benchmark Queries + run: | + cd benchmarks + ../build/benchmarks/benchmark_queries + sanitizers: runs-on: ubuntu-latest @@ -70,3 +76,11 @@ jobs: run: | cd build-sanitizers ctest --output-on-failure + + - name: Benchmark Queries With Sanitizers + env: + ASAN_OPTIONS: detect_leaks=1:strict_string_checks=1:check_initialization_order=1 + UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 + run: | + cd benchmarks + ../build-sanitizers/benchmarks/benchmark_queries diff --git a/.gitignore b/.gitignore index 3eef705..14918bf 100644 --- a/.gitignore +++ b/.gitignore @@ -104,7 +104,6 @@ examples/out_columnar # benchmarks benchmarks/hits_sample.csv benchmarks/hits_sample_new.csv -benchmarks/schema_sample.csv benchmarks/schema_sample_new.csv benchmarks/hits_sample.columnar benchmarks/benchmark_csv_to_columnar diff --git a/CMakeLists.txt b/CMakeLists.txt index aeeaf52..fbd4b6c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,18 +14,10 @@ include(cmake/dependencies_package.cmake) include(cmake/sanitizers.cmake) set(COLUMNAR_INCLUDE_DIRS - ${PROJECT_SOURCE_DIR}/include/convert - ${PROJECT_SOURCE_DIR}/include/io - ${PROJECT_SOURCE_DIR}/include/model - ${PROJECT_SOURCE_DIR}/include/support - ${PROJECT_SOURCE_DIR}/include/testing - ${PROJECT_SOURCE_DIR}/include/executor - ${PROJECT_SOURCE_DIR}/include/sql_parser - ${PROJECT_SOURCE_DIR}/include/tl + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/third-party ) -include_directories(${COLUMNAR_INCLUDE_DIRS}) - add_compile_options(-Wall -Wextra -Werror -Wpedantic -O2) set(COLUMNAR_SANITIZERS "" CACHE STRING "Comma or semicolon separated list of sanitizers: address,undefined,leak,thread,memory") diff --git a/README.md b/README.md index 94cd7d0..b094083 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,28 @@ # columnar-engine -Колононый движок с нуля (in progress) \ No newline at end of file +Колононый движок с нуля (in progress) + + +## Dashboard + + +| Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| [0](benchmarks/queries/query_0.sql) | 1076.34 | 11.43 | 11.67 | 9.57 | 12.81 | ok | +| [1](benchmarks/queries/query_1.sql) | 41.93 | 27.20 | 27.21 | 27.19 | 27.22 | ok | +| [2](benchmarks/queries/query_2.sql) | 64.18 | 51.76 | 51.70 | 51.57 | 52.08 | ok | +| [3](benchmarks/queries/query_3.sql) | 69.35 | 51.94 | 52.19 | 51.02 | 52.38 | ok | +| [4](benchmarks/queries/query_4.sql) | 44.52 | 43.83 | 43.77 | 43.72 | 44.07 | ok | +| [5](benchmarks/queries/query_5.sql) | 59.39 | 48.10 | 45.38 | 44.72 | 56.93 | ok | +| [6](benchmarks/queries/query_6.sql) | 757.49 | 679.54 | 678.98 | 676.05 | 684.13 | ok | +| [7](benchmarks/queries/query_7.sql) | 28.79 | 28.27 | 28.27 | 28.13 | 28.44 | ok | +| [8](benchmarks/queries/query_8.sql) | 126.09 | 112.90 | 112.57 | 111.87 | 114.58 | ok | +| [9](benchmarks/queries/query_9.sql) | 164.71 | 166.90 | 166.97 | 164.56 | 169.11 | ok | +| [10](benchmarks/queries/query_10.sql) | 51.34 | 37.20 | 37.17 | 37.06 | 37.38 | ok | +| [11](benchmarks/queries/query_11.sql) | 50.58 | 38.50 | 38.49 | 38.39 | 38.65 | ok | +| [12](benchmarks/queries/query_12.sql) | 63.54 | 64.18 | 64.19 | 63.69 | 64.67 | ok | +| [13](benchmarks/queries/query_13.sql) | 96.63 | 83.26 | 76.40 | 76.12 | 104.12 | ok | +| [14](benchmarks/queries/query_14.sql) | 83.27 | 113.13 | 87.14 | 72.80 | 205.45 | ok | +| [15](benchmarks/queries/query_15.sql) | 105.19 | 108.28 | 108.08 | 104.50 | 112.46 | ok | +| [16](benchmarks/queries/query_16.sql) | 204.87 | 201.82 | 200.27 | 199.42 | 207.34 | ok | + diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 15f0bdc..7fe3cd2 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -9,6 +9,9 @@ target_link_libraries(benchmark_csv_to_columnar add_executable(benchmark_queries benchmark_queries.cpp + support/query_result_checks.cpp + support/query_result_checks.h + support/query_runners.h ) target_link_libraries(benchmark_queries @@ -19,7 +22,7 @@ target_link_libraries(benchmark_queries target_compile_definitions(benchmark_csv_to_columnar PRIVATE COLUMNAR_BENCHMARK_DEFAULT_DATA="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample.csv" - COLUMNAR_BENCHMARK_DEFAULT_SCHEMA="${PROJECT_SOURCE_DIR}/benchmarks/schema_sample.csv" + COLUMNAR_BENCHMARK_DEFAULT_SCHEMA="${PROJECT_SOURCE_DIR}/benchmarks/scheme.csv" COLUMNAR_BENCHMARK_DEFAULT_OUTPUT="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample.columnar" COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_DATA="${PROJECT_SOURCE_DIR}/benchmarks/hits_sample_new.csv" COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA="${PROJECT_SOURCE_DIR}/benchmarks/schema_sample_new.csv" diff --git a/benchmarks/benchmark_csv_to_columnar.cpp b/benchmarks/benchmark_csv_to_columnar.cpp index 77632ba..3db1be1 100644 --- a/benchmarks/benchmark_csv_to_columnar.cpp +++ b/benchmarks/benchmark_csv_to_columnar.cpp @@ -4,9 +4,9 @@ #include #include -#include "csv.h" -#include "csv_columnar.h" -#include "error.h" +#include "io/csv.h" +#include "convert/csv_columnar.h" +#include "common/error.h" #ifndef COLUMNAR_BENCHMARK_DEFAULT_DATA #define COLUMNAR_BENCHMARK_DEFAULT_DATA "benchmarks/hits_sample.csv" @@ -28,7 +28,7 @@ #define COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA "benchmarks/schema_sample_new.csv" #endif -static constexpr size_t kRowsPerGroup = 50000; +static constexpr size_t RowsPerGroup = 50000; struct CsvCompareResult { bool equal = true; @@ -64,7 +64,7 @@ static CsvCompareResult CompareCsvFiles(const std::filesystem::path& lhs_path, c int main(const int argc, char** argv) { try { - size_t rows_per_group = kRowsPerGroup; + size_t rows_per_group = RowsPerGroup; if (argc == 2) { rows_per_group = std::stoull(argv[1]); } diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index 90e8a79..64cf866 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -1,46 +1,112 @@ +#include +#include #include +#include #include #include -#include "fileio.h" +#include "executor/executor.h" +#include "io/file.h" +#include "support/query_result_checks.h" +#include "support/query_runners.h" #ifndef COLUMNAR_BENCHMARK_QUERIES_DIR #define COLUMNAR_BENCHMARK_QUERIES_DIR "benchmarks/queries" #endif -static constexpr int kFirstQueryId = 0; -static constexpr int kLastQueryId = 42; +static constexpr int FirstQueryId = 0; +static constexpr int LastQueryId = 16; -void RunQuerySkeleton(const std::filesystem::path& path, const int query_id) { - const std::string sql = ReadTextFile(path); - std::cout << "query_" << query_id << ": found, bytes=" << sql.size() << ", status=skeleton_only" << std::endl; +void RunQuery(const Executor& executor, const std::filesystem::path& path, const int query_id, bool record = false) { + const auto& runners = GetQueryRunners(); + const auto it = runners.find(query_id); + const std::string query_sql = std::filesystem::exists(path) ? ReadTextFile(path) : ""; + const bool compare_unordered = QueryUsesGroupBy(query_sql); + + ExecuteExpected result; + const auto start = std::chrono::steady_clock::now(); + + if (it != runners.end()) { + result = it->second(executor); + } else if (std::filesystem::exists(path)) { + result = executor.Execute(query_sql); + } else { + std::cout << "query_" << query_id << ": missing, status=skipped" << std::endl; + return; + } + + const auto end = std::chrono::steady_clock::now(); + const auto duration_ms = std::chrono::duration_cast(end - start).count(); + + if (!result.has_value()) { + std::cout << "query_" << query_id << ": error=\"" << result.error().GetMessage() << "\", status=failed" + << std::endl; + return; + } + + const std::string actual_result = BatchToString(*result); + const std::filesystem::path ref_path = + std::filesystem::path(COLUMNAR_BENCHMARK_QUERIES_DIR) / ("query_" + std::to_string(query_id) + ".csv"); + + std::string status; + + if (record) { + std::ofstream ref_file(ref_path); + ref_file << actual_result; + status = "recorded"; + } else if (std::filesystem::exists(ref_path)) { + const std::string expected_result = ReadTextFile(ref_path); + if (compare_unordered ? EqualAsRowMultisets(actual_result, expected_result) + : actual_result == expected_result) { + status = "passed"; + } else { + status = "failed"; + std::cout << "query_" << query_id << ": expected:\n" << expected_result; + if (!expected_result.empty() && expected_result.back() != '\n') { + std::cout << std::endl; + } + std::cout << "query_" << query_id << ": actual:\n" << actual_result; + if (!actual_result.empty() && actual_result.back() != '\n') { + std::cout << std::endl; + } + } + } else { + status = "no_reference"; + } + + std::cout << "query_" << query_id << ": rows=" << result->RowsCount() << ", time=" << duration_ms + << "ms, status=" << status << std::endl; } -int main() { +int main(const int argc, char** argv) { try { + bool record = false; + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "--record") { + record = true; + } + } + const std::filesystem::path queries_dir = COLUMNAR_BENCHMARK_QUERIES_DIR; + const std::filesystem::path data_path = queries_dir.parent_path() / "hits_sample.columnar"; + + Executor executor; + executor.RegisterTable("hits", data_path); size_t found_queries = 0; - size_t missing_queries = 0; std::cout << "queries_dir=" << queries_dir << std::endl; - for (int query_id = kFirstQueryId; query_id <= kLastQueryId; ++query_id) { + for (int query_id = FirstQueryId; query_id <= LastQueryId; ++query_id) { const std::filesystem::path query_path = queries_dir / ("query_" + std::to_string(query_id) + ".sql"); - if (!std::filesystem::exists(query_path)) { - ++missing_queries; - std::cout << "query_" << query_id << ": missing, status=skipped" << std::endl; - continue; + if (GetQueryRunners().contains(query_id) || std::filesystem::exists(query_path)) { + ++found_queries; + RunQuery(executor, query_path, query_id, record); } - - ++found_queries; - - RunQuerySkeleton(query_path, query_id); } - std::cout << "found_queries=" << found_queries << std::endl; - std::cout << "missing_queries=" << missing_queries << std::endl; + std::cout << "total_executed=" << found_queries << std::endl; } catch (const std::exception& ex) { std::cerr << ex.what() << std::endl; return 1; diff --git a/benchmarks/queries/queries.sql b/benchmarks/queries/queries.sql new file mode 100644 index 0000000..49dd47c --- /dev/null +++ b/benchmarks/queries/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; +SELECT AVG(UserID) FROM hits; +SELECT COUNT(DISTINCT UserID) FROM hits; +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; +SELECT MIN(EventDate), MAX(EventDate) FROM hits; +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID FROM hits WHERE UserID = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; +SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; \ No newline at end of file diff --git a/benchmarks/queries/query_0.csv b/benchmarks/queries/query_0.csv new file mode 100644 index 0000000..4b9dfa7 --- /dev/null +++ b/benchmarks/queries/query_0.csv @@ -0,0 +1 @@ +999977 diff --git a/benchmarks/queries/query_1.csv b/benchmarks/queries/query_1.csv new file mode 100644 index 0000000..f8cb17a --- /dev/null +++ b/benchmarks/queries/query_1.csv @@ -0,0 +1 @@ +14174 diff --git a/benchmarks/queries/query_10.csv b/benchmarks/queries/query_10.csv new file mode 100644 index 0000000..4734f82 --- /dev/null +++ b/benchmarks/queries/query_10.csv @@ -0,0 +1,10 @@ +iPad,2303 +iPhone,107 +A500,34 +N8-00,12 +GT-P7300B,12 +iPho,11 +3110000,6 +IQ245Plus,5 +eagle75,4 +GT-S5830,3 diff --git a/benchmarks/queries/query_10.sql b/benchmarks/queries/query_10.sql new file mode 100644 index 0000000..d0ea7e3 --- /dev/null +++ b/benchmarks/queries/query_10.sql @@ -0,0 +1 @@ +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_11.csv b/benchmarks/queries/query_11.csv new file mode 100644 index 0000000..3f24c76 --- /dev/null +++ b/benchmarks/queries/query_11.csv @@ -0,0 +1,10 @@ +1,iPad,1967 +5,iPad,97 +7,iPad,79 +6,iPad,55 +6,iPhone,37 +26,iPhone,36 +118,A500,34 +32,iPad,29 +60,iPad,22 +13,iPad,12 diff --git a/benchmarks/queries/query_11.sql b/benchmarks/queries/query_11.sql new file mode 100644 index 0000000..2a316d1 --- /dev/null +++ b/benchmarks/queries/query_11.sql @@ -0,0 +1 @@ +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_12.csv b/benchmarks/queries/query_12.csv new file mode 100644 index 0000000..2dbd091 --- /dev/null +++ b/benchmarks/queries/query_12.csv @@ -0,0 +1,10 @@ +ведомосквы вместу,4943 +ведомосквы вы из,2471 +ведомосквиталия страции,2026 +ведомосковский,1686 +покеты рецепт засня,961 +рецепты сбербан,788 +авторий,705 +ведомосква,446 +ведомосквы новые водительная болгарин,411 +инстанец жизнь,391 diff --git a/benchmarks/queries/query_12.sql b/benchmarks/queries/query_12.sql new file mode 100644 index 0000000..7a02546 --- /dev/null +++ b/benchmarks/queries/query_12.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_13.csv b/benchmarks/queries/query_13.csv new file mode 100644 index 0000000..0dc3e1d --- /dev/null +++ b/benchmarks/queries/query_13.csv @@ -0,0 +1,10 @@ +ведомосквы вместу,1381 +ведомосквы вы из,678 +ведомосквиталия страции,658 +рецепты сбербан,594 +ведомосковский,407 +инстанец жизнь,292 +покеты рецепт засня,281 +авторий,196 +рецепт блиноленские,135 +ведомосква,129 diff --git a/benchmarks/queries/query_13.sql b/benchmarks/queries/query_13.sql new file mode 100644 index 0000000..4ce0fee --- /dev/null +++ b/benchmarks/queries/query_13.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_14.csv b/benchmarks/queries/query_14.csv new file mode 100644 index 0000000..c40691f --- /dev/null +++ b/benchmarks/queries/query_14.csv @@ -0,0 +1,10 @@ +2,ведомосквы вместу,3478 +2,ведомосквы вы из,1857 +2,ведомосковский,1682 +2,ведомосквиталия страции,1434 +4,покеты рецепт засня,959 +2,рецепты сбербан,737 +3,ведомосквы вместу,660 +2,авторий,576 +3,ведомосквиталия страции,494 +4,ведомосквы вместу,442 diff --git a/benchmarks/queries/query_14.sql b/benchmarks/queries/query_14.sql new file mode 100644 index 0000000..8e85255 --- /dev/null +++ b/benchmarks/queries/query_14.sql @@ -0,0 +1 @@ +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_15.csv b/benchmarks/queries/query_15.csv new file mode 100644 index 0000000..32d99e7 --- /dev/null +++ b/benchmarks/queries/query_15.csv @@ -0,0 +1,10 @@ +1508127196834704092,1303 +3205616454965152970,949 +502693359570399458,893 +873022393995828557,876 +2256536417172705921,695 +340634745528635910,610 +72709437341035504,560 +5705194083846317709,532 +1257144732630861346,524 +4885305169967046117,516 diff --git a/benchmarks/queries/query_15.sql b/benchmarks/queries/query_15.sql new file mode 100644 index 0000000..f959e98 --- /dev/null +++ b/benchmarks/queries/query_15.sql @@ -0,0 +1 @@ +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/benchmarks/queries/query_16.csv b/benchmarks/queries/query_16.csv new file mode 100644 index 0000000..43d129b --- /dev/null +++ b/benchmarks/queries/query_16.csv @@ -0,0 +1,10 @@ +1508127196834704092,,1303 +3205616454965152970,,949 +502693359570399458,,893 +873022393995828557,,876 +2256536417172705921,,695 +340634745528635910,,610 +72709437341035504,,560 +5705194083846317709,,532 +614605011960296602,,506 +775643969820522877,,483 diff --git a/benchmarks/queries/query_16.sql b/benchmarks/queries/query_16.sql new file mode 100644 index 0000000..50b1f38 --- /dev/null +++ b/benchmarks/queries/query_16.sql @@ -0,0 +1 @@ +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/benchmarks/queries/query_17.sql b/benchmarks/queries/query_17.sql new file mode 100644 index 0000000..454cdb5 --- /dev/null +++ b/benchmarks/queries/query_17.sql @@ -0,0 +1 @@ +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; diff --git a/benchmarks/queries/query_18.sql b/benchmarks/queries/query_18.sql new file mode 100644 index 0000000..6520cc5 --- /dev/null +++ b/benchmarks/queries/query_18.sql @@ -0,0 +1 @@ +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/benchmarks/queries/query_19.sql b/benchmarks/queries/query_19.sql new file mode 100644 index 0000000..3f84066 --- /dev/null +++ b/benchmarks/queries/query_19.sql @@ -0,0 +1 @@ +SELECT UserID FROM hits WHERE UserID = 435090932899640449; diff --git a/benchmarks/queries/query_2.csv b/benchmarks/queries/query_2.csv new file mode 100644 index 0000000..49e722f --- /dev/null +++ b/benchmarks/queries/query_2.csv @@ -0,0 +1 @@ +80778,999977,1604 diff --git a/benchmarks/queries/query_20.sql b/benchmarks/queries/query_20.sql new file mode 100644 index 0000000..4426afa --- /dev/null +++ b/benchmarks/queries/query_20.sql @@ -0,0 +1 @@ +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; diff --git a/benchmarks/queries/query_21.sql b/benchmarks/queries/query_21.sql new file mode 100644 index 0000000..bd28609 --- /dev/null +++ b/benchmarks/queries/query_21.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_22.sql b/benchmarks/queries/query_22.sql new file mode 100644 index 0000000..4ee87ac --- /dev/null +++ b/benchmarks/queries/query_22.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_23.sql b/benchmarks/queries/query_23.sql new file mode 100644 index 0000000..935169e --- /dev/null +++ b/benchmarks/queries/query_23.sql @@ -0,0 +1 @@ +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; diff --git a/benchmarks/queries/query_24.sql b/benchmarks/queries/query_24.sql new file mode 100644 index 0000000..1bcfd4c --- /dev/null +++ b/benchmarks/queries/query_24.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; diff --git a/benchmarks/queries/query_25.sql b/benchmarks/queries/query_25.sql new file mode 100644 index 0000000..58ea761 --- /dev/null +++ b/benchmarks/queries/query_25.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; diff --git a/benchmarks/queries/query_26.sql b/benchmarks/queries/query_26.sql new file mode 100644 index 0000000..88ed7ba --- /dev/null +++ b/benchmarks/queries/query_26.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; diff --git a/benchmarks/queries/query_27.sql b/benchmarks/queries/query_27.sql new file mode 100644 index 0000000..8f89977 --- /dev/null +++ b/benchmarks/queries/query_27.sql @@ -0,0 +1 @@ +SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/benchmarks/queries/query_28.sql b/benchmarks/queries/query_28.sql new file mode 100644 index 0000000..f55fac1 --- /dev/null +++ b/benchmarks/queries/query_28.sql @@ -0,0 +1 @@ +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/benchmarks/queries/query_29.sql b/benchmarks/queries/query_29.sql new file mode 100644 index 0000000..630450b --- /dev/null +++ b/benchmarks/queries/query_29.sql @@ -0,0 +1 @@ +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; diff --git a/benchmarks/queries/query_3.csv b/benchmarks/queries/query_3.csv new file mode 100644 index 0000000..c7b6e3a --- /dev/null +++ b/benchmarks/queries/query_3.csv @@ -0,0 +1 @@ +1948165676197850120 diff --git a/benchmarks/queries/query_30.sql b/benchmarks/queries/query_30.sql new file mode 100644 index 0000000..f8a80d9 --- /dev/null +++ b/benchmarks/queries/query_30.sql @@ -0,0 +1 @@ +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_31.sql b/benchmarks/queries/query_31.sql new file mode 100644 index 0000000..ba8ac2f --- /dev/null +++ b/benchmarks/queries/query_31.sql @@ -0,0 +1 @@ +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_32.sql b/benchmarks/queries/query_32.sql new file mode 100644 index 0000000..893773a --- /dev/null +++ b/benchmarks/queries/query_32.sql @@ -0,0 +1 @@ +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_33.sql b/benchmarks/queries/query_33.sql new file mode 100644 index 0000000..f00a3e8 --- /dev/null +++ b/benchmarks/queries/query_33.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_34.sql b/benchmarks/queries/query_34.sql new file mode 100644 index 0000000..2137530 --- /dev/null +++ b/benchmarks/queries/query_34.sql @@ -0,0 +1 @@ +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_35.sql b/benchmarks/queries/query_35.sql new file mode 100644 index 0000000..581e5e3 --- /dev/null +++ b/benchmarks/queries/query_35.sql @@ -0,0 +1 @@ +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/queries/query_36.sql b/benchmarks/queries/query_36.sql new file mode 100644 index 0000000..d6e06ce --- /dev/null +++ b/benchmarks/queries/query_36.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; diff --git a/benchmarks/queries/query_37.sql b/benchmarks/queries/query_37.sql new file mode 100644 index 0000000..f63f7a8 --- /dev/null +++ b/benchmarks/queries/query_37.sql @@ -0,0 +1 @@ +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; diff --git a/benchmarks/queries/query_38.sql b/benchmarks/queries/query_38.sql new file mode 100644 index 0000000..498db57 --- /dev/null +++ b/benchmarks/queries/query_38.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; diff --git a/benchmarks/queries/query_39.sql b/benchmarks/queries/query_39.sql new file mode 100644 index 0000000..9d3d319 --- /dev/null +++ b/benchmarks/queries/query_39.sql @@ -0,0 +1 @@ +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; diff --git a/benchmarks/queries/query_4.csv b/benchmarks/queries/query_4.csv new file mode 100644 index 0000000..d3d401f --- /dev/null +++ b/benchmarks/queries/query_4.csv @@ -0,0 +1 @@ +79842 diff --git a/benchmarks/queries/query_40.sql b/benchmarks/queries/query_40.sql new file mode 100644 index 0000000..f54f6db --- /dev/null +++ b/benchmarks/queries/query_40.sql @@ -0,0 +1 @@ +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; diff --git a/benchmarks/queries/query_41.sql b/benchmarks/queries/query_41.sql new file mode 100644 index 0000000..dc1b7e6 --- /dev/null +++ b/benchmarks/queries/query_41.sql @@ -0,0 +1 @@ +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; diff --git a/benchmarks/queries/query_42.sql b/benchmarks/queries/query_42.sql new file mode 100644 index 0000000..4ac2815 --- /dev/null +++ b/benchmarks/queries/query_42.sql @@ -0,0 +1 @@ +SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; diff --git a/benchmarks/queries/query_5.csv b/benchmarks/queries/query_5.csv new file mode 100644 index 0000000..d2c24b0 --- /dev/null +++ b/benchmarks/queries/query_5.csv @@ -0,0 +1 @@ +18316 diff --git a/benchmarks/queries/query_5.sql b/benchmarks/queries/query_5.sql new file mode 100644 index 0000000..e9615f8 --- /dev/null +++ b/benchmarks/queries/query_5.sql @@ -0,0 +1 @@ +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; diff --git a/benchmarks/queries/query_6.csv b/benchmarks/queries/query_6.csv new file mode 100644 index 0000000..b72aa99 --- /dev/null +++ b/benchmarks/queries/query_6.csv @@ -0,0 +1 @@ +2013-07-15,2013-07-15 diff --git a/benchmarks/queries/query_6.sql b/benchmarks/queries/query_6.sql new file mode 100644 index 0000000..03fbb82 --- /dev/null +++ b/benchmarks/queries/query_6.sql @@ -0,0 +1 @@ +SELECT MIN(EventDate), MAX(EventDate) FROM hits; diff --git a/benchmarks/queries/query_7.csv b/benchmarks/queries/query_7.csv new file mode 100644 index 0000000..0e1c4d4 --- /dev/null +++ b/benchmarks/queries/query_7.csv @@ -0,0 +1,5 @@ +2,9543 +13,4592 +52,34 +50,4 +28,1 diff --git a/benchmarks/queries/query_7.sql b/benchmarks/queries/query_7.sql new file mode 100644 index 0000000..25a10b1 --- /dev/null +++ b/benchmarks/queries/query_7.sql @@ -0,0 +1 @@ +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; diff --git a/benchmarks/queries/query_8.csv b/benchmarks/queries/query_8.csv new file mode 100644 index 0000000..5f52dd6 --- /dev/null +++ b/benchmarks/queries/query_8.csv @@ -0,0 +1,10 @@ +229,27961 +2,10413 +208,3073 +1,1720 +34,1428 +158,1110 +184,987 +107,966 +42,956 +47,943 diff --git a/benchmarks/queries/query_8.sql b/benchmarks/queries/query_8.sql new file mode 100644 index 0000000..a6bf47e --- /dev/null +++ b/benchmarks/queries/query_8.sql @@ -0,0 +1 @@ +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; diff --git a/benchmarks/queries/query_9.csv b/benchmarks/queries/query_9.csv new file mode 100644 index 0000000..355f967 --- /dev/null +++ b/benchmarks/queries/query_9.csv @@ -0,0 +1,10 @@ +229,38044,426412,1612,27961 +2,12801,148193,1593,10413 +208,2673,30614,1490,3073 +1,1802,28577,1623,1720 +34,508,14329,1592,1428 +47,1041,13661,1637,943 +158,78,13294,1576,1110 +7,1166,11679,1627,647 +42,642,11547,1625,956 +184,30,10157,1614,987 diff --git a/benchmarks/queries/query_9.sql b/benchmarks/queries/query_9.sql new file mode 100644 index 0000000..f4a9ee3 --- /dev/null +++ b/benchmarks/queries/query_9.sql @@ -0,0 +1 @@ +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv new file mode 100644 index 0000000..1f96c84 --- /dev/null +++ b/benchmarks/results/readme_benchmarks.csv @@ -0,0 +1,18 @@ +query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,status,error +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,1076.34,11.43,11.67,9.57,12.81,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,41.93,27.20,27.21,27.19,27.22,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",64.18,51.76,51.70,51.57,52.08,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,69.35,51.94,52.19,51.02,52.38,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,44.52,43.83,43.77,43.72,44.07,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,59.39,48.10,45.38,44.72,56.93,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",757.49,679.54,678.98,676.05,684.13,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",28.79,28.27,28.27,28.13,28.44,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",126.09,112.90,112.57,111.87,114.58,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",164.71,166.90,166.97,164.56,169.11,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",51.34,37.20,37.17,37.06,37.38,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",50.58,38.50,38.49,38.39,38.65,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",63.54,64.18,64.19,63.69,64.67,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",96.63,83.26,76.40,76.12,104.12,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",83.27,113.13,87.14,72.80,205.45,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",105.19,108.28,108.08,104.50,112.46,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",204.87,201.82,200.27,199.42,207.34,ok, diff --git a/benchmarks/scheme.csv b/benchmarks/scheme.csv new file mode 100644 index 0000000..881bb54 --- /dev/null +++ b/benchmarks/scheme.csv @@ -0,0 +1,105 @@ +WatchID,int64 +JavaEnable,int16 +Title,string +GoodEvent,int16 +EventTime,TIMESTAMP +EventDate,DATE +CounterID,int32 +ClientIP,int32 +RegionID,int32 +UserID,int64 +CounterClass,int16 +OS,int16 +UserAgent,int16 +URL,string +Referer,string +IsRefresh,int16 +RefererCategoryID,int16 +RefererRegionID,int32 +URLCategoryID,int16 +URLRegionID,int32 +ResolutionWidth,int16 +ResolutionHeight,int16 +ResolutionDepth,int16 +FlashMajor,int16 +FlashMinor,int16 +FlashMinor2,string +NetMajor,int16 +NetMinor,int16 +UserAgentMajor,int16 +UserAgentMinor,string +CookieEnable,int16 +JavascriptEnable,int16 +IsMobile,int16 +MobilePhone,int16 +MobilePhoneModel,string +Params,string +IPNetworkID,int32 +TraficSourceID,int16 +SearchEngineID,int16 +SearchPhrase,string +AdvEngineID,int16 +IsArtifical,int16 +WindowClientWidth,int16 +WindowClientHeight,int16 +ClientTimeZone,int16 +ClientEventTime,TIMESTAMP +SilverlightVersion1,int16 +SilverlightVersion2,int16 +SilverlightVersion3,int32 +SilverlightVersion4,int16 +PageCharset,string +CodeVersion,int32 +IsLink,int16 +IsDownload,int16 +IsNotBounce,int16 +FUniqID,int64 +OriginalURL,string +HID,int32 +IsOldCounter,int16 +IsEvent,int16 +IsParameter,int16 +DontCountHits,int16 +WithHash,int16 +HitColor,string +LocalEventTime,TIMESTAMP +Age,int16 +Sex,int16 +Income,int16 +Interests,int16 +Robotness,int16 +RemoteIP,int32 +WindowName,int32 +OpenerName,int32 +HistoryLength,int16 +BrowserLanguage,string +BrowserCountry,string +SocialNetwork,string +SocialAction,string +HTTPError,int16 +SendTiming,int32 +DNSTiming,int32 +ConnectTiming,int32 +ResponseStartTiming,int32 +ResponseEndTiming,int32 +FetchTiming,int32 +SocialSourceNetworkID,int16 +SocialSourcePage,string +ParamPrice,int64 +ParamOrderID,string +ParamCurrency,string +ParamCurrencyID,int16 +OpenstatServiceName,string +OpenstatCampaignID,string +OpenstatAdID,string +OpenstatSourceID,string +UTMSource,string +UTMMedium,string +UTMCampaign,string +UTMContent,string +UTMTerm,string +FromTag,string +HasGCLID,int16 +RefererHash,int64 +URLHash,int64 +CLID,int32 diff --git a/benchmarks/support/query_result_checks.cpp b/benchmarks/support/query_result_checks.cpp new file mode 100644 index 0000000..80a44a3 --- /dev/null +++ b/benchmarks/support/query_result_checks.cpp @@ -0,0 +1,60 @@ +#include "query_result_checks.h" + +#include +#include +#include + +#include "io/csv_batch.h" +#include "common/ascii.h" + +std::string BatchToString(const Batch& batch) { + std::vector> rows; + AppendBatchRows(batch, rows); + + std::string result; + + for (const auto& row : rows) { + for (size_t i = 0; i < row.size(); ++i) { + result += row[i]; + if (i + 1 < row.size()) { + result += ","; + } + } + result += "\n"; + } + + return result; +} + +bool QueryUsesGroupBy(const std::string_view sql) { return ToUpperAscii(sql).find("GROUP BY") != std::string::npos; } + +static std::vector SplitRows(const std::string_view text) { + std::vector rows; + size_t start = 0; + + while (start < text.size()) { + const size_t end = text.find('\n', start); + if (end == std::string_view::npos) { + rows.emplace_back(text.substr(start)); + break; + } + rows.emplace_back(text.substr(start, end - start)); + start = end + 1; + } + + if (!rows.empty() && rows.back().empty()) { + rows.pop_back(); + } + + return rows; +} + +bool EqualAsRowMultisets(const std::string_view lhs, const std::string_view rhs) { + std::vector lhs_rows = SplitRows(lhs); + std::vector rhs_rows = SplitRows(rhs); + + std::ranges::sort(lhs_rows); + std::ranges::sort(rhs_rows); + + return lhs_rows == rhs_rows; +} diff --git a/benchmarks/support/query_result_checks.h b/benchmarks/support/query_result_checks.h new file mode 100644 index 0000000..ceea1b7 --- /dev/null +++ b/benchmarks/support/query_result_checks.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +#include "model/batch.h" + +std::string BatchToString(const Batch& batch); +bool QueryUsesGroupBy(std::string_view sql); +bool EqualAsRowMultisets(std::string_view lhs, std::string_view rhs); diff --git a/benchmarks/support/query_runners.h b/benchmarks/support/query_runners.h new file mode 100644 index 0000000..b1ca5cf --- /dev/null +++ b/benchmarks/support/query_runners.h @@ -0,0 +1,98 @@ +#pragma once + +#include +#include + +#include "executor/executor.h" + +using QueryRunner = std::function; + +inline ExecuteExpected RunQuery0(const Executor& executor) { return executor.Execute("SELECT COUNT(*) FROM hits;"); } + +inline ExecuteExpected RunQuery1(const Executor& executor) { + return executor.Execute("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); +} + +inline ExecuteExpected RunQuery2(const Executor& executor) { + return executor.Execute("SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;"); +} + +inline ExecuteExpected RunQuery3(const Executor& executor) { return executor.Execute("SELECT AVG(UserID) FROM hits;"); } + +inline ExecuteExpected RunQuery4(const Executor& executor) { + return executor.Execute("SELECT COUNT(DISTINCT UserID) FROM hits;"); +} + +inline ExecuteExpected RunQuery5(const Executor& executor) { + return executor.Execute("SELECT COUNT(DISTINCT SearchPhrase) FROM hits;"); +} + +inline ExecuteExpected RunQuery6(const Executor& executor) { + return executor.Execute("SELECT MIN(EventDate), MAX(EventDate) FROM hits;"); +} + +inline ExecuteExpected RunQuery7(const Executor& executor) { + return executor.Execute( + "SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;"); +} + +inline ExecuteExpected RunQuery8(const Executor& executor) { + return executor.Execute( + "SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC " + "LIMIT 10;"); +} + +inline ExecuteExpected RunQuery9(const Executor& executor) { + return executor.Execute( + "SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), " + "COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery10(const Executor& executor) { + return executor.Execute( + "SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE " + "MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery11(const Executor& executor) { + return executor.Execute( + "SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE " + "MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery12(const Executor& executor) { + return executor.Execute( + "SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY " + "SearchPhrase ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery13(const Executor& executor) { + return executor.Execute( + "SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' " + "GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery14(const Executor& executor) { + return executor.Execute( + "SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' " + "GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery15(const Executor& executor) { + return executor.Execute("SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery16(const Executor& executor) { + return executor.Execute( + "SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC " + "LIMIT 10;"); +} + +inline const std::map& GetQueryRunners() { + static const std::map Runners = { + {0, RunQuery0}, {1, RunQuery1}, {2, RunQuery2}, {3, RunQuery3}, {4, RunQuery4}, {5, RunQuery5}, + {6, RunQuery6}, {7, RunQuery7}, {8, RunQuery8}, {9, RunQuery9}, {10, RunQuery10}, {11, RunQuery11}, + {12, RunQuery12}, {13, RunQuery13}, {14, RunQuery14}, {15, RunQuery15}, {16, RunQuery16}, + }; + return Runners; +} diff --git a/include/support/ascii.h b/include/common/ascii.h similarity index 100% rename from include/support/ascii.h rename to include/common/ascii.h diff --git a/include/support/error.h b/include/common/error.h similarity index 77% rename from include/support/error.h rename to include/common/error.h index 8cd7a87..6dbfe34 100644 --- a/include/support/error.h +++ b/include/common/error.h @@ -1,5 +1,3 @@ -/* (What is this? — Модуль ошибок) */ - #pragma once #include @@ -12,37 +10,28 @@ class Error final : public std::exception { public: enum class Code : uint16_t { InvalidArgument = 1, - InvalidData = 2, + MalformedData = 2, NotFound = 3, OutOfRange = 4, - Mismatch = 5, + InconsistentData = 5, Overflow = 6, InvalidState = 7, Unsupported = 8, IoFailure = 9, }; - enum class Type : uint8_t { - Validation, - Data, - State, - Io, - }; - - public: - Error(Code code, std::string module, std::string module_instance, Type type, std::string message); + Error(Code code, std::string module, std::string module_instance, std::string message); Error(const Error&) = default; Error(Error&&) noexcept = default; Error& operator=(const Error&) = default; Error& operator=(Error&&) noexcept = default; ~Error() override = default; - public: static Error InvalidArgument(std::string module, std::string message, std::string module_instance = {}); - static Error InvalidData(std::string module, std::string message, std::string module_instance = {}); + static Error MalformedData(std::string module, std::string message, std::string module_instance = {}); static Error NotFound(std::string module, std::string message, std::string module_instance = {}); static Error OutOfRange(std::string module, std::string message, std::string module_instance = {}); - static Error Mismatch(std::string module, std::string message, std::string module_instance = {}); + static Error InconsistentData(std::string module, std::string message, std::string module_instance = {}); static Error Overflow(std::string module, std::string message, std::string module_instance = {}); static Error InvalidState(std::string module, std::string message, std::string module_instance = {}); static Error Unsupported(std::string module, std::string message, std::string module_instance = {}); @@ -51,22 +40,18 @@ class Error final : public std::exception { const char* what() const noexcept override; Code GetCode() const noexcept; - Type GetType() const noexcept; const std::string& GetModule() const noexcept; const std::string& GetModuleInstance() const noexcept; const std::string& GetMessage() const noexcept; static std::string_view CodeToString(Code code) noexcept; - static std::string_view TypeToString(Type type) noexcept; private: static std::string BuildWhat(std::string_view module, std::string_view message); - private: Code code_; std::string module_; std::string module_instance_; - Type type_; std::string message_; std::string what_; }; diff --git a/include/support/int128.h b/include/common/int128.h similarity index 86% rename from include/support/int128.h rename to include/common/int128.h index 7b9f28a..0b15cbf 100644 --- a/include/support/int128.h +++ b/include/common/int128.h @@ -1,5 +1,3 @@ -/* (What is this? — Алиасы для 128-битных целых) */ - #pragma once #if !defined(__SIZEOF_INT128__) diff --git a/include/support/parsing.h b/include/common/parsing.h similarity index 55% rename from include/support/parsing.h rename to include/common/parsing.h index 97ec50d..d779adb 100644 --- a/include/support/parsing.h +++ b/include/common/parsing.h @@ -1,13 +1,12 @@ -/* (What is this? — Парсеры) */ - #pragma once #include +#include #include #include -#include "int128.h" -#include "schema.h" +#include "model/schema.h" +#include "common/int128.h" bool ParseBoolean(const std::string& value); int16_t ParseInt16(const std::string& value); @@ -18,6 +17,15 @@ int32_t ParseDate(const std::string& value); int64_t ParseTimestamp(const std::string& value); char ParseCharacter(const std::string& value); +std::optional TryParseBoolean(std::string_view value); +std::optional TryParseInt16(std::string_view value); +std::optional TryParseInt32(std::string_view value); +std::optional TryParseInt64(std::string_view value); +std::optional TryParseInt128(std::string_view value); +std::optional TryParseDate(std::string_view value); +std::optional TryParseTimestamp(std::string_view value); +std::optional TryParseCharacter(std::string_view value); + std::string BooleanToString(bool value); std::string Int128ToString(Int128 value); std::string DateToString(int32_t value); diff --git a/include/convert/csv_columnar.h b/include/convert/csv_columnar.h index d58a734..cd2b5f5 100644 --- a/include/convert/csv_columnar.h +++ b/include/convert/csv_columnar.h @@ -1,5 +1,3 @@ -/* (What is this? — CSV <-> колоночный формат) */ - #pragma once #include diff --git a/include/executor/aggregate_function.h b/include/executor/aggregate_function.h index 0494d4e..aeecdad 100644 --- a/include/executor/aggregate_function.h +++ b/include/executor/aggregate_function.h @@ -1,13 +1,12 @@ -/* (What is this? — Объявление функций агрегации) */ - #pragma once +#include #include #include #include #include -#include "schema.h" +#include "model/schema.h" class AggState; struct PlannedAgg; @@ -15,10 +14,23 @@ struct PlannedAgg; using AggFactory = std::unique_ptr (*)(const PlannedAgg&); using AggTypeSupport = bool (*)(ColumnType); +enum class AggCallFeature : std::uint8_t { + None = 0, + Distinct = 1, + Star = 2, +}; + +constexpr AggCallFeature operator|(const AggCallFeature lhs, const AggCallFeature rhs) { + return static_cast(static_cast(lhs) | static_cast(rhs)); +} + +constexpr bool HasAggCallFeature(const AggCallFeature features, const AggCallFeature feature) { + return (static_cast(features) & static_cast(feature)); +} + struct AggFuncDefinition { std::string_view canonical_name; - bool distinct = false; - bool star = false; + AggCallFeature call_features = AggCallFeature::None; AggTypeSupport supports_type = nullptr; AggFactory factory = nullptr; }; @@ -42,4 +54,3 @@ struct AggRegistrar { const AggFuncDefinition& ResolveAggFunc(std::string_view name); bool AggSupportsInputType(const AggFuncDefinition& definition, ColumnType type); -std::string FormatAgg(const AggFuncDefinition& definition, std::string_view column_name, bool distinct, bool star); diff --git a/include/executor/aggregate_state.h b/include/executor/aggregate_state.h new file mode 100644 index 0000000..c4d22a9 --- /dev/null +++ b/include/executor/aggregate_state.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include + +#include "executor/query_plan.h" + +class AggState { + public: + AggState() = default; + AggState(const AggState&) = delete; + AggState(AggState&&) noexcept = default; + AggState& operator=(const AggState&) = delete; + AggState& operator=(AggState&&) noexcept = default; + virtual ~AggState() = default; + + virtual void ConsumeValue(std::string_view value) = 0; + virtual void ConsumeRow() = 0; + virtual std::string Finalize() const = 0; +}; + +std::unique_ptr CreateAggState(const PlannedAgg& aggregate); diff --git a/include/executor/executor.h b/include/executor/executor.h index be10e0b..1c54192 100644 --- a/include/executor/executor.h +++ b/include/executor/executor.h @@ -1,120 +1,23 @@ -/* (What is this? — Объявление классов и функций исполнителя запросов) */ - #pragma once #include -#include -#include #include #include #include -#include -#include "aggregate_function.h" -#include "batch.h" -#include "error.h" -#include "expected.hpp" -#include "tokenizer.h" +#include "common/error.h" +#include "model/batch.h" +#include "tl/expected.hpp" using ExecuteExpected = tl::expected; -class Operator { - public: - Operator() = default; - Operator(const Operator&) = delete; - Operator(Operator&&) noexcept = default; - Operator& operator=(const Operator&) = delete; - Operator& operator=(Operator&&) noexcept = default; - virtual ~Operator() = default; - - public: - virtual std::optional Next() = 0; -}; - class Executor { public: Executor() = default; - public: void RegisterTable(const std::string& name, std::filesystem::path path); ExecuteExpected Execute(std::string_view query) const; private: std::unordered_map tables_; }; - -enum class ComparisonKind { - Equal, - NotEqual, - Less, - LessOrEqual, - Greater, - GreaterOrEqual, -}; - -struct ParsedLiteral { - std::string text; - Tokens token_type = Tokens::StringLiteral; -}; - -struct FilterSpec { - std::string column_name; - ComparisonKind comparison = ComparisonKind::Equal; - ParsedLiteral literal; -}; - -struct AggSpec { - const AggFuncDefinition* function = nullptr; - bool distinct = false; - bool star = false; - std::string column_name; - std::string output_name; -}; - -struct ParsedQuery { - std::string table_name; - std::vector aggregates; - std::optional filter; -}; - -struct PlannedAgg { - const AggFuncDefinition* function = nullptr; - bool distinct = false; - bool star = false; - std::optional column_index; - ColumnType input_type = ColumnType::String; - std::string output_name; -}; - -struct PlannedFilter { - size_t column_index = 0; - ColumnType column_type = ColumnType::String; - ComparisonKind comparison = ComparisonKind::Equal; - std::string literal_text; -}; - -struct PlannedQuery { - std::filesystem::path table_path; - std::vector projection_indexes; - std::optional filter; - std::vector aggregates; -}; - -class AggState { - public: - AggState() = default; - AggState(const AggState&) = delete; - AggState(AggState&&) noexcept = default; - AggState& operator=(const AggState&) = delete; - AggState& operator=(AggState&&) noexcept = default; - virtual ~AggState() = default; - - public: - virtual void Consume(std::optional value) = 0; - virtual std::string Finalize() const = 0; -}; - -ParsedQuery ParseQuery(std::string_view query); -PlannedQuery PlanQuery(const ParsedQuery& parsed, const std::unordered_map& tables); -std::unique_ptr CreateAggState(const PlannedAgg& aggregate); -std::unique_ptr BuildPlan(const PlannedQuery& planned); diff --git a/include/executor/operator.h b/include/executor/operator.h new file mode 100644 index 0000000..4906cf1 --- /dev/null +++ b/include/executor/operator.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +#include "executor/query_plan.h" +#include "model/batch.h" + +class Operator { + public: + Operator() = default; + Operator(const Operator&) = delete; + Operator(Operator&&) noexcept = default; + Operator& operator=(const Operator&) = delete; + Operator& operator=(Operator&&) noexcept = default; + virtual ~Operator() = default; + + virtual std::optional Next() = 0; +}; + +std::unique_ptr BuildPlan(const PlannedQuery& planned); diff --git a/include/executor/operator_utils.h b/include/executor/operator_utils.h new file mode 100644 index 0000000..d247c5d --- /dev/null +++ b/include/executor/operator_utils.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +#include "executor/query_plan.h" + +std::strong_ordering CompareValues(ColumnType type, std::string_view lhs, std::string_view rhs); +bool MatchesComparison(ColumnType type, std::string_view lhs, std::string_view rhs, ComparisonKind kind); +ColumnType AggregateOutputType(const PlannedAgg& aggregate); diff --git a/include/executor/planner_utils.h b/include/executor/planner_utils.h new file mode 100644 index 0000000..70d19af --- /dev/null +++ b/include/executor/planner_utils.h @@ -0,0 +1,16 @@ +#pragma once + +#include +#include + +#include "executor/query.h" +#include "model/schema.h" + +size_t FindColumnIndex(const Schema& schema, std::string_view column_name); +void ValidateColumnQualifier(const Query& query, const ColumnRef& column); +std::string NormalizeLiteral(const QueryLiteral& literal, ColumnType type); + +bool SameColumnName(std::string_view lhs, std::string_view rhs); +bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs); +bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs); +bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs); diff --git a/include/executor/query.h b/include/executor/query.h new file mode 100644 index 0000000..2cd4b88 --- /dev/null +++ b/include/executor/query.h @@ -0,0 +1,88 @@ +#pragma once + +#include +#include +#include +#include + +enum class ComparisonKind { + Equal, + NotEqual, + Less, + LessOrEqual, + Greater, + GreaterOrEqual, +}; + +enum class LiteralKind { + Numeric, + String, + Identifier, +}; + +struct QueryLiteral { + std::string text; + LiteralKind kind = LiteralKind::String; +}; + +struct ColumnRef { + std::string qualifier; + std::string name; +}; + +enum class QueryValueKind { + Column, + Literal, +}; + +struct QueryValue { + QueryValueKind kind = QueryValueKind::Literal; + ColumnRef column; + QueryLiteral literal; +}; + +struct FilterSpec { + QueryValue left; + ComparisonKind comparison = ComparisonKind::Equal; + QueryValue right; +}; + +enum class AggArgumentKind { + Column, + Star, +}; + +struct AggSpec { + std::string function_name; + bool distinct = false; + AggArgumentKind argument_kind = AggArgumentKind::Column; + ColumnRef column; + std::string output_name; +}; + +enum class SelectItemKind { + GroupKey, + Aggregate, +}; + +struct SelectItemSpec { + SelectItemKind kind = SelectItemKind::GroupKey; + ColumnRef column; + AggSpec aggregate; + std::string output_name; +}; + +struct OrderBySpec { + SelectItemSpec item; + bool descending = false; +}; + +struct Query { + std::string table_name; + std::string table_alias; + std::vector select_items; + std::vector group_by; + std::optional filter; + std::vector order_by; + std::optional limit; +}; diff --git a/include/executor/query_parser.h b/include/executor/query_parser.h new file mode 100644 index 0000000..5b823d0 --- /dev/null +++ b/include/executor/query_parser.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +#include "executor/query.h" + +Query ParseQuery(std::string_view query); diff --git a/include/executor/query_plan.h b/include/executor/query_plan.h new file mode 100644 index 0000000..7cfaba7 --- /dev/null +++ b/include/executor/query_plan.h @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "executor/aggregate_function.h" +#include "executor/query.h" +#include "model/schema.h" + +struct PlannedAgg { + const AggFuncDefinition* function = nullptr; + bool distinct = false; + AggArgumentKind argument_kind = AggArgumentKind::Column; + size_t column_index = 0; + ColumnType input_type = ColumnType::String; + std::string output_name; +}; + +struct PlannedGroupKey { + size_t column_index = 0; + ColumnType column_type = ColumnType::String; + std::string output_name; +}; + +struct PlannedSelectItem { + SelectItemKind kind = SelectItemKind::GroupKey; + size_t index = 0; + ColumnType output_type = ColumnType::String; + std::string output_name; +}; + +struct PlannedOrderBy { + size_t result_column_index = 0; + bool descending = false; + ColumnType value_type = ColumnType::String; +}; + +enum class PlannedValueKind { + Column, + Literal, +}; + +struct PlannedValue { + PlannedValueKind kind = PlannedValueKind::Literal; + ColumnType column_type = ColumnType::String; + size_t column_index = 0; + std::string literal_text; +}; + +struct PlannedFilter { + PlannedValue left; + ComparisonKind comparison = ComparisonKind::Equal; + PlannedValue right; + ColumnType comparison_type = ColumnType::String; +}; + +struct PlannedQuery { + std::filesystem::path table_path; + std::vector projection_indexes; + std::optional filter; + std::vector group_keys; + std::vector aggregates; + std::vector select_items; + std::vector order_by; + std::optional limit; +}; diff --git a/include/executor/query_planner.h b/include/executor/query_planner.h new file mode 100644 index 0000000..913d6f4 --- /dev/null +++ b/include/executor/query_planner.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include +#include + +#include "executor/query.h" +#include "executor/query_plan.h" + +PlannedQuery PlanQuery(const Query& query, const std::unordered_map& tables); diff --git a/include/io/batch_io.h b/include/io/batch.h similarity index 85% rename from include/io/batch_io.h rename to include/io/batch.h index 3fea3d1..70e8703 100644 --- a/include/io/batch_io.h +++ b/include/io/batch.h @@ -1,10 +1,8 @@ -/* (What is this? — Потоковое чтение/запись батчей из CSV в колоночный формат) */ - #pragma once #include -#include "batch.h" +#include "model/batch.h" struct BatchSizing { std::optional max_rows; @@ -23,7 +21,6 @@ class BatchReader { BatchReader& operator=(BatchReader&&) noexcept = default; virtual ~BatchReader() = default; - public: virtual std::optional ReadNext() = 0; }; @@ -36,7 +33,6 @@ class BatchWriter { BatchWriter& operator=(BatchWriter&&) noexcept = default; virtual ~BatchWriter() = default; - public: virtual void Write(const Batch& batch) = 0; virtual void Flush() = 0; }; diff --git a/include/io/columnar_batch_io.h b/include/io/columnar_batch.h similarity index 85% rename from include/io/columnar_batch_io.h rename to include/io/columnar_batch.h index d04f02b..9dfebb0 100644 --- a/include/io/columnar_batch_io.h +++ b/include/io/columnar_batch.h @@ -1,14 +1,11 @@ -/* (What is this? — Чтение/запись батчей из CSV в колоночный формат) */ - #pragma once #include -#include #include -#include "batch_io.h" -#include "fileio.h" -#include "metadata.h" +#include "io/batch.h" +#include "io/file.h" +#include "model/metadata.h" class ColumnarBatchReader final : public BatchReader { public: @@ -19,7 +16,6 @@ class ColumnarBatchReader final : public BatchReader { ColumnarBatchReader& operator=(ColumnarBatchReader&&) noexcept = default; ~ColumnarBatchReader() override = default; - public: std::optional ReadNext() override; const Schema& GetSchema() const { return metadata_.schema; } @@ -28,9 +24,8 @@ class ColumnarBatchReader final : public BatchReader { private: static ColumnarMetadata ReadFileMetadata(const std::filesystem::path& path); - private: std::filesystem::path path_; - std::ifstream in_; + InputFile input_; ColumnarMetadata metadata_; size_t next_group_ = 0; @@ -45,11 +40,10 @@ class ColumnarBatchWriter final : public BatchWriter { ColumnarBatchWriter& operator=(ColumnarBatchWriter&&) noexcept = default; ~ColumnarBatchWriter() override = default; - public: void Write(const Batch& batch) override; void Flush() override; - void Finalize(); + void Finalize() &&; const ColumnarMetadata& GetMetadata() const { return metadata_; } diff --git a/include/io/csv.h b/include/io/csv.h index e01e998..f55cc43 100644 --- a/include/io/csv.h +++ b/include/io/csv.h @@ -1,5 +1,3 @@ -/* (What is this? — Чтение/запись CSV) */ - #pragma once #include @@ -18,13 +16,11 @@ class CsvReader { CsvReader& operator=(CsvReader&& other) noexcept; ~CsvReader() = default; - public: bool ReadRow(std::vector& row) const; private: void RebindAfterMove(bool uses_owned_stream, std::istream* source_stream) noexcept; - private: std::ifstream owned_in_; std::istream* in_ = nullptr; }; @@ -39,14 +35,12 @@ class CsvWriter { CsvWriter& operator=(CsvWriter&& other) noexcept; ~CsvWriter() = default; - public: void WriteRow(const std::vector& row) const; void Flush() const; private: void RebindAfterMove(bool uses_owned_stream, std::ostream* source_stream) noexcept; - private: std::ofstream owned_out_; std::ostream* out_ = nullptr; }; diff --git a/include/io/csv_batch_io.h b/include/io/csv_batch.h similarity index 93% rename from include/io/csv_batch_io.h rename to include/io/csv_batch.h index 86c108a..97dbab2 100644 --- a/include/io/csv_batch_io.h +++ b/include/io/csv_batch.h @@ -1,13 +1,11 @@ -/* (What is this? — Чтение/запись CSV) */ - #pragma once #include #include #include -#include "batch_io.h" -#include "csv.h" +#include "io/batch.h" +#include "io/csv.h" class CsvBatchReader final : public BatchReader { public: @@ -18,7 +16,6 @@ class CsvBatchReader final : public BatchReader { CsvBatchReader& operator=(CsvBatchReader&&) noexcept = default; ~CsvBatchReader() override = default; - public: std::optional ReadNext() override; const Schema& GetSchema() const { return schema_; } @@ -41,7 +38,6 @@ class CsvBatchWriter final : public BatchWriter { CsvBatchWriter& operator=(CsvBatchWriter&&) noexcept = default; ~CsvBatchWriter() override = default; - public: void Write(const Batch& batch) override; void Flush() override; diff --git a/include/io/file.h b/include/io/file.h new file mode 100644 index 0000000..7ab909d --- /dev/null +++ b/include/io/file.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +struct FileMetadata { + uintmax_t size = 0; + bool is_regular = false; + bool is_directory = false; + std::filesystem::file_time_type last_write_time; +}; + +std::optional GetFileMetadata(const std::filesystem::path& path); + +bool FileExists(const std::filesystem::path& path); +void EnsureParentDirectory(const std::filesystem::path& path); + +std::ifstream OpenInputFile(const std::filesystem::path& path); +void SeekInputFile(std::istream& in, const std::filesystem::path& path, uint64_t offset); + +class InputFile { + public: + explicit InputFile(std::filesystem::path path); + InputFile(const InputFile&) = delete; + InputFile(InputFile&&) noexcept = default; + InputFile& operator=(const InputFile&) = delete; + InputFile& operator=(InputFile&&) noexcept = default; + ~InputFile() = default; + + void ReadAt(char* dst, size_t size, uint64_t offset); + std::string ReadStringAt(uint64_t offset, size_t size); + std::istream& StreamAt(uint64_t offset); + + template + T ReadAt(const uint64_t offset) { + T value = 0; + ReadAt(reinterpret_cast(&value), sizeof(value), offset); + return value; + } + + private: + std::filesystem::path path_; + std::ifstream in_; +}; + +enum class FileOpenMode { + Truncate, + Append, +}; + +std::ofstream OpenOutputFile(const std::filesystem::path& path, FileOpenMode mode = FileOpenMode::Truncate); + +std::string ReadTextFile(const std::filesystem::path& path); +std::vector ReadFileBytes(const std::filesystem::path& path); +void WriteFileBytes(const std::filesystem::path& path, std::span bytes); +void WriteFileBytes(const std::filesystem::path& path, const std::vector& bytes); diff --git a/include/io/fileio.h b/include/io/fileio.h deleted file mode 100644 index c2d9b36..0000000 --- a/include/io/fileio.h +++ /dev/null @@ -1,35 +0,0 @@ -/* (What is this? — Обертки над потоками, чтение и запись) */ - -#pragma once - -#include -#include -#include -#include -#include - -struct FileMetadata { - uintmax_t size = 0; - bool is_regular = false; - bool is_directory = false; - std::filesystem::file_time_type last_write_time; -}; - -std::optional TryGetFileMetadata(const std::filesystem::path& path); - -bool FileExists(const std::filesystem::path& path); -void EnsureParentDirectory(const std::filesystem::path& path); - -std::ifstream OpenInputFile(const std::filesystem::path& path); - -enum class FileOpenMode { - Truncate, - Append, -}; - -std::ofstream OpenOutputFile(const std::filesystem::path& path, FileOpenMode mode = FileOpenMode::Truncate); - -std::string ReadTextFile(const std::filesystem::path& path); -std::vector ReadFileBytes(const std::filesystem::path& path); -void WriteFileBytes(const std::filesystem::path& path, std::span bytes); -void WriteFileBytes(const std::filesystem::path& path, const std::vector& bytes); diff --git a/include/io/stream.h b/include/io/stream.h index 676400a..a1b2358 100644 --- a/include/io/stream.h +++ b/include/io/stream.h @@ -1,5 +1,3 @@ -/* (What is this? — Обертки над потоками, чтение и запись) */ - #pragma once #include @@ -13,7 +11,7 @@ #include #include -#include "error.h" +#include "common/error.h" void ReadBytes(std::istream& in, char* dst, size_t size); void WriteBytes(std::ostream& out, std::string_view bytes); diff --git a/include/model/batch.h b/include/model/batch.h index 08d5667..6d8703b 100644 --- a/include/model/batch.h +++ b/include/model/batch.h @@ -1,12 +1,13 @@ -/* (What is this? — Класс батча) */ - #pragma once +#include +#include #include +#include #include -#include "column.h" -#include "schema.h" +#include "model/column.h" +#include "model/schema.h" class Batch { public: @@ -19,23 +20,20 @@ class Batch { Batch& operator=(Batch&& other) noexcept = default; ~Batch() = default; - public: const Schema& GetSchema() const { return schema_; } size_t ColumnsCount() const; size_t RowsCount() const; void Reserve(size_t n) const; + void AppendValueFromString(size_t column_index, const std::string& value) const; + void ReadColumnFrom(size_t column_index, std::istream& in, uint32_t row_count, uint64_t size) const; const Column& ColumnAt(size_t i) const; - Column& ColumnAt(size_t i); - - const std::vector>& GetColumns() const { return columns_; } - std::vector>& GetColumns() { return columns_; } void Validate() const; private: Schema schema_; - std::vector> columns_; + std::vector> columns_; }; diff --git a/include/model/column.h b/include/model/column.h index dc9c770..118c236 100644 --- a/include/model/column.h +++ b/include/model/column.h @@ -1,14 +1,11 @@ -/* (What is this? — Абстракция колонок) */ - #pragma once #include #include #include #include -#include -#include "schema.h" +#include "model/schema.h" class Column { public: @@ -19,20 +16,13 @@ class Column { Column& operator=(Column&&) noexcept = default; virtual ~Column() = default; - public: ColumnType Type() const { return type_; } virtual size_t Size() const = 0; - virtual void Reserve(size_t n) = 0; - virtual void Clear() = 0; - - virtual void AppendFromString(const std::string& value) = 0; - virtual uint64_t EstimateSizeFromString(std::string_view value) const = 0; virtual std::string ValueAsString(size_t row) const = 0; virtual std::unique_ptr Clone() const = 0; virtual void WriteTo(std::ostream& out) const = 0; - virtual void ReadFrom(std::istream& in, uint32_t row_count, uint64_t size) = 0; protected: static void CheckRowIndex(const char* module, size_t row, size_t size); @@ -41,4 +31,16 @@ class Column { ColumnType type_; }; -std::unique_ptr CreateColumn(ColumnType type); +class MutableColumn : public Column { + public: + using Column::Column; + ~MutableColumn() override = default; + + virtual void Reserve(size_t n) = 0; + virtual void Clear() = 0; + virtual void AppendFromString(const std::string& value) = 0; + virtual void ReadFrom(std::istream& in, uint32_t row_count, uint64_t size) = 0; + virtual std::unique_ptr CloneMutable() const = 0; +}; + +std::unique_ptr CreateColumn(ColumnType type); diff --git a/include/model/column_boolean.h b/include/model/column_boolean.h index 4194b4a..a60e7b3 100644 --- a/include/model/column_boolean.h +++ b/include/model/column_boolean.h @@ -1,15 +1,10 @@ -/* (What is this? — Колонка логического типа) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class BooleanColumn final : public FixedColumn { public: using FixedColumn::FixedColumn; static const char* ModuleName() { return "column_boolean"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; }; diff --git a/include/model/column_character.h b/include/model/column_character.h index 9ba8684..cdb0842 100644 --- a/include/model/column_character.h +++ b/include/model/column_character.h @@ -1,15 +1,10 @@ -/* (What is this? — Колонка символьного типа) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class CharacterColumn final : public FixedColumn { public: using FixedColumn::FixedColumn; static const char* ModuleName() { return "column_character"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; }; diff --git a/include/model/column_date.h b/include/model/column_date.h index fa185a4..99bc659 100644 --- a/include/model/column_date.h +++ b/include/model/column_date.h @@ -1,15 +1,10 @@ -/* (What is this? — Колонка типа DATE) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class DateColumn final : public FixedColumn { public: using FixedColumn::FixedColumn; static const char* ModuleName() { return "column_date"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; }; diff --git a/include/model/column_int128.h b/include/model/column_int128.h index d24250d..5c00df9 100644 --- a/include/model/column_int128.h +++ b/include/model/column_int128.h @@ -1,16 +1,11 @@ -/* (What is this? — Колонка типа INT128) */ - #pragma once -#include "fixed_column.h" -#include "int128.h" +#include "model/fixed_column.h" +#include "common/int128.h" class Int128Column final : public FixedColumn { public: using FixedColumn::FixedColumn; static const char* ModuleName() { return "column_int128"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; }; diff --git a/include/model/column_int16.h b/include/model/column_int16.h index 14a0811..81a370f 100644 --- a/include/model/column_int16.h +++ b/include/model/column_int16.h @@ -1,15 +1,10 @@ -/* (What is this? — Колонка типа INT16) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class Int16Column final : public FixedColumn { public: using FixedColumn::FixedColumn; static const char* ModuleName() { return "column_int16"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; }; diff --git a/include/model/column_int32.h b/include/model/column_int32.h index 458e60a..7ab56ee 100644 --- a/include/model/column_int32.h +++ b/include/model/column_int32.h @@ -1,15 +1,10 @@ -/* (What is this? — Колонка типа INT32) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class Int32Column final : public FixedColumn { public: using FixedColumn::FixedColumn; static const char* ModuleName() { return "column_int32"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; }; diff --git a/include/model/column_int64.h b/include/model/column_int64.h index aa44865..5c44800 100644 --- a/include/model/column_int64.h +++ b/include/model/column_int64.h @@ -1,15 +1,10 @@ -/* (What is this? — Колонка типа INT64) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class Int64Column final : public FixedColumn { public: using FixedColumn::FixedColumn; static const char* ModuleName() { return "column_int64"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; }; diff --git a/include/model/column_string.h b/include/model/column_string.h index 60589f8..dc6236a 100644 --- a/include/model/column_string.h +++ b/include/model/column_string.h @@ -1,15 +1,33 @@ -/* (What is this? — Колонка типа STRING) */ - #pragma once -#include "variant_column.h" +#include +#include + +#include "model/column.h" -class StringColumn final : public VariantColumn { +class StringColumn final : public MutableColumn { public: - using VariantColumn::VariantColumn; + StringColumn(); + StringColumn(const StringColumn&) = default; + StringColumn(StringColumn&&) noexcept = default; + StringColumn& operator=(const StringColumn&) = default; + StringColumn& operator=(StringColumn&&) noexcept = default; + ~StringColumn() override = default; static const char* ModuleName() { return "column_string"; } + size_t Size() const override; + void Reserve(size_t n) override; + void Clear() override; + void AppendFromString(const std::string& value) override; std::string ValueAsString(size_t row) const override; + std::unique_ptr Clone() const override; + std::unique_ptr CloneMutable() const override; + + void WriteTo(std::ostream& out) const override; + void ReadFrom(std::istream& in, uint32_t row_count, uint64_t size) override; + + private: + std::vector values_; }; diff --git a/include/model/column_timestamp.h b/include/model/column_timestamp.h index 6fb061b..a446b94 100644 --- a/include/model/column_timestamp.h +++ b/include/model/column_timestamp.h @@ -1,15 +1,10 @@ -/* (What is this? — Колонка типа TIMESTAMP) */ - #pragma once -#include "fixed_column.h" +#include "model/fixed_column.h" class TimestampColumn final : public FixedColumn { public: using FixedColumn::FixedColumn; static const char* ModuleName() { return "column_timestamp"; } - - void AppendFromString(const std::string& value) override; - std::string ValueAsString(size_t row) const override; }; diff --git a/include/model/column_traits.h b/include/model/column_traits.h new file mode 100644 index 0000000..890e393 --- /dev/null +++ b/include/model/column_traits.h @@ -0,0 +1,111 @@ +#pragma once + +#include +#include +#include + +#include "common/error.h" +#include "common/int128.h" +#include "common/parsing.h" +#include "model/schema.h" + +template +struct ColumnValueTraits; + +template <> +struct ColumnValueTraits { + using Type = uint8_t; + + static Type Parse(const std::string& value) { return ParseBoolean(value) ? 1 : 0; } + static std::string ToString(const Type value) { return BooleanToString(value != 0); } +}; + +template <> +struct ColumnValueTraits { + using Type = int16_t; + + static Type Parse(const std::string& value) { return ParseInt16(value); } + static std::string ToString(const Type value) { return std::to_string(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = int32_t; + + static Type Parse(const std::string& value) { return ParseInt32(value); } + static std::string ToString(const Type value) { return std::to_string(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = int64_t; + + static Type Parse(const std::string& value) { return ParseInt64(value); } + static std::string ToString(const Type value) { return std::to_string(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = Int128; + + static Type Parse(const std::string& value) { return ParseInt128(value); } + static std::string ToString(const Type value) { return Int128ToString(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = int32_t; + + static Type Parse(const std::string& value) { return ParseDate(value); } + static std::string ToString(const Type value) { return DateToString(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = int64_t; + + static Type Parse(const std::string& value) { return ParseTimestamp(value); } + static std::string ToString(const Type value) { return TimestampToString(value); } +}; + +template <> +struct ColumnValueTraits { + using Type = char; + + static Type Parse(const std::string& value) { return ParseCharacter(value); } + static std::string ToString(const Type value) { return std::string(1, value); } +}; + +template <> +struct ColumnValueTraits { + using Type = std::string; + + static Type Parse(const std::string& value) { return value; } + static std::string ToString(const Type& value) { return value; } +}; + +template +decltype(auto) VisitColumnType(const ColumnType type, Visitor&& visitor) { + switch (type) { + case ColumnType::Boolean: + return std::forward(visitor).template operator()(); + case ColumnType::Int16: + return std::forward(visitor).template operator()(); + case ColumnType::Int32: + return std::forward(visitor).template operator()(); + case ColumnType::Int64: + return std::forward(visitor).template operator()(); + case ColumnType::Int128: + return std::forward(visitor).template operator()(); + case ColumnType::String: + return std::forward(visitor).template operator()(); + case ColumnType::Date: + return std::forward(visitor).template operator()(); + case ColumnType::Timestamp: + return std::forward(visitor).template operator()(); + case ColumnType::Character: + return std::forward(visitor).template operator()(); + } + + throw Error::Unsupported("model", "unsupported column type"); +} diff --git a/include/model/fixed_column.h b/include/model/fixed_column.h index ce5b169..df904ef 100644 --- a/include/model/fixed_column.h +++ b/include/model/fixed_column.h @@ -1,35 +1,38 @@ -/* (What is this? — База для колонок фиксированного размера) */ - #pragma once #include -#include "column.h" -#include "error.h" -#include "stream.h" +#include "common/error.h" +#include "io/stream.h" +#include "model/column.h" +#include "model/column_traits.h" -template -class FixedColumn : public Column { +template +class FixedColumn : public MutableColumn { public: - FixedColumn() : Column(TypeValue) {} + FixedColumn() : MutableColumn(TypeValue) {} FixedColumn(const FixedColumn&) = default; FixedColumn(FixedColumn&&) noexcept = default; FixedColumn& operator=(const FixedColumn&) = default; FixedColumn& operator=(FixedColumn&&) noexcept = default; ~FixedColumn() override = default; - public: size_t Size() const override { return values_.size(); } void Reserve(const size_t n) override { values_.reserve(n); } void Clear() override { values_.clear(); } - - uint64_t EstimateSizeFromString(const std::string_view value) const override { - (void)value; - return sizeof(T); + void AppendFromString(const std::string& value) override { + AppendValue(ColumnValueTraits::Parse(value)); + } + std::string ValueAsString(const size_t row) const override { + return ColumnValueTraits::ToString(ValueAt(row)); } std::unique_ptr Clone() const override { - return std::make_unique(static_cast(*this)); + return std::make_unique(static_cast(*this)); + } + + std::unique_ptr CloneMutable() const override { + return std::make_unique(static_cast(*this)); } void WriteTo(std::ostream& out) const override { @@ -39,7 +42,7 @@ class FixedColumn : public Column { void ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) override { const uint64_t expected = static_cast(row_count) * sizeof(T); if (size != expected) { - throw Error::Mismatch(Derived::ModuleName(), "column chunk size mismatch"); + throw Error::InconsistentData(ColumnImpl::ModuleName(), "column chunk size mismatch"); } values_.resize(row_count); ReadBytes(in, reinterpret_cast(values_.data()), values_.size() * sizeof(T)); @@ -49,10 +52,9 @@ class FixedColumn : public Column { void AppendValue(const T value) { values_.push_back(value); } T ValueAt(const size_t row) const { - CheckRowIndex(Derived::ModuleName(), row, values_.size()); + CheckRowIndex(ColumnImpl::ModuleName(), row, values_.size()); return values_[row]; } - protected: std::vector values_; }; diff --git a/include/model/metadata.h b/include/model/metadata.h index f442ac0..9a9b339 100644 --- a/include/model/metadata.h +++ b/include/model/metadata.h @@ -1,12 +1,10 @@ -/* (What is this? — Сериализация метаданных) */ - #pragma once #include #include #include -#include "schema.h" +#include "model/schema.h" struct ColumnChunkMetadata { uint64_t offset = 0; diff --git a/include/model/schema.h b/include/model/schema.h index 2e40fd7..f37530d 100644 --- a/include/model/schema.h +++ b/include/model/schema.h @@ -1,10 +1,8 @@ -/* (What is this? — Чтение/запись схемы из CSV) */ - #pragma once #include -#include #include +#include #include enum class ColumnType : uint8_t { @@ -21,7 +19,10 @@ enum class ColumnType : uint8_t { struct ColumnSchema { std::string name; - ColumnType type = ColumnType::String; + ColumnType type; + + ColumnSchema(std::string column_name, const ColumnType column_type) + : name(std::move(column_name)), type(column_type) {} bool operator==(const ColumnSchema& other) const = default; }; @@ -31,7 +32,3 @@ struct Schema { bool operator==(const Schema& other) const = default; }; - -Schema ReadSchemaCsv(const std::filesystem::path& path); -Schema InferSchemaCsv(const std::filesystem::path& path); -void WriteSchemaCsv(const std::filesystem::path& path, const Schema& schema); diff --git a/include/model/schema_csv.h b/include/model/schema_csv.h new file mode 100644 index 0000000..cc08543 --- /dev/null +++ b/include/model/schema_csv.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +#include "model/schema.h" + +Schema ReadSchemaCsv(const std::filesystem::path& path); +Schema InferSchemaCsv(const std::filesystem::path& path); +void WriteSchemaCsv(const std::filesystem::path& path, const Schema& schema); diff --git a/include/model/variant_column.h b/include/model/variant_column.h deleted file mode 100644 index bbea74f..0000000 --- a/include/model/variant_column.h +++ /dev/null @@ -1,94 +0,0 @@ -/* (What is this? — База для колонок переменной длины) */ - -#pragma once - -#include -#include -#include - -#include "column.h" -#include "error.h" -#include "stream.h" - -template -class VariantColumn : public Column { - public: - VariantColumn() : Column(TypeValue) {} - VariantColumn(const VariantColumn&) = default; - VariantColumn(VariantColumn&&) noexcept = default; - VariantColumn& operator=(const VariantColumn&) = default; - VariantColumn& operator=(VariantColumn&&) noexcept = default; - ~VariantColumn() override = default; - - public: - size_t Size() const override { return values_.size(); } - void Reserve(const size_t n) override { values_.reserve(n); } - void Clear() override { values_.clear(); } - - uint64_t EstimateSizeFromString(const std::string_view value) const override { return value.size(); } - - std::unique_ptr Clone() const override { - return std::make_unique(static_cast(*this)); - } - - void WriteTo(std::ostream& out) const override { - size_t total_size = 0; - - for (const auto& value : values_) { - if (value.size() > std::numeric_limits::max()) { - throw Error::Overflow(Derived::ModuleName(), "value exceeds supported size"); - } - total_size += sizeof(uint32_t) + value.size(); - } - - std::string buffer(total_size, '\0'); - char* dst = buffer.data(); - - for (const auto& value : values_) { - const uint32_t length = value.size(); - - std::memcpy(dst, &length, sizeof(length)); - dst += sizeof(length); - - if (!value.empty()) { - std::memcpy(dst, value.data(), value.size()); - dst += value.size(); - } - } - - WriteBytes(out, buffer); - } - - void ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) override { - values_.clear(); - values_.reserve(row_count); - - uint64_t consumed = 0; - - for (uint32_t row_index = 0; row_index < row_count; ++row_index) { - const uint32_t length = ReadStream(in); - consumed += sizeof(uint32_t); - - std::string value(length, '\0'); - ReadBytes(in, value.data(), value.size()); - - consumed += length; - values_.push_back(std::move(value)); - } - - if (consumed != size) { - throw Error::Mismatch(Derived::ModuleName(), "column chunk size mismatch"); - } - } - - protected: - void AppendValue(std::string value) { values_.push_back(std::move(value)); } - - const std::string& ValueAt(const size_t row) const { - CheckRowIndex(Derived::ModuleName(), row, values_.size()); - return values_[row]; - } - - protected: - std::vector values_; -}; diff --git a/include/sql_parser/tokenizer.h b/include/sql_parser/tokenizer.h index 19bacb8..04401d7 100644 --- a/include/sql_parser/tokenizer.h +++ b/include/sql_parser/tokenizer.h @@ -6,8 +6,8 @@ #include #include -#include "error.h" -#include "expected.hpp" +#include "common/error.h" +#include "tl/expected.hpp" enum class Tokens { NameToken, @@ -48,7 +48,7 @@ enum class Tokens { Greater, GreaterOrEqual, - EOI, + EndOfInput, }; std::string_view TokenTypeToString(Tokens type) noexcept; @@ -91,217 +91,57 @@ class IOperatorToken : public Command { ~IOperatorToken() override = default; }; -class TSelectToken : public Command { +template +class TypedToken : public BaseToken { public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TCreateToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; + using BaseToken::BaseToken; -class TFromToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; + Tokens GetType() const noexcept override { return TokenType; } }; -class TLimitToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; +using TSelectToken = TypedToken; +using TCreateToken = TypedToken; +using TFromToken = TypedToken; +using TLimitToken = TypedToken; +using TOrderToken = TypedToken; +using TWhereToken = TypedToken; +using TGroupToken = TypedToken; -class TOrderToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; +using TSumToken = TypedToken; +using TMinToken = TypedToken; +using TMaxToken = TypedToken; +using TCountToken = TypedToken; +using TDistinctToken = TypedToken; +using TAvgToken = TypedToken; +using TLengthToken = TypedToken; +using TPlusToken = TypedToken; +using TMinusToken = TypedToken; -class TWhereToken : public Command { +class TNameToken : public TypedToken { public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TGroupToken : public Command { - public: - using Command::Command; - Tokens GetType() const noexcept override; -}; - -class TSumToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TMinToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TMaxToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TCountToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TDistinctToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TAvgToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TLengthToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TPlusToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TMinusToken : public IOperatorToken { - public: - using IOperatorToken::IOperatorToken; - Tokens GetType() const noexcept override; -}; - -class TNameToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; + using TypedToken::TypedToken; std::string_view GetName() const noexcept; }; -class TNumericLiteralToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TStringLiteralToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TAsToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TAndToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TByToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TOpenBracketToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TCommaToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TCloseBracketToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TAsteriskToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TDotToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TSemicolonToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TNotEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TLessToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TLessOrEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TGreaterToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TGreaterOrEqualToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; - -class TEndOfInputToken : public Token { - public: - using Token::Token; - Tokens GetType() const noexcept override; -}; +using TNumericLiteralToken = TypedToken; +using TStringLiteralToken = TypedToken; +using TAsToken = TypedToken; +using TAndToken = TypedToken; +using TByToken = TypedToken; +using TOpenBracketToken = TypedToken; +using TCommaToken = TypedToken; +using TCloseBracketToken = TypedToken; +using TAsteriskToken = TypedToken; +using TDotToken = TypedToken; +using TSemicolonToken = TypedToken; +using TEqualToken = TypedToken; +using TNotEqualToken = TypedToken; +using TLessToken = TypedToken; +using TLessOrEqualToken = TypedToken; +using TGreaterToken = TypedToken; +using TGreaterOrEqualToken = TypedToken; +using TEndOfInputToken = TypedToken; class Tokenizer { public: diff --git a/include/testing/executor_test_utils.h b/include/testing/executor_test_utils.h index 55f3c9d..2aa6ae2 100644 --- a/include/testing/executor_test_utils.h +++ b/include/testing/executor_test_utils.h @@ -3,12 +3,9 @@ #include #include -#include "csv_batch_io.h" -#include "gtest/gtest.h" +#include "io/csv_batch.h" inline std::vector SingleRowValues(const Batch& batch) { - EXPECT_EQ(batch.RowsCount(), 1u); - std::vector> rows; rows.reserve(1); @@ -21,6 +18,15 @@ inline std::vector SingleRowValues(const Batch& batch) { return std::move(rows.front()); } +inline std::vector> BatchRows(const Batch& batch) { + std::vector> rows; + rows.reserve(batch.RowsCount()); + + AppendBatchRows(batch, rows); + + return rows; +} + inline std::vector BatchColumnNames(const Batch& batch) { std::vector names; names.reserve(batch.ColumnsCount()); diff --git a/include/testing/temp_file.h b/include/testing/temp_file.h index f4cea19..2eb9307 100644 --- a/include/testing/temp_file.h +++ b/include/testing/temp_file.h @@ -1,5 +1,3 @@ -/* (What is this? — Модуль временного файла) */ - #pragma once #include @@ -35,7 +33,6 @@ class TempFile { } ~TempFile() { Remove(); } - public: const std::filesystem::path& Path() const { return path_; } private: @@ -47,6 +44,5 @@ class TempFile { std::filesystem::remove(path_, ec); } - private: std::filesystem::path path_; }; diff --git a/script/convert.sh b/script/convert.sh index 9cd90cc..d6bc604 100755 --- a/script/convert.sh +++ b/script/convert.sh @@ -8,13 +8,15 @@ fi input_csv="$1" columnar_path="$2" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(cd "${script_dir}/.." && pwd)" build_dir="${BUILD_DIR:-build}" row_group_size="${ROW_GROUP_SIZE:-50000}" -schema_path="${COLUMNAR_SCHEMA:-${columnar_path}.schema.csv}" -binary="./${build_dir}/columnar_engine" +schema_path="${COLUMNAR_SCHEMA:-${repo_dir}/benchmarks/scheme.csv}" +binary="${repo_dir}/${build_dir}/columnar_engine" if [[ ! -x "${binary}" ]]; then - binary="./${build_dir}/src/columnar_engine" + binary="${repo_dir}/${build_dir}/src/columnar_engine" fi if [[ ! -x "${binary}" ]]; then @@ -22,12 +24,12 @@ if [[ ! -x "${binary}" ]]; then exit 1 fi -mkdir -p "$(dirname "${columnar_path}")" -mkdir -p "$(dirname "${schema_path}")" +if [[ ! -f "${schema_path}" ]]; then + echo "schema file not found: ${schema_path}" >&2 + exit 1 +fi -"${binary}" infer-schema \ - --input "${input_csv}" \ - --output "${schema_path}" +mkdir -p "$(dirname "${columnar_path}")" "${binary}" convert \ --schema "${schema_path}" \ diff --git a/script/run_query.sh b/script/run_query.sh index 2996f56..6fe4cdf 100755 --- a/script/run_query.sh +++ b/script/run_query.sh @@ -29,7 +29,7 @@ columnar_path=${columnar_path} query_file=${query_file} message=Add SQL text to benchmarks/queries/query_${query_num}.sql EOF - exit 0 + return 0 2>/dev/null fi if [[ ! -x "${binary}" ]]; then diff --git a/script/setup.sh b/script/setup.sh index 3e6f8f2..087e96a 100755 --- a/script/setup.sh +++ b/script/setup.sh @@ -11,7 +11,7 @@ packages=( if ! command -v apt-get >/dev/null 2>&1; then echo "apt-get not found; assuming build dependencies are already installed." - exit 0 + return 0 2>/dev/null fi missing=() @@ -23,14 +23,14 @@ done if ((${#missing[@]} == 0)); then echo "Build dependencies already installed." - exit 0 + return 0 2>/dev/null fi if [[ "$(id -u)" -eq 0 ]]; then apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${missing[@]}" rm -rf /var/lib/apt/lists/* - exit 0 + return 0 2>/dev/null fi sudo apt-get update diff --git a/scripts/dashboard.py b/scripts/dashboard.py new file mode 100644 index 0000000..9d4f052 --- /dev/null +++ b/scripts/dashboard.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import pathlib +import statistics +import subprocess +import sys +import tempfile +import time + +README_START = "" +README_END = "" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run the first N benchmark queries, write CSV stats, and refresh the README benchmark table." + ) + parser.add_argument("--count", type=int, default=17, help="Number of first queries to benchmark.") + parser.add_argument("--warm-runs", type=int, default=4, help="Number of warm runs after the first measured run.") + parser.add_argument( + "--query-dir", + type=pathlib.Path, + default=pathlib.Path("benchmarks/queries"), + help="Directory with query_.sql files.", + ) + parser.add_argument( + "--input", + type=pathlib.Path, + default=pathlib.Path("benchmarks/hits_sample.columnar"), + help="Columnar input file.", + ) + parser.add_argument( + "--binary", + type=pathlib.Path, + default=pathlib.Path("build/src/columnar_engine"), + help="Path to the columnar_engine binary.", + ) + parser.add_argument( + "--csv-out", + type=pathlib.Path, + default=pathlib.Path("benchmarks/results/readme_benchmarks.csv"), + help="Where to write the benchmark CSV.", + ) + parser.add_argument( + "--readme", + type=pathlib.Path, + default=pathlib.Path("README.md"), + help="README file to update.", + ) + parser.add_argument( + "--table-name", + default="hits", + help="Logical table name passed to run-query.", + ) + return parser.parse_args() + + +def repo_root() -> pathlib.Path: + return pathlib.Path(__file__).resolve().parent.parent + + +def resolve_path(path: pathlib.Path, root: pathlib.Path) -> pathlib.Path: + return path if path.is_absolute() else root / path + + +def format_ms(value: float) -> str: + return f"{value:.2f}" + + +def compact_sql(sql: str, max_len: int = 88) -> str: + normalized = " ".join(sql.split()) + if len(normalized) <= max_len: + return normalized + return normalized[: max_len - 1] + "…" + + +def markdown_escape(text: str) -> str: + return text.replace("|", "\\|") + + +def run_query( + binary: pathlib.Path, + input_path: pathlib.Path, + query_path: pathlib.Path, + output_path: pathlib.Path, + table_name: str, +) -> float: + command = [ + str(binary), + "run-query", + "--input", + str(input_path), + "--output", + str(output_path), + "--table-name", + table_name, + "--query-file", + str(query_path), + ] + + started_at = time.perf_counter() + completed = subprocess.run(command, capture_output=True, text=True) + elapsed_ms = (time.perf_counter() - started_at) * 1000.0 + + if completed.returncode != 0: + stderr = completed.stderr.strip() + stdout = completed.stdout.strip() + message = stderr or stdout or f"exit code {completed.returncode}" + raise RuntimeError(message) + + return elapsed_ms + + +def collect_rows( + *, + root: pathlib.Path, + binary: pathlib.Path, + input_path: pathlib.Path, + query_dir: pathlib.Path, + count: int, + warm_runs: int, + table_name: str, +) -> list[dict[str, str]]: + rows: list[dict[str, str]] = [] + + with tempfile.TemporaryDirectory(prefix="readme_benchmarks_") as temp_dir_name: + temp_dir = pathlib.Path(temp_dir_name) + + for query_id in range(count): + query_path = query_dir / f"query_{query_id}.sql" + if not query_path.exists(): + break + + sql = query_path.read_text(encoding="utf-8").strip() + output_path = temp_dir / f"query_{query_id}.csv" + + first_error = "" + status = "ok" + first_run_ms = "" + warm_avg_ms = "" + warm_min_ms = "" + warm_max_ms = "" + warm_median_ms = "" + + try: + first_run_value = run_query(binary, input_path, query_path, output_path, table_name) + first_run_ms = format_ms(first_run_value) + + warm_values: list[float] = [] + for _ in range(warm_runs): + warm_values.append(run_query(binary, input_path, query_path, output_path, table_name)) + + if warm_values: + warm_avg_ms = format_ms(sum(warm_values) / len(warm_values)) + warm_min_ms = format_ms(min(warm_values)) + warm_max_ms = format_ms(max(warm_values)) + warm_median_ms = format_ms(statistics.median(warm_values)) + except RuntimeError as error: + status = "failed" + first_error = str(error) + + rows.append( + { + "query_id": str(query_id), + "query_file": str(query_path.relative_to(root)), + "sql": sql, + "first_run_ms": first_run_ms, + "warm_avg_ms": warm_avg_ms, + "warm_median_ms": warm_median_ms, + "warm_min_ms": warm_min_ms, + "warm_max_ms": warm_max_ms, + "status": status, + "error": first_error, + } + ) + + return rows + + +def write_csv(csv_path: pathlib.Path, rows: list[dict[str, str]]) -> None: + csv_path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = [ + "query_id", + "query_file", + "sql", + "first_run_ms", + "warm_avg_ms", + "warm_median_ms", + "warm_min_ms", + "warm_max_ms", + "status", + "error", + ] + with csv_path.open("w", encoding="utf-8", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def render_markdown(csv_path: pathlib.Path, rows: list[dict[str, str]], count: int, warm_runs: int) -> str: + generated_at = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + lines = [ + "## Dashboard", + "", + "", + "| Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status |", + "| --- | ---: | ---: | ---: | ---: | ---: | --- |", + ] + + for row in rows: + query_file = row["query_file"] + query_label = f"[{row['query_id']}]({query_file})" + status = row["status"] + if row["error"]: + status = f"{status}: {markdown_escape(compact_sql(row['error'], max_len=48))}" + + lines.append( + "| " + + " | ".join( + [ + query_label, + row["first_run_ms"] or "—", + row["warm_avg_ms"] or "—", + row["warm_median_ms"] or "—", + row["warm_min_ms"] or "—", + row["warm_max_ms"] or "—", + status, + ] + ) + + " |" + ) + + return "\n".join(lines) + + +def update_readme(readme_path: pathlib.Path, markdown_block: str) -> None: + if readme_path.exists(): + content = readme_path.read_text(encoding="utf-8") + else: + content = "" + + wrapped = f"{README_START}\n{markdown_block}\n{README_END}" + + if README_START in content and README_END in content: + start = content.index(README_START) + end = content.index(README_END) + len(README_END) + updated = content[:start] + wrapped + content[end:] + else: + suffix = "" if not content or content.endswith("\n") else "\n" + updated = content + suffix + "\n" + wrapped + "\n" + + readme_path.write_text(updated, encoding="utf-8") + + +def main() -> int: + args = parse_args() + if args.count <= 0: + raise SystemExit("--count must be > 0") + if args.warm_runs < 0: + raise SystemExit("--warm-runs must be >= 0") + + root = repo_root() + binary = resolve_path(args.binary, root) + input_path = resolve_path(args.input, root) + query_dir = resolve_path(args.query_dir, root) + csv_path = resolve_path(args.csv_out, root) + readme_path = resolve_path(args.readme, root) + + if not binary.exists(): + raise SystemExit(f"binary not found: {binary}") + if not input_path.exists(): + raise SystemExit(f"input file not found: {input_path}") + if not query_dir.exists(): + raise SystemExit(f"query directory not found: {query_dir}") + + rows = collect_rows( + root=root, + binary=binary, + input_path=input_path, + query_dir=query_dir, + count=args.count, + warm_runs=args.warm_runs, + table_name=args.table_name, + ) + write_csv(csv_path, rows) + markdown_block = render_markdown(csv_path.relative_to(root), rows, args.count, args.warm_runs) + update_readme(readme_path, markdown_block) + print(f"updated {readme_path.relative_to(root)} from {csv_path.relative_to(root)} with {len(rows)} row(s)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2e0e551..22adee1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,35 +1,67 @@ -add_library(columnar_engine_lib +add_library(columnar_engine_core model/batch.cpp - io/batch_io.cpp model/column.cpp - model/column_boolean.cpp - model/column_character.cpp - model/column_date.cpp - model/column_int16.cpp - model/column_int32.cpp - model/column_int128.cpp - model/column_int64.cpp model/column_string.cpp - model/column_timestamp.cpp - convert/csv_columnar.cpp - io/csv.cpp - support/error.cpp - support/ascii.cpp - io/fileio.cpp model/metadata.cpp - support/parsing.cpp - model/schema.cpp + io/batch.cpp + io/file.cpp io/stream.cpp + common/error.cpp + common/ascii.cpp + common/parsing.cpp +) + +target_include_directories(columnar_engine_core PUBLIC ${COLUMNAR_INCLUDE_DIRS}) + +add_library(columnar_engine_columnar + io/columnar_batch.cpp +) + +target_link_libraries(columnar_engine_columnar PUBLIC columnar_engine_core) + +add_library(columnar_engine_csv + io/csv.cpp + io/csv_batch.cpp + model/schema_csv.cpp +) + +target_link_libraries(columnar_engine_csv PUBLIC columnar_engine_core) + +add_library(columnar_engine_convert + convert/csv_columnar.cpp +) + +target_link_libraries(columnar_engine_convert + PUBLIC + columnar_engine_columnar + columnar_engine_csv +) + +add_library(columnar_engine_executor executor/executor.cpp executor/query_parser.cpp executor/query_planner.cpp + executor/planner_utils.cpp executor/aggregate_states.cpp + executor/operator_utils.cpp executor/operators.cpp sql_parser/tokenizer_factory.cpp sql_parser/tokenizer_tokens.cpp sql_parser/tokenizer.cpp ) -target_include_directories(columnar_engine_lib PUBLIC ${COLUMNAR_INCLUDE_DIRS}) +target_link_libraries(columnar_engine_executor PUBLIC columnar_engine_columnar) + +add_library(columnar_engine_lib INTERFACE) + +target_link_libraries(columnar_engine_lib + INTERFACE + columnar_engine_core + columnar_engine_columnar + columnar_engine_csv + columnar_engine_convert + columnar_engine_executor +) + add_executable(columnar_engine app/main.cpp) target_link_libraries(columnar_engine PRIVATE columnar_engine_lib argparse) diff --git a/src/app/main.cpp b/src/app/main.cpp index 31ed2c5..dfc3d62 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -3,12 +3,13 @@ #include #include -#include "csv_batch_io.h" -#include "csv_columnar.h" -#include "error.h" -#include "executor.h" -#include "fileio.h" -#include "schema.h" +#include "convert/csv_columnar.h" +#include "executor/executor.h" +#include "io/csv_batch.h" +#include "io/file.h" +#include "model/schema.h" +#include "model/schema_csv.h" +#include "common/error.h" void ConfigureInferSchemaCommand(argparse::ArgumentParser& command) { command.add_description("Infer a schema from a CSV file."); @@ -21,7 +22,7 @@ void ConfigureConvertCommand(argparse::ArgumentParser& command) { command.add_argument("--schema").required(); command.add_argument("--input").required(); command.add_argument("--output").required(); - command.add_argument("--row-group-size").scan<'u', size_t>().default_value(50000); + command.add_argument("--row-group-size").scan<'u', size_t>().default_value(size_t{50000}); } void ConfigureRoundtripCommand(argparse::ArgumentParser& command) { @@ -43,27 +44,34 @@ void ConfigureRunQueryCommand(argparse::ArgumentParser& command) { int RunInferSchema(const argparse::ArgumentParser& command) { const auto input_path = std::filesystem::path(command.get("--input")); const auto output_path = std::filesystem::path(command.get("--output")); + EnsureParentDirectory(output_path); WriteSchemaCsv(output_path, InferSchemaCsv(input_path)); + return 0; } int RunConvert(const argparse::ArgumentParser& command) { const auto output_path = std::filesystem::path(command.get("--output")); + EnsureParentDirectory(output_path); ConvertCsvToColumnar(std::filesystem::path(command.get("--schema")), std::filesystem::path(command.get("--input")), output_path, command.get("--row-group-size")); + return 0; } int RunRoundtrip(const argparse::ArgumentParser& command) { const auto schema_output_path = std::filesystem::path(command.get("--schema-output")); const auto csv_output_path = std::filesystem::path(command.get("--csv-output")); + EnsureParentDirectory(schema_output_path); EnsureParentDirectory(csv_output_path); + ConvertColumnarToCsv(std::filesystem::path(command.get("--input")), schema_output_path, csv_output_path); + return 0; } @@ -117,12 +125,15 @@ int main(const int argc, char** argv) { if (program.is_subcommand_used("infer-schema")) { return RunInferSchema(infer_schema_command); } + if (program.is_subcommand_used("convert")) { return RunConvert(convert_command); } + if (program.is_subcommand_used("roundtrip")) { return RunRoundtrip(roundtrip_command); } + if (program.is_subcommand_used("run-query")) { return RunQuery(run_query_command); } diff --git a/src/support/ascii.cpp b/src/common/ascii.cpp similarity index 94% rename from src/support/ascii.cpp rename to src/common/ascii.cpp index 19246ec..8a2ae21 100644 --- a/src/support/ascii.cpp +++ b/src/common/ascii.cpp @@ -1,21 +1,25 @@ -#include "ascii.h" +#include "common/ascii.h" #include std::string ToLowerAscii(const std::string_view input) { std::string output; output.reserve(input.size()); + for (const unsigned char ch : input) { output.push_back(static_cast(std::tolower(ch))); } + return output; } std::string ToUpperAscii(const std::string_view input) { std::string output; output.reserve(input.size()); + for (const unsigned char ch : input) { output.push_back(static_cast(std::toupper(ch))); } + return output; } diff --git a/src/support/error.cpp b/src/common/error.cpp similarity index 65% rename from src/support/error.cpp rename to src/common/error.cpp index f374ff2..c4dcbd3 100644 --- a/src/support/error.cpp +++ b/src/common/error.cpp @@ -1,4 +1,4 @@ -#include "error.h" +#include "common/error.h" #include @@ -6,50 +6,47 @@ std::string Error::BuildWhat(const std::string_view module, const std::string_vi return std::string(module) + ": " + std::string(message); } -Error::Error(const Code code, std::string module, std::string module_instance, const Type type, std::string message) +Error::Error(const Code code, std::string module, std::string module_instance, std::string message) : code_(code), module_(std::move(module)), module_instance_(std::move(module_instance)), - type_(type), message_(std::move(message)), what_(BuildWhat(module_, message_)) {} Error Error::InvalidArgument(std::string module, std::string message, std::string module_instance) { - return Error(Code::InvalidArgument, std::move(module), std::move(module_instance), Type::Validation, - std::move(message)); + return Error(Code::InvalidArgument, std::move(module), std::move(module_instance), std::move(message)); } -Error Error::InvalidData(std::string module, std::string message, std::string module_instance) { - return Error(Code::InvalidData, std::move(module), std::move(module_instance), Type::Data, std::move(message)); +Error Error::MalformedData(std::string module, std::string message, std::string module_instance) { + return Error(Code::MalformedData, std::move(module), std::move(module_instance), std::move(message)); } Error Error::NotFound(std::string module, std::string message, std::string module_instance) { - return Error(Code::NotFound, std::move(module), std::move(module_instance), Type::State, std::move(message)); + return Error(Code::NotFound, std::move(module), std::move(module_instance), std::move(message)); } Error Error::OutOfRange(std::string module, std::string message, std::string module_instance) { - return Error(Code::OutOfRange, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); + return Error(Code::OutOfRange, std::move(module), std::move(module_instance), std::move(message)); } -Error Error::Mismatch(std::string module, std::string message, std::string module_instance) { - return Error(Code::Mismatch, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); +Error Error::InconsistentData(std::string module, std::string message, std::string module_instance) { + return Error(Code::InconsistentData, std::move(module), std::move(module_instance), std::move(message)); } Error Error::Overflow(std::string module, std::string message, std::string module_instance) { - return Error(Code::Overflow, std::move(module), std::move(module_instance), Type::Validation, std::move(message)); + return Error(Code::Overflow, std::move(module), std::move(module_instance), std::move(message)); } Error Error::InvalidState(std::string module, std::string message, std::string module_instance) { - return Error(Code::InvalidState, std::move(module), std::move(module_instance), Type::State, std::move(message)); + return Error(Code::InvalidState, std::move(module), std::move(module_instance), std::move(message)); } Error Error::Unsupported(std::string module, std::string message, std::string module_instance) { - return Error(Code::Unsupported, std::move(module), std::move(module_instance), Type::Validation, - std::move(message)); + return Error(Code::Unsupported, std::move(module), std::move(module_instance), std::move(message)); } Error Error::Io(std::string module, std::string message, std::string module_instance) { - return Error(Code::IoFailure, std::move(module), std::move(module_instance), Type::Io, std::move(message)); + return Error(Code::IoFailure, std::move(module), std::move(module_instance), std::move(message)); } Error Error::PathIo(std::string module, const std::filesystem::path& path, const std::string_view action) { @@ -60,8 +57,6 @@ const char* Error::what() const noexcept { return what_.c_str(); } Error::Code Error::GetCode() const noexcept { return code_; } -Error::Type Error::GetType() const noexcept { return type_; } - const std::string& Error::GetModule() const noexcept { return module_; } const std::string& Error::GetModuleInstance() const noexcept { return module_instance_; } @@ -72,14 +67,14 @@ std::string_view Error::CodeToString(const Code code) noexcept { switch (code) { case Code::InvalidArgument: return "invalid_argument"; - case Code::InvalidData: - return "invalid_data"; + case Code::MalformedData: + return "malformed_data"; case Code::NotFound: return "not_found"; case Code::OutOfRange: return "out_of_range"; - case Code::Mismatch: - return "mismatch"; + case Code::InconsistentData: + return "inconsistent_data"; case Code::Overflow: return "overflow"; case Code::InvalidState: @@ -89,19 +84,6 @@ std::string_view Error::CodeToString(const Code code) noexcept { case Code::IoFailure: return "io_failure"; } - return "unknown"; -} -std::string_view Error::TypeToString(const Type type) noexcept { - switch (type) { - case Type::Validation: - return "validation"; - case Type::Data: - return "data"; - case Type::State: - return "state"; - case Type::Io: - return "io"; - } return "unknown"; } diff --git a/src/support/parsing.cpp b/src/common/parsing.cpp similarity index 59% rename from src/support/parsing.cpp rename to src/common/parsing.cpp index 2ddda39..302b9dd 100644 --- a/src/support/parsing.cpp +++ b/src/common/parsing.cpp @@ -1,64 +1,86 @@ -#include "parsing.h" +#include "common/parsing.h" #include #include +#include #include #include +#include #include -#include "ascii.h" -#include "error.h" +#include "common/ascii.h" +#include "common/error.h" template -T ParseInteger(const std::string& value, const std::string_view type_name) { +bool TryParseInteger(const std::string_view value, T& result) { if (value.empty()) { - throw Error::InvalidData("parsing", "empty value for " + std::string(type_name)); + return false; } - T result = 0; const char* begin = value.data(); const char* end = begin + value.size(); + const auto [ptr, ec] = std::from_chars(begin, end, result); + + return ec == std::errc() && ptr == end; +} + +template +T ParseInteger(const std::string& value, const std::string_view type_name) { + if (value.empty()) { + throw Error::MalformedData("common", "empty value for " + std::string(type_name)); + } - if (const auto [ptr, ec] = std::from_chars(begin, end, result); ec != std::errc() || ptr != end) { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); + T result = 0; + if (!TryParseInteger(std::string_view(value), result)) { + throw Error::MalformedData("common", "invalid " + std::string(type_name) + " value"); } return result; } -static unsigned ParseUnsignedPart(const std::string_view input, const size_t offset, const size_t length, - const std::string_view type_name) { - unsigned value = 0; +static bool TryParseUnsignedPart(const std::string_view input, const size_t offset, const size_t length, + unsigned& value) { const char* begin = input.data() + offset; const char* end = begin + length; - - if (const auto [ptr, ec] = std::from_chars(begin, end, value); ec != std::errc() || ptr != end) { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); - } - - return value; + const auto [ptr, ec] = std::from_chars(begin, end, value); + return ec == std::errc() && ptr == end; } -static std::chrono::sys_days ParseDatePoint(const std::string_view value, const std::string_view type_name) { +static bool TryParseDatePoint(const std::string_view value, std::chrono::sys_days& result) { if (value.size() != 10 || value[4] != '-' || value[7] != '-') { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); + return false; } - const int year = static_cast(ParseUnsignedPart(value, 0, 4, type_name)); - const unsigned month = ParseUnsignedPart(value, 5, 2, type_name); - const unsigned day = ParseUnsignedPart(value, 8, 2, type_name); + unsigned year = 0; + unsigned month = 0; + unsigned day = 0; + + if (!TryParseUnsignedPart(value, 0, 4, year) || !TryParseUnsignedPart(value, 5, 2, month) || + !TryParseUnsignedPart(value, 8, 2, day)) { + return false; + } const std::chrono::year_month_day ymd{ - std::chrono::year{year}, + std::chrono::year{static_cast(year)}, std::chrono::month{month}, std::chrono::day{day}, }; if (!ymd.ok()) { - throw Error::InvalidData("parsing", "invalid " + std::string(type_name) + " value"); + return false; } - return std::chrono::sys_days{ymd}; + result = std::chrono::sys_days{ymd}; + + return true; +} + +static std::chrono::sys_days ParseDatePoint(const std::string_view value, const std::string_view type_name) { + std::chrono::sys_days result{}; + if (!TryParseDatePoint(value, result)) { + throw Error::MalformedData("common", "invalid " + std::string(type_name) + " value"); + } + return result; } static std::string FormatDateParts(const std::chrono::year_month_day ymd) { @@ -68,7 +90,7 @@ static std::string FormatDateParts(const std::chrono::year_month_day ymd) { return out.str(); } -bool ParseBoolean(const std::string& value) { +std::optional TryParseBoolean(const std::string_view value) { const std::string lowered = ToLowerAscii(value); if (lowered == "true" || lowered == "1") { return true; @@ -76,18 +98,50 @@ bool ParseBoolean(const std::string& value) { if (lowered == "false" || lowered == "0") { return false; } - throw Error::InvalidData("parsing", "invalid bool value"); + return std::nullopt; +} + +bool ParseBoolean(const std::string& value) { + const std::optional result = TryParseBoolean(value); + if (!result.has_value()) { + throw Error::MalformedData("common", "invalid bool value"); + } + return *result; +} + +std::optional TryParseInt16(const std::string_view value) { + int16_t result = 0; + if (!TryParseInteger(value, result)) { + return std::nullopt; + } + return result; } int16_t ParseInt16(const std::string& value) { return ParseInteger(value, "int16"); } +std::optional TryParseInt32(const std::string_view value) { + int32_t result = 0; + if (!TryParseInteger(value, result)) { + return std::nullopt; + } + return result; +} + int32_t ParseInt32(const std::string& value) { return ParseInteger(value, "int32"); } +std::optional TryParseInt64(const std::string_view value) { + int64_t result = 0; + if (!TryParseInteger(value, result)) { + return std::nullopt; + } + return result; +} + int64_t ParseInt64(const std::string& value) { return ParseInteger(value, "int64"); } -Int128 ParseInt128(const std::string& value) { +std::optional TryParseInt128(const std::string_view value) { if (value.empty()) { - throw Error::InvalidData("parsing", "empty value for int128"); + return std::nullopt; } size_t pos = 0; @@ -99,7 +153,7 @@ Int128 ParseInt128(const std::string& value) { } if (pos == value.size()) { - throw Error::InvalidData("parsing", "invalid int128 value"); + return std::nullopt; } const UInt128 positive_limit = ~UInt128{0} >> 1; @@ -111,19 +165,21 @@ Int128 ParseInt128(const std::string& value) { for (; pos < value.size(); ++pos) { const char ch = value[pos]; if (ch < '0' || ch > '9') { - throw Error::InvalidData("parsing", "invalid int128 value"); + return std::nullopt; } const auto digit = static_cast(ch - '0'); if (magnitude > (limit - digit) / 10) { - throw Error::InvalidData("parsing", "invalid int128 value"); + return std::nullopt; } + magnitude = magnitude * 10 + digit; } if (!negative) { return static_cast(magnitude); } + if (magnitude == negative_limit) { return -static_cast(positive_limit) - 1; } @@ -131,53 +187,95 @@ Int128 ParseInt128(const std::string& value) { return -static_cast(magnitude); } +Int128 ParseInt128(const std::string& value) { + if (value.empty()) { + throw Error::MalformedData("common", "empty value for int128"); + } + + const std::optional result = TryParseInt128(value); + if (!result.has_value()) { + throw Error::MalformedData("common", "invalid int128 value"); + } + + return *result; +} + +std::optional TryParseDate(const std::string_view value) { + std::chrono::sys_days day_point{}; + if (!TryParseDatePoint(value, day_point)) { + return std::nullopt; + } + + const auto days_since_epoch = day_point.time_since_epoch().count(); + + if (days_since_epoch < std::numeric_limits::min() || + days_since_epoch > std::numeric_limits::max()) { + return std::nullopt; + } + + return days_since_epoch; +} + int32_t ParseDate(const std::string& value) { const auto day_point = ParseDatePoint(value, "date"); const auto days_since_epoch = day_point.time_since_epoch().count(); if (days_since_epoch < std::numeric_limits::min() || days_since_epoch > std::numeric_limits::max()) { - throw Error::Overflow("parsing", "date value exceeds supported range"); + throw Error::Overflow("common", "date value exceeds supported range"); } return days_since_epoch; } -int64_t ParseTimestamp(const std::string& value) { +std::optional TryParseTimestamp(const std::string_view value) { if (value.size() < 19) { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; } - const auto day_point = ParseDatePoint(std::string_view(value).substr(0, 10), "timestamp"); + std::chrono::sys_days day_point{}; + if (!TryParseDatePoint(value.substr(0, 10), day_point)) { + return std::nullopt; + } if (value[10] != ' ' && value[10] != 'T') { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; } + if (value[13] != ':' || value[16] != ':') { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; } - const unsigned hour = ParseUnsignedPart(value, 11, 2, "timestamp"); - const unsigned minute = ParseUnsignedPart(value, 14, 2, "timestamp"); - const unsigned second = ParseUnsignedPart(value, 17, 2, "timestamp"); + unsigned hour = 0; + unsigned minute = 0; + unsigned second = 0; + + if (!TryParseUnsignedPart(value, 11, 2, hour) || !TryParseUnsignedPart(value, 14, 2, minute) || + !TryParseUnsignedPart(value, 17, 2, second)) { + return std::nullopt; + } if (hour > 23 || minute > 59 || second > 59) { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; } int64_t microseconds = 0; if (value.size() > 19) { if (value[19] != '.') { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; } const size_t digits = value.size() - 20; if (digits == 0 || digits > 6) { - throw Error::InvalidData("parsing", "invalid timestamp value"); + return std::nullopt; + } + + unsigned fractional = 0; + if (!TryParseUnsignedPart(value, 20, digits, fractional)) { + return std::nullopt; } - const auto fractional = ParseUnsignedPart(value, 20, digits, "timestamp"); microseconds = static_cast(fractional); for (size_t i = digits; i < 6; ++i) { microseconds *= 10; @@ -190,13 +288,29 @@ int64_t ParseTimestamp(const std::string& value) { return std::chrono::duration_cast(timestamp.time_since_epoch()).count(); } -char ParseCharacter(const std::string& value) { +int64_t ParseTimestamp(const std::string& value) { + const std::optional result = TryParseTimestamp(value); + if (!result.has_value()) { + throw Error::MalformedData("common", "invalid timestamp value"); + } + return *result; +} + +std::optional TryParseCharacter(const std::string_view value) { if (value.size() != 1) { - throw Error::InvalidData("parsing", "invalid char value"); + return std::nullopt; } return value[0]; } +char ParseCharacter(const std::string& value) { + const std::optional result = TryParseCharacter(value); + if (!result.has_value()) { + throw Error::MalformedData("common", "invalid char value"); + } + return *result; +} + std::string BooleanToString(const bool value) { return value ? "true" : "false"; } std::string Int128ToString(const Int128 value) { @@ -220,6 +334,7 @@ std::string Int128ToString(const Int128 value) { out.push_back(static_cast('0' + magnitude % 10)); magnitude /= 10; } + if (negative) { out.push_back('-'); } @@ -237,7 +352,7 @@ std::string DateToString(const int32_t value) { std::string TimestampToString(const int64_t value) { const std::chrono::sys_time timestamp{std::chrono::microseconds{value}}; const auto day_point = std::chrono::floor(timestamp); - const std::chrono::hh_mm_ss tod{timestamp - day_point}; + const std::chrono::hh_mm_ss tod{timestamp - day_point}; std::ostringstream out; out << FormatDateParts(std::chrono::year_month_day{day_point}) << ' ' << std::setfill('0') << std::setw(2) @@ -260,34 +375,44 @@ std::string TimestampToString(const int64_t value) { ColumnType ParseColumnType(const std::string_view input) { const std::string lowered = ToLowerAscii(input); + if (lowered == "bool") { return ColumnType::Boolean; } + if (lowered == "int16") { return ColumnType::Int16; } + if (lowered == "int32") { return ColumnType::Int32; } + if (lowered == "int64") { return ColumnType::Int64; } + if (lowered == "int128") { return ColumnType::Int128; } + if (lowered == "string") { return ColumnType::String; } + if (lowered == "date") { return ColumnType::Date; } + if (lowered == "timestamp") { return ColumnType::Timestamp; } + if (lowered == "char") { return ColumnType::Character; } - throw Error::Unsupported("parsing", "unknown column type"); + + throw Error::Unsupported("common", "unknown column type"); } std::string ColumnTypeToString(const ColumnType type) { @@ -311,5 +436,6 @@ std::string ColumnTypeToString(const ColumnType type) { case ColumnType::Character: return "char"; } - throw Error::Unsupported("parsing", "unknown column type"); + + throw Error::Unsupported("common", "unknown column type"); } diff --git a/src/convert/csv_columnar.cpp b/src/convert/csv_columnar.cpp index 3f9d600..eff1ace 100644 --- a/src/convert/csv_columnar.cpp +++ b/src/convert/csv_columnar.cpp @@ -1,14 +1,16 @@ -#include "csv_columnar.h" +#include "convert/csv_columnar.h" -#include "columnar_batch_io.h" -#include "csv_batch_io.h" -#include "error.h" -#include "schema.h" +#include + +#include "common/error.h" +#include "io/columnar_batch.h" +#include "io/csv_batch.h" +#include "model/schema_csv.h" void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, const std::filesystem::path& output_path, size_t max_rows_per_group) { if (max_rows_per_group == 0) { - throw Error::InvalidArgument("columnar", "row group size must be > 0"); + throw Error::InvalidArgument("convert", "row group size must be > 0"); } const Schema schema = ReadSchemaCsv(schema_path); @@ -23,7 +25,7 @@ void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::f batch_writer.Write(*batch); } - batch_writer.Finalize(); + std::move(batch_writer).Finalize(); } void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index e9d00ef..2c2b95e 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -4,59 +4,22 @@ #include #include -#include "aggregate_function.h" -#include "ascii.h" -#include "executor.h" -#include "int128.h" -#include "parsing.h" +#include "common/ascii.h" +#include "common/error.h" +#include "common/int128.h" +#include "common/parsing.h" +#include "executor/aggregate_function.h" +#include "executor/aggregate_state.h" +#include "model/column_traits.h" static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view candidate, const std::string_view current, const bool is_min) { - switch (type) { - case ColumnType::Boolean: { - const bool left = ParseBoolean(std::string(candidate)); - const bool right = ParseBoolean(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Int16: { - const int16_t left = ParseInt16(std::string(candidate)); - const int16_t right = ParseInt16(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Int32: { - const int32_t left = ParseInt32(std::string(candidate)); - const int32_t right = ParseInt32(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Int64: { - const int64_t left = ParseInt64(std::string(candidate)); - const int64_t right = ParseInt64(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Int128: { - const Int128 left = ParseInt128(std::string(candidate)); - const Int128 right = ParseInt128(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::String: - return is_min ? candidate < current : candidate > current; - case ColumnType::Date: { - const int32_t left = ParseDate(std::string(candidate)); - const int32_t right = ParseDate(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Timestamp: { - const int64_t left = ParseTimestamp(std::string(candidate)); - const int64_t right = ParseTimestamp(std::string(current)); - return is_min ? left < right : left > right; - } - case ColumnType::Character: { - const char left = ParseCharacter(std::string(candidate)); - const char right = ParseCharacter(std::string(current)); - return is_min ? left < right : left > right; - } - } - throw Error::Unsupported("executor", "unsupported column type in comparison"); + return VisitColumnType(type, [&]() { + const auto left = ColumnValueTraits::Parse(std::string(candidate)); + const auto right = ColumnValueTraits::Parse(std::string(current)); + + return is_min ? left < right : left > right; + }); } static Int128 ParseToInt128(const ColumnType type, const std::string_view value) { @@ -78,86 +41,21 @@ static Int128 ParseToInt128(const ColumnType type, const std::string_view value) case ColumnType::Timestamp: break; } - throw Error::Unsupported("executor", "unsupported numeric conversion"); -} - -static UInt128 ToMagnitude(const Int128 value) { - if (value >= 0) { - return static_cast(value); - } - - UInt128 magnitude = static_cast(-(value + 1)); - ++magnitude; - return magnitude; -} - -static std::string UInt128ToString(UInt128 value) { - if (value == 0) { - return "0"; - } - - std::string out; - - while (value > 0) { - out.push_back(static_cast('0' + value % 10)); - value /= 10; - } - - std::ranges::reverse(out); - - return out; + throw Error::Unsupported("executor", "unsupported numeric conversion"); } static std::string FormatAverage(const Int128 sum, const uint64_t count) { if (count == 0) { return "0"; } - - const bool negative = sum < 0; - const UInt128 magnitude = ToMagnitude(sum); - const UInt128 divisor = count; - const UInt128 integer = magnitude / divisor; - UInt128 remainder = magnitude % divisor; - - std::string output = UInt128ToString(integer); - - if (remainder != 0) { - output.push_back('.'); - - std::string fractional; - fractional.reserve(6); - - for (size_t i = 0; i < 6 && remainder != 0; ++i) { - remainder *= 10; - fractional.push_back(static_cast('0' + remainder / divisor)); - remainder %= divisor; - } - - while (!fractional.empty() && fractional.back() == '0') { - fractional.pop_back(); - } - - if (!fractional.empty()) { - output += fractional; - } else { - output.pop_back(); - } - } - - if (negative && output != "0") { - output.insert(output.begin(), '-'); - } - - return output; + return Int128ToString(sum / static_cast(count)); } class CountAS final : public AggState { public: - void Consume(const std::optional value) override { - (void)value; - ++count_; - } + void ConsumeValue(const std::string_view) override { ConsumeRow(); } + void ConsumeRow() override { ++count_; } std::string Finalize() const override { return std::to_string(count_); } @@ -169,13 +67,8 @@ class SumAS final : public AggState { public: explicit SumAS(const ColumnType type) : type_(type) {} - public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "SUM requires a value"); - } - sum_ += ParseToInt128(type_, *value); - } + void ConsumeValue(const std::string_view value) override { sum_ += ParseToInt128(type_, value); } + void ConsumeRow() override { throw Error::InvalidState("executor", "SUM requires a value"); } std::string Finalize() const override { return Int128ToString(sum_); } @@ -188,14 +81,11 @@ class AvgAS final : public AggState { public: explicit AvgAS(const ColumnType type) : type_(type) {} - public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "AVG requires a value"); - } - sum_ += ParseToInt128(type_, *value); + void ConsumeValue(const std::string_view value) override { + sum_ += ParseToInt128(type_, value); ++count_; } + void ConsumeRow() override { throw Error::InvalidState("executor", "AVG requires a value"); } std::string Finalize() const override { return FormatAverage(sum_, count_); } @@ -209,21 +99,17 @@ class ExtremumAS final : public AggState { public: ExtremumAS(const ColumnType type, const bool is_min) : type_(type), is_min_(is_min) {} - public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "MIN/MAX requires a value"); - } - + void ConsumeValue(const std::string_view value) override { if (!extremum_.has_value()) { - extremum_ = std::string(*value); + extremum_ = std::string(value); return; } - if (ShouldReplaceExtremum(type_, *value, *extremum_, is_min_)) { - extremum_ = std::string(*value); + if (ShouldReplaceExtremum(type_, value, *extremum_, is_min_)) { + extremum_ = std::string(value); } } + void ConsumeRow() override { throw Error::InvalidState("executor", "MIN/MAX requires a value"); } std::string Finalize() const override { return extremum_.value_or(""); } @@ -237,16 +123,12 @@ class DistinctAS final : public AggState { public: explicit DistinctAS(std::unique_ptr nested) : nested_(std::move(nested)) {} - public: - void Consume(const std::optional value) override { - if (!value.has_value()) { - throw Error::InvalidState("executor", "DISTINCT requires a value"); - } - - if (seen_.insert(std::string(*value)).second) { - nested_->Consume(value); + void ConsumeValue(const std::string_view value) override { + if (seen_.insert(std::string(value)).second) { + nested_->ConsumeValue(value); } } + void ConsumeRow() override { throw Error::InvalidState("executor", "DISTINCT requires a value"); } std::string Finalize() const override { return nested_->Finalize(); } @@ -255,10 +137,7 @@ class DistinctAS final : public AggState { std::unordered_set seen_; }; -static bool SupportsAnyType(const ColumnType type) { - (void)type; - return true; -} +static bool SupportsAnyType(const ColumnType) { return true; } static bool SupportsNumericType(const ColumnType type) { switch (type) { @@ -274,13 +153,11 @@ static bool SupportsNumericType(const ColumnType type) { case ColumnType::Timestamp: return false; } + return false; } -static std::unique_ptr CreateCountState(const PlannedAgg& aggregate) { - (void)aggregate; - return std::make_unique(); -} +static std::unique_ptr CreateCountState(const PlannedAgg&) { return std::make_unique(); } static std::unique_ptr CreateSumState(const PlannedAgg& aggregate) { return std::make_unique(aggregate.input_type); @@ -317,40 +194,31 @@ AggRegistrar::AggRegistrar(const AggFuncDefinition& def) { AggRegistry::Instance [[maybe_unused]] static AggRegistrar register_count({ .canonical_name = "COUNT", - .distinct = true, - .star = true, + .call_features = AggCallFeature::Distinct | AggCallFeature::Star, .supports_type = &SupportsAnyType, .factory = &CreateCountState, }); [[maybe_unused]] static AggRegistrar register_sum({ .canonical_name = "SUM", - .distinct = false, - .star = false, .supports_type = &SupportsNumericType, .factory = &CreateSumState, }); [[maybe_unused]] static AggRegistrar register_avg({ .canonical_name = "AVG", - .distinct = false, - .star = false, .supports_type = &SupportsNumericType, .factory = &CreateAvgState, }); [[maybe_unused]] static AggRegistrar register_min({ .canonical_name = "MIN", - .distinct = false, - .star = false, .supports_type = &SupportsAnyType, .factory = &CreateMinState, }); [[maybe_unused]] static AggRegistrar register_max({ .canonical_name = "MAX", - .distinct = false, - .star = false, .supports_type = &SupportsAnyType, .factory = &CreateMaxState, }); @@ -370,37 +238,26 @@ bool AggSupportsInputType(const AggFuncDefinition& definition, const ColumnType return definition.supports_type(type); } -std::string FormatAgg(const AggFuncDefinition& definition, const std::string_view column_name, const bool distinct, - const bool star) { - std::string output(definition.canonical_name); - output += '('; - if (star) { - output += '*'; - } else { - if (distinct) { - output += "DISTINCT "; - } - output += column_name; - } - output += ')'; - return output; -} - std::unique_ptr CreateAggState(const PlannedAgg& aggregate) { if (aggregate.function == nullptr || aggregate.function->factory == nullptr) { throw Error::Unsupported("executor", "aggregate is not registered"); } - if (aggregate.distinct && !aggregate.function->distinct) { + + if (aggregate.distinct && !HasAggCallFeature(aggregate.function->call_features, AggCallFeature::Distinct)) { throw Error::Unsupported("executor", "DISTINCT is not supported for aggregate '" + std::string(aggregate.function->canonical_name) + "'"); } - if (aggregate.star && !aggregate.function->star) { + + if (aggregate.argument_kind == AggArgumentKind::Star && + !HasAggCallFeature(aggregate.function->call_features, AggCallFeature::Star)) { throw Error::Unsupported( "executor", "aggregate '" + std::string(aggregate.function->canonical_name) + "' does not support '*'"); } + std::unique_ptr state = aggregate.function->factory(aggregate); if (aggregate.distinct) { state = std::make_unique(std::move(state)); } + return state; } diff --git a/src/executor/executor.cpp b/src/executor/executor.cpp index 5f8bc82..97a21e6 100644 --- a/src/executor/executor.cpp +++ b/src/executor/executor.cpp @@ -1,8 +1,12 @@ -#include "executor.h" +#include "executor/executor.h" +#include #include -#include "ascii.h" +#include "executor/operator.h" +#include "executor/query_parser.h" +#include "executor/query_planner.h" +#include "common/ascii.h" void Executor::RegisterTable(const std::string& name, std::filesystem::path path) { tables_[ToLowerAscii(name)] = std::move(path); @@ -10,10 +14,10 @@ void Executor::RegisterTable(const std::string& name, std::filesystem::path path ExecuteExpected Executor::Execute(const std::string_view query) const { try { - const ParsedQuery parsed = ParseQuery(query); - const PlannedQuery planned = PlanQuery(parsed, tables_); + const Query query_ast = ParseQuery(query); + const PlannedQuery planned = PlanQuery(query_ast, tables_); - std::unique_ptr root = BuildPlan(planned); + const std::unique_ptr root = BuildPlan(planned); auto batch = root->Next(); if (!batch.has_value()) { diff --git a/src/executor/operator_utils.cpp b/src/executor/operator_utils.cpp new file mode 100644 index 0000000..21d0744 --- /dev/null +++ b/src/executor/operator_utils.cpp @@ -0,0 +1,71 @@ +#include "executor/operator_utils.h" + +#include +#include + +#include "common/ascii.h" +#include "common/error.h" +#include "model/column_traits.h" + +static bool EvaluateOrdering(const std::strong_ordering ordering, const ComparisonKind kind) { + if (kind == ComparisonKind::Equal) { + return ordering == 0; + } + + if (kind == ComparisonKind::NotEqual) { + return ordering != 0; + } + + if (kind == ComparisonKind::Less) { + return ordering < 0; + } + + if (kind == ComparisonKind::LessOrEqual) { + return ordering <= 0; + } + + if (kind == ComparisonKind::Greater) { + return ordering > 0; + } + + if (kind == ComparisonKind::GreaterOrEqual) { + return ordering >= 0; + } + + throw Error::Unsupported("executor", "unsupported comparison kind"); +} + +template +static std::strong_ordering CompareValuesImpl(const T left, const T right) { + return left <=> right; +} + +std::strong_ordering CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { + return VisitColumnType(type, [&]() { + return CompareValuesImpl(ColumnValueTraits::Parse(std::string(lhs)), + ColumnValueTraits::Parse(std::string(rhs))); + }); +} + +bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, + const ComparisonKind kind) { + return EvaluateOrdering(CompareValues(type, lhs, rhs), kind); +} + +ColumnType AggregateOutputType(const PlannedAgg& aggregate) { + const std::string name = ToUpperAscii(aggregate.function->canonical_name); + + if (name == "COUNT") { + return ColumnType::Int64; + } + + if (name == "SUM" || name == "AVG") { + return ColumnType::Int128; + } + + if (name == "MIN" || name == "MAX") { + return aggregate.input_type; + } + + throw Error::Unsupported("executor", "unsupported aggregate result type"); +} diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index d12602b..b9cd3f9 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -1,73 +1,22 @@ -#include +#include +#include +#include #include #include -#include "columnar_batch_io.h" -#include "executor.h" -#include "fileio.h" -#include "parsing.h" - -static void SeekRead(const std::filesystem::path& path, std::ifstream& in, const uint64_t offset) { - in.seekg(static_cast(offset)); - if (!in) { - throw Error::PathIo("operators", path, "seek file"); - } -} - -template -static bool EvaluateComparison(const T left, const T right, const ComparisonKind kind) { - if (kind == ComparisonKind::Equal) { - return left == right; - } - if (kind == ComparisonKind::NotEqual) { - return left != right; - } - if (kind == ComparisonKind::Less) { - return left < right; - } - if (kind == ComparisonKind::LessOrEqual) { - return left <= right; - } - if (kind == ComparisonKind::Greater) { - return left > right; - } - if (kind == ComparisonKind::GreaterOrEqual) { - return left >= right; - } - throw Error::Unsupported("operators", "unsupported comparison kind"); -} - -static bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, - const ComparisonKind kind) { - switch (type) { - case ColumnType::Boolean: - return EvaluateComparison(ParseBoolean(std::string(lhs)), ParseBoolean(std::string(rhs)), kind); - case ColumnType::Int16: - return EvaluateComparison(ParseInt16(std::string(lhs)), ParseInt16(std::string(rhs)), kind); - case ColumnType::Int32: - return EvaluateComparison(ParseInt32(std::string(lhs)), ParseInt32(std::string(rhs)), kind); - case ColumnType::Int64: - return EvaluateComparison(ParseInt64(std::string(lhs)), ParseInt64(std::string(rhs)), kind); - case ColumnType::Int128: - return EvaluateComparison(ParseInt128(std::string(lhs)), ParseInt128(std::string(rhs)), kind); - case ColumnType::String: - return EvaluateComparison(std::string(lhs), std::string(rhs), kind); - case ColumnType::Date: - return EvaluateComparison(ParseDate(std::string(lhs)), ParseDate(std::string(rhs)), kind); - case ColumnType::Timestamp: - return EvaluateComparison(ParseTimestamp(std::string(lhs)), ParseTimestamp(std::string(rhs)), kind); - case ColumnType::Character: - return EvaluateComparison(ParseCharacter(std::string(lhs)), ParseCharacter(std::string(rhs)), kind); - } - throw Error::Unsupported("operators", "unsupported column type in comparison"); -} +#include "common/error.h" +#include "executor/aggregate_state.h" +#include "executor/operator.h" +#include "executor/operator_utils.h" +#include "io/columnar_batch.h" +#include "io/file.h" class ScanOperator final : public Operator { public: ScanOperator(std::filesystem::path path, std::vector projection_indexes) - : path_(std::move(path)), in_(OpenInputFile(path_)), metadata_(ColumnarBatchReader(path_).GetMetadata()) { + : path_(std::move(path)), input_(path_), metadata_(ColumnarBatchReader(path_).GetMetadata()) { if (metadata_.schema.columns.empty()) { - throw Error::InvalidData("operators", "columnar schema is empty", path_.string()); + throw Error::MalformedData("executor", "columnar schema is empty", path_.string()); } projection_indexes_ = std::move(projection_indexes); @@ -75,13 +24,13 @@ class ScanOperator final : public Operator { for (const size_t source_index : projection_indexes_) { if (source_index >= metadata_.schema.columns.size()) { - throw Error::OutOfRange("operators", "projection index out of range", path_.string()); + throw Error::OutOfRange("executor", "projection index out of range", path_.string()); } + projected_schema_.columns.push_back(metadata_.schema.columns[source_index]); } } - public: std::optional Next() override { if (next_group_ >= metadata_.row_groups.size()) { return std::nullopt; @@ -93,8 +42,8 @@ class ScanOperator final : public Operator { for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { const size_t source_index = projection_indexes_[projected_index]; const auto& chunk = row_group.columns[source_index]; - SeekRead(path_, in_, chunk.offset); - batch.ColumnAt(projected_index).ReadFrom(in_, row_group.row_count, chunk.size); + + batch.ReadColumnFrom(projected_index, input_.StreamAt(chunk.offset), row_group.row_count, chunk.size); } return batch; @@ -102,7 +51,7 @@ class ScanOperator final : public Operator { private: std::filesystem::path path_; - std::ifstream in_; + InputFile input_; ColumnarMetadata metadata_; std::vector projection_indexes_; @@ -115,20 +64,20 @@ class FilterOperator final : public Operator { FilterOperator(std::unique_ptr child, PlannedFilter filter) : child_(std::move(child)), filter_(std::move(filter)) {} - public: std::optional Next() override { while (auto batch = child_->Next()) { Batch filtered(batch->GetSchema()); for (size_t row = 0; row < batch->RowsCount(); ++row) { - const std::string value = batch->ColumnAt(filter_.column_index).ValueAsString(row); + const std::string left = MaterializeValue(filter_.left, *batch, row); + const std::string right = MaterializeValue(filter_.right, *batch, row); - if (!MatchesComparison(filter_.column_type, value, filter_.literal_text, filter_.comparison)) { + if (!MatchesComparison(filter_.comparison_type, left, right, filter_.comparison)) { continue; } for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { - filtered.ColumnAt(column_index).AppendFromString(batch->ColumnAt(column_index).ValueAsString(row)); + filtered.AppendValueFromString(column_index, batch->ColumnAt(column_index).ValueAsString(row)); } } @@ -141,43 +90,113 @@ class FilterOperator final : public Operator { } private: + static std::string MaterializeValue(const PlannedValue& value, const Batch& batch, const size_t row) { + if (value.kind == PlannedValueKind::Literal) { + return value.literal_text; + } + return batch.ColumnAt(value.column_index).ValueAsString(row); + } + std::unique_ptr child_; PlannedFilter filter_; }; +class AggInput { + public: + virtual ~AggInput() = default; + + virtual void Consume(const Batch& batch, size_t row, AggState& state) const = 0; +}; + +class ColumnAggInput final : public AggInput { + public: + explicit ColumnAggInput(const size_t column_index) : column_index_(column_index) {} + + void Consume(const Batch& batch, const size_t row, AggState& state) const override { + const std::string materialized = batch.ColumnAt(column_index_).ValueAsString(row); + state.ConsumeValue(materialized); + } + + private: + size_t column_index_ = 0; +}; + +class RowAggInput final : public AggInput { + public: + void Consume(const Batch&, const size_t, AggState& state) const override { state.ConsumeRow(); } +}; + +static std::unique_ptr CreateAggInput(const PlannedAgg& aggregate) { + if (aggregate.argument_kind == AggArgumentKind::Column) { + return std::make_unique(aggregate.column_index); + } + return std::make_unique(); +} + struct AggBinding { PlannedAgg aggregate; + std::unique_ptr input; std::unique_ptr state; }; +struct MaterializedGroupKey { + std::string id; + std::vector values; +}; + +class GroupKeyEncoder { + public: + explicit GroupKeyEncoder(std::vector group_keys) : group_keys_(std::move(group_keys)) {} + + MaterializedGroupKey Encode(const Batch& batch, const size_t row) const { + MaterializedGroupKey key; + key.values.reserve(group_keys_.size()); + + for (const auto& group_key : group_keys_) { + std::string value = batch.ColumnAt(group_key.column_index).ValueAsString(row); + AppendValue(key.id, value); + key.values.push_back(std::move(value)); + } + + return key; + } + + private: + static void AppendValue(std::string& id, const std::string_view value) { + id += std::to_string(value.size()); + id.push_back(':'); + + id += value; + } + + std::vector group_keys_; +}; + class AggOperator final : public Operator { public: AggOperator(std::unique_ptr child, const std::vector& aggregates) : child_(std::move(child)) { bindings_.reserve(aggregates.size()); + for (const auto& aggregate : aggregates) { bindings_.push_back(AggBinding{ .aggregate = aggregate, + .input = CreateAggInput(aggregate), .state = CreateAggState(aggregate), }); } } - public: std::optional Next() override { if (returned_) { return std::nullopt; } + returned_ = true; while (auto batch = child_->Next()) { for (size_t row = 0; row < batch->RowsCount(); ++row) { - for (auto& binding : bindings_) { - if (!binding.aggregate.star) { - std::string materialized = batch->ColumnAt(*binding.aggregate.column_index).ValueAsString(row); - binding.state->Consume(materialized); - } else { - binding.state->Consume(std::nullopt); - } + for (const auto& binding : bindings_) { + binding.input->Consume(*batch, row, *binding.state); } } } @@ -186,16 +205,14 @@ class AggOperator final : public Operator { schema.columns.reserve(bindings_.size()); for (const auto& binding : bindings_) { - schema.columns.push_back(ColumnSchema{ - .name = binding.aggregate.output_name, - .type = ColumnType::String, - }); + schema.columns.push_back( + ColumnSchema(binding.aggregate.output_name, AggregateOutputType(binding.aggregate))); } Batch result(schema); for (size_t i = 0; i < bindings_.size(); ++i) { - result.ColumnAt(i).AppendFromString(bindings_[i].state->Finalize()); + result.AppendValueFromString(i, bindings_[i].state->Finalize()); } return result; @@ -207,11 +224,349 @@ class AggOperator final : public Operator { bool returned_ = false; }; +class GroupAggOperator final : public Operator { + public: + GroupAggOperator(std::unique_ptr child, std::vector group_keys, + const std::vector& aggregates, std::vector select_items) + : child_(std::move(child)), group_key_encoder_(std::move(group_keys)), select_items_(std::move(select_items)) { + bindings_.reserve(aggregates.size()); + + for (const auto& aggregate : aggregates) { + bindings_.push_back(GroupAggBinding{ + .aggregate = aggregate, + .input = CreateAggInput(aggregate), + }); + } + } + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + while (auto batch = child_->Next()) { + for (size_t row = 0; row < batch->RowsCount(); ++row) { + MaterializedGroupKey key = group_key_encoder_.Encode(*batch, row); + + size_t group_index = 0; + const auto it = group_indexes_.find(key.id); + if (it == group_indexes_.end()) { + group_index = groups_.size(); + group_indexes_.emplace(std::move(key.id), group_index); + + groups_.push_back(GroupState{ + .key_values = std::move(key.values), + .states = CreateStates(), + }); + } else { + group_index = it->second; + } + + GroupState& group = groups_[group_index]; + for (size_t i = 0; i < bindings_.size(); ++i) { + bindings_[i].input->Consume(*batch, row, *group.states[i]); + } + } + } + + Schema schema; + schema.columns.reserve(select_items_.size()); + + for (const auto& item : select_items_) { + schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); + } + + std::vector> rows; + rows.reserve(groups_.size()); + + for (const auto& group : groups_) { + rows.push_back(MaterializeRow(group)); + } + + Batch result(schema, rows.size()); + + for (const auto& row : rows) { + for (size_t i = 0; i < row.size(); ++i) { + result.AppendValueFromString(i, row[i]); + } + } + + return result; + } + + private: + struct GroupState { + std::vector key_values; + std::vector> states; + }; + + struct GroupAggBinding { + PlannedAgg aggregate; + std::unique_ptr input; + }; + + std::vector> CreateStates() const { + std::vector> states; + states.reserve(bindings_.size()); + + for (const auto& binding : bindings_) { + states.push_back(CreateAggState(binding.aggregate)); + } + + return states; + } + + std::vector MaterializeRow(const GroupState& group) const { + std::vector row; + row.reserve(select_items_.size()); + + for (const auto& item : select_items_) { + if (item.kind == SelectItemKind::GroupKey) { + row.push_back(group.key_values[item.index]); + } else { + row.push_back(group.states[item.index]->Finalize()); + } + } + + return row; + } + + std::unique_ptr child_; + GroupKeyEncoder group_key_encoder_; + std::vector bindings_; + std::vector select_items_; + std::unordered_map group_indexes_; + std::vector groups_; + bool returned_ = false; +}; + +struct SortedRow { + std::vector values; + size_t ordinal = 0; +}; + +class RowOrdering { + public: + RowOrdering(std::vector order_by, std::vector select_items) + : order_by_(std::move(order_by)), select_items_(std::move(select_items)) {} + + bool operator()(const SortedRow& lhs, const SortedRow& rhs) const { + for (const auto& order : order_by_) { + const std::strong_ordering comparison = CompareValues( + order.value_type, lhs.values[order.result_column_index], rhs.values[order.result_column_index]); + if (comparison != 0) { + return order.descending ? comparison > 0 : comparison < 0; + } + } + + for (size_t i = 0; i < select_items_.size(); ++i) { + if (select_items_[i].kind != SelectItemKind::GroupKey) { + continue; + } + const std::strong_ordering comparison = + CompareValues(select_items_[i].output_type, lhs.values[i], rhs.values[i]); + if (comparison != 0) { + return comparison > 0; + } + } + + return lhs.ordinal < rhs.ordinal; + } + + private: + std::vector order_by_; + std::vector select_items_; +}; + +static std::vector MaterializeBatchRow(const Batch& batch, const size_t row) { + std::vector values; + values.reserve(batch.ColumnsCount()); + + for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { + values.push_back(batch.ColumnAt(column_index).ValueAsString(row)); + } + + return values; +} + +static Batch BuildBatchFromSortedRows(const Schema& schema, const std::vector& rows) { + Batch result(schema, rows.size()); + + for (const auto& row : rows) { + for (size_t i = 0; i < row.values.size(); ++i) { + result.AppendValueFromString(i, row.values[i]); + } + } + + return result; +} + +class OrderByOperator final : public Operator { + public: + OrderByOperator(std::unique_ptr child, std::vector order_by, + std::vector select_items) + : child_(std::move(child)), ordering_(std::move(order_by), std::move(select_items)) {} + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + std::optional schema; + std::vector rows; + size_t ordinal = 0; + + while (auto batch = child_->Next()) { + if (!schema.has_value()) { + schema = batch->GetSchema(); + } + + rows.reserve(rows.size() + batch->RowsCount()); + + for (size_t row = 0; row < batch->RowsCount(); ++row) { + rows.push_back(SortedRow{ + .values = MaterializeBatchRow(*batch, row), + .ordinal = ordinal++, + }); + } + } + + if (!schema.has_value()) { + return std::nullopt; + } + + SortRows(rows); + + return BuildBatchFromSortedRows(*schema, rows); + } + + private: + void SortRows(std::vector& rows) const { std::ranges::sort(rows, ordering_); } + + std::unique_ptr child_; + RowOrdering ordering_; + bool returned_ = false; +}; + +class TopKOperator final : public Operator { + public: + TopKOperator(std::unique_ptr child, std::vector order_by, + std::vector select_items, const size_t limit) + : child_(std::move(child)), ordering_(std::move(order_by), std::move(select_items)), limit_(limit) {} + + std::optional Next() override { + if (returned_ || limit_ == 0) { + return std::nullopt; + } + + returned_ = true; + + std::optional schema; + std::vector top_rows; + size_t ordinal = 0; + + while (auto batch = child_->Next()) { + if (!schema.has_value()) { + schema = batch->GetSchema(); + } + + for (size_t row = 0; row < batch->RowsCount(); ++row) { + SortedRow candidate{ + .values = MaterializeBatchRow(*batch, row), + .ordinal = ordinal++, + }; + + if (top_rows.size() < limit_) { + top_rows.push_back(std::move(candidate)); + std::ranges::push_heap(top_rows, ordering_); + } else if (ordering_(candidate, top_rows.front())) { + std::ranges::pop_heap(top_rows, ordering_); + top_rows.back() = std::move(candidate); + std::ranges::push_heap(top_rows, ordering_); + } + } + } + + if (!schema.has_value()) { + return std::nullopt; + } + + std::ranges::sort(top_rows, ordering_); + + return BuildBatchFromSortedRows(*schema, top_rows); + } + + private: + std::unique_ptr child_; + RowOrdering ordering_; + size_t limit_; + bool returned_ = false; +}; + +class LimitOperator final : public Operator { + public: + LimitOperator(std::unique_ptr child, const size_t limit) : child_(std::move(child)), remaining_(limit) {} + + std::optional Next() override { + if (remaining_ == 0) { + return std::nullopt; + } + + while (auto batch = child_->Next()) { + if (batch->RowsCount() <= remaining_) { + remaining_ -= batch->RowsCount(); + + return batch; + } + + Batch limited(batch->GetSchema(), remaining_); + + for (size_t row = 0; row < remaining_; ++row) { + for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { + limited.AppendValueFromString(column_index, batch->ColumnAt(column_index).ValueAsString(row)); + } + } + + remaining_ = 0; + + return limited; + } + + remaining_ = 0; + + return std::nullopt; + } + + private: + std::unique_ptr child_; + size_t remaining_; +}; + std::unique_ptr BuildPlan(const PlannedQuery& planned) { std::unique_ptr root = std::make_unique(planned.table_path, planned.projection_indexes); + if (planned.filter.has_value()) { root = std::make_unique(std::move(root), *planned.filter); } - root = std::make_unique(std::move(root), planned.aggregates); + + if (!planned.group_keys.empty()) { + root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, + planned.select_items); + } else { + root = std::make_unique(std::move(root), planned.aggregates); + } + + if (!planned.order_by.empty() && planned.limit.has_value()) { + root = std::make_unique(std::move(root), planned.order_by, planned.select_items, *planned.limit); + } else if (!planned.order_by.empty()) { + root = std::make_unique(std::move(root), planned.order_by, planned.select_items); + } else if (planned.limit.has_value()) { + root = std::make_unique(std::move(root), *planned.limit); + } + return root; } diff --git a/src/executor/planner_utils.cpp b/src/executor/planner_utils.cpp new file mode 100644 index 0000000..060c851 --- /dev/null +++ b/src/executor/planner_utils.cpp @@ -0,0 +1,104 @@ +#include "executor/planner_utils.h" + +#include "common/ascii.h" +#include "common/error.h" + +size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { + const std::string needle = ToLowerAscii(column_name); + + for (size_t i = 0; i < schema.columns.size(); ++i) { + if (ToLowerAscii(schema.columns[i].name) == needle) { + return i; + } + } + + throw Error::InvalidArgument("executor", "unknown column '" + std::string(column_name) + "'"); +} + +static bool SameQualifier(const std::string_view lhs, const std::string_view rhs) { + return ToLowerAscii(lhs) == ToLowerAscii(rhs); +} + +void ValidateColumnQualifier(const Query& query, const ColumnRef& column) { + if (column.qualifier.empty()) { + return; + } + + if (SameQualifier(column.qualifier, query.table_name)) { + return; + } + + if (!query.table_alias.empty() && SameQualifier(column.qualifier, query.table_alias)) { + return; + } + + throw Error::InvalidArgument("executor", "unknown table qualifier '" + column.qualifier + "'"); +} + +static std::string UnescapeSqlString(const std::string_view text) { + if (text.size() < 2 || text.front() != '\'' || text.back() != '\'') { + throw Error::InvalidArgument("executor", "invalid string literal"); + } + + std::string result; + result.reserve(text.size() - 2); + + for (size_t i = 1; i + 1 < text.size(); ++i) { + if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { + result.push_back('\''); + ++i; + continue; + } + + result.push_back(text[i]); + } + + return result; +} + +std::string NormalizeLiteral(const QueryLiteral& literal, const ColumnType type) { + switch (literal.kind) { + case LiteralKind::String: + return UnescapeSqlString(literal.text); + case LiteralKind::Numeric: + case LiteralKind::Identifier: + if (type == ColumnType::String && literal.kind == LiteralKind::Identifier) { + return literal.text; + } + return literal.text; + default: + throw Error::InvalidArgument("executor", "unsupported literal type"); + } +} + +bool SameColumnName(const std::string_view lhs, const std::string_view rhs) { + return ToLowerAscii(lhs) == ToLowerAscii(rhs); +} + +bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs) { + if (!SameColumnName(lhs.name, rhs.name)) { + return false; + } + + if (lhs.qualifier.empty() || rhs.qualifier.empty()) { + return true; + } + + return SameQualifier(lhs.qualifier, rhs.qualifier); +} + +bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { + if (lhs.kind != rhs.kind) { + return false; + } + + if (lhs.kind == SelectItemKind::GroupKey) { + return SameColumnRef(lhs.column, rhs.column); + } + + return ToLowerAscii(lhs.aggregate.output_name) == ToLowerAscii(rhs.aggregate.output_name); +} + +bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { + return SameColumnName(lhs.output_name, rhs.output_name); +} diff --git a/src/executor/query_parser.cpp b/src/executor/query_parser.cpp index 396b093..11233b6 100644 --- a/src/executor/query_parser.cpp +++ b/src/executor/query_parser.cpp @@ -1,25 +1,32 @@ +#include "executor/query_parser.h" + #include +#include #include +#include -#include "executor.h" +#include "common/ascii.h" +#include "common/error.h" +#include "sql_parser/tokenizer.h" class TokenCursor { public: explicit TokenCursor(std::vector tokens) : tokens_(std::move(tokens)) {} - public: - const Token& Peek() const { - if (pos_ >= tokens_.size()) { - throw Error::InvalidArgument("query_parser", "unexpected end of query"); + const Token& Peek(const size_t lookahead = 0) const { + if (pos_ + lookahead >= tokens_.size()) { + throw Error::InvalidArgument("executor", "unexpected end of query"); } - return *tokens_[pos_]; + return *tokens_[pos_ + lookahead]; } - bool Match(const Tokens type) const { return pos_ < tokens_.size() && tokens_[pos_]->GetType() == type; } + bool Match(const Tokens type, const size_t lookahead = 0) const { + return pos_ + lookahead < tokens_.size() && tokens_[pos_ + lookahead]->GetType() == type; + } const Token& Consume(const Tokens type, const std::string_view message) { if (!Match(type)) { - throw Error::InvalidArgument("query_parser", std::string(message)); + throw Error::InvalidArgument("executor", std::string(message)); } return *tokens_[pos_++]; } @@ -34,7 +41,7 @@ class TokenCursor { const Token& ConsumeFunctionToken(const std::string_view message) { if (pos_ >= tokens_.size()) { - throw Error::InvalidArgument("query_parser", std::string(message)); + throw Error::InvalidArgument("executor", std::string(message)); } const Tokens type = tokens_[pos_]->GetType(); @@ -52,8 +59,8 @@ class TokenCursor { case Tokens::LessOrEqual: case Tokens::Greater: case Tokens::GreaterOrEqual: - case Tokens::EOI: - throw Error::InvalidArgument("query_parser", std::string(message)); + case Tokens::EndOfInput: + throw Error::InvalidArgument("executor", std::string(message)); default: break; } @@ -81,41 +88,210 @@ static ComparisonKind ComparisonKindFromToken(const Tokens type) { case Tokens::GreaterOrEqual: return ComparisonKind::GreaterOrEqual; default: - throw Error::InvalidArgument("query_parser", "expected comparison operator in WHERE"); + throw Error::InvalidArgument("executor", "expected comparison operator in WHERE"); + } +} + +static std::string FormatColumnRef(const ColumnRef& column) { + if (column.qualifier.empty()) { + return column.name; + } + return column.qualifier + "." + column.name; +} + +static ColumnRef ParseColumnRef(TokenCursor& cursor, const std::string_view message) { + ColumnRef column{ + .qualifier = {}, + .name = std::string(cursor.Consume(Tokens::NameToken, message).GetText()), + }; + + if (cursor.TryConsume(Tokens::Dot)) { + column.qualifier = std::move(column.name); + column.name = std::string(cursor.Consume(Tokens::NameToken, "expected column name after '.'").GetText()); + } + + return column; +} + +static QueryValue QueryColumnValue(ColumnRef column) { + return QueryValue{ + .kind = QueryValueKind::Column, + .column = std::move(column), + .literal = {}, + }; +} + +static QueryValue QueryLiteralValue(QueryLiteral literal) { + return QueryValue{ + .kind = QueryValueKind::Literal, + .column = {}, + .literal = std::move(literal), + }; +} + +static QueryValue ParseFilterValue(TokenCursor& cursor, const std::string_view message, + const bool bare_identifier_is_column) { + if (cursor.TryConsume(Tokens::Minus)) { + const Token& literal = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after '-'"); + return QueryLiteralValue(QueryLiteral{ + .text = "-" + std::string(literal.GetText()), + .kind = LiteralKind::Numeric, + }); + } + + const Token& token = cursor.Peek(); + switch (token.GetType()) { + case Tokens::NumericLiteral: + cursor.Consume(Tokens::NumericLiteral, message); + return QueryLiteralValue(QueryLiteral{ + .text = std::string(token.GetText()), + .kind = LiteralKind::Numeric, + }); + case Tokens::StringLiteral: + cursor.Consume(Tokens::StringLiteral, message); + return QueryLiteralValue(QueryLiteral{ + .text = std::string(token.GetText()), + .kind = LiteralKind::String, + }); + case Tokens::NameToken: + if (bare_identifier_is_column || cursor.Match(Tokens::Dot, 1)) { + return QueryColumnValue(ParseColumnRef(cursor, message)); + } + cursor.Consume(Tokens::NameToken, message); + return QueryLiteralValue(QueryLiteral{ + .text = std::string(token.GetText()), + .kind = LiteralKind::Identifier, + }); + default: + throw Error::InvalidArgument("executor", std::string(message)); + } +} + +static std::string FormatAggregateCall(const std::string_view function_name, const std::string_view column_name, + const bool distinct, const AggArgumentKind argument_kind) { + std::string output(function_name); + output += '('; + + if (argument_kind == AggArgumentKind::Star) { + output += '*'; + } else { + if (distinct) { + output += "DISTINCT "; + } + output += column_name; } + + output += ')'; + + return output; +} + +static std::optional ParseOptionalLimit(TokenCursor& cursor) { + if (!cursor.TryConsume(Tokens::Limit)) { + return std::nullopt; + } + + const Token& limit_token = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after LIMIT"); + size_t limit = 0; + + const std::string_view text = limit_token.GetText(); + const char* begin = text.data(); + const char* end = begin + text.size(); + + if (const auto [ptr, ec] = std::from_chars(begin, end, limit); ec != std::errc() || ptr != end) { + throw Error::InvalidArgument("executor", "invalid LIMIT value"); + } + + return limit; } static AggSpec ParseAggregate(TokenCursor& cursor) { const Token& function_token = cursor.ConsumeFunctionToken("expected aggregate function"); AggSpec spec; - spec.function = &ResolveAggFunc(function_token.GetText()); + spec.function_name = ToUpperAscii(function_token.GetText()); cursor.Consume(Tokens::OpenBracket, "expected '(' after aggregate function"); if (cursor.TryConsume(Tokens::Asterisk)) { - if (!spec.function->star) { - throw Error::InvalidArgument("query_parser", - std::string(spec.function->canonical_name) + " does not support '*'"); - } - spec.star = true; + spec.argument_kind = AggArgumentKind::Star; } else { spec.distinct = cursor.TryConsume(Tokens::Distinct); - if (spec.distinct && !spec.function->distinct) { - throw Error::InvalidArgument("query_parser", - std::string(spec.function->canonical_name) + " does not support DISTINCT"); - } - const auto& token = cursor.Consume(Tokens::NameToken, "expected column name in aggregate"); - spec.column_name = std::string(token.GetText()); + spec.column = ParseColumnRef(cursor, "expected column name in aggregate"); } cursor.Consume(Tokens::CloseBracket, "expected ')' after aggregate arguments"); - spec.output_name = FormatAgg(*spec.function, spec.column_name, spec.distinct, spec.star); + spec.output_name = + FormatAggregateCall(spec.function_name, FormatColumnRef(spec.column), spec.distinct, spec.argument_kind); return spec; } -ParsedQuery ParseQuery(const std::string_view query) { +static SelectItemSpec ParseSelectExpression(TokenCursor& cursor) { + if (cursor.Match(Tokens::NameToken) && !cursor.Match(Tokens::OpenBracket, 1)) { + const ColumnRef column = ParseColumnRef(cursor, "expected column name in SELECT"); + return SelectItemSpec{ + .kind = SelectItemKind::GroupKey, + .column = column, + .aggregate = {}, + .output_name = column.name, + }; + } + + return SelectItemSpec{ + .kind = SelectItemKind::Aggregate, + .column = {}, + .aggregate = ParseAggregate(cursor), + .output_name = {}, + }; +} + +static std::string ParseSelectItemAlias(TokenCursor& cursor) { + if (cursor.TryConsume(Tokens::As)) { + return std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); + } + + if (cursor.Match(Tokens::NameToken)) { + return std::string(cursor.Consume(Tokens::NameToken, "expected alias").GetText()); + } + + return {}; +} + +static SelectItemSpec ParseSelectItem(TokenCursor& cursor) { + SelectItemSpec item = ParseSelectExpression(cursor); + + if (const std::string alias = ParseSelectItemAlias(cursor); !alias.empty()) { + item.output_name = alias; + } else if (item.kind == SelectItemKind::Aggregate) { + item.output_name = item.aggregate.output_name; + } + + return item; +} + +static OrderBySpec ParseOrderByItem(TokenCursor& cursor) { + SelectItemSpec item = ParseSelectExpression(cursor); + item.output_name = item.kind == SelectItemKind::Aggregate ? item.aggregate.output_name : item.column.name; + + bool descending = false; + if (cursor.Match(Tokens::NameToken)) { + const std::string direction = ToUpperAscii(cursor.Peek().GetText()); + if (direction == "DESC") { + cursor.Consume(Tokens::NameToken, "expected DESC"); + descending = true; + } else if (direction == "ASC") { + cursor.Consume(Tokens::NameToken, "expected ASC"); + } + } + + return OrderBySpec{ + .item = std::move(item), + .descending = descending, + }; +} + +Query ParseQuery(const std::string_view query) { auto tokenized = TokenizeSql(query); if (!tokenized.has_value()) { throw tokenized.error(); @@ -123,53 +299,57 @@ ParsedQuery ParseQuery(const std::string_view query) { TokenCursor cursor(std::move(tokenized.value())); - ParsedQuery parsed; + Query query_ast; cursor.Consume(Tokens::Select, "expected SELECT"); - parsed.aggregates.push_back(ParseAggregate(cursor)); + query_ast.select_items.push_back(ParseSelectItem(cursor)); while (cursor.TryConsume(Tokens::Comma)) { - parsed.aggregates.push_back(ParseAggregate(cursor)); + query_ast.select_items.push_back(ParseSelectItem(cursor)); } cursor.Consume(Tokens::From, "expected FROM"); - parsed.table_name = std::string(cursor.Consume(Tokens::NameToken, "expected table name").GetText()); + query_ast.table_name = std::string(cursor.Consume(Tokens::NameToken, "expected table name").GetText()); + + if (cursor.TryConsume(Tokens::As)) { + query_ast.table_alias = std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); + } else if (cursor.Match(Tokens::NameToken)) { + query_ast.table_alias = std::string(cursor.Consume(Tokens::NameToken, "expected table alias").GetText()); + } if (cursor.TryConsume(Tokens::Where)) { FilterSpec filter; - filter.column_name = - std::string(cursor.Consume(Tokens::NameToken, "expected column name after WHERE").GetText()); + filter.left = ParseFilterValue(cursor, "expected left operand after WHERE", true); const Tokens comparison_token = cursor.Peek().GetType(); filter.comparison = ComparisonKindFromToken(comparison_token); cursor.Consume(comparison_token, "expected comparison operator in WHERE"); + filter.right = ParseFilterValue(cursor, "expected right operand after comparison operator", false); - if (cursor.TryConsume(Tokens::Minus)) { - const Token& literal = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after '-'"); - filter.literal = - ParsedLiteral{.text = "-" + std::string(literal.GetText()), .token_type = Tokens::NumericLiteral}; - } else { - const Token& literal = cursor.Peek(); - switch (literal.GetType()) { - case Tokens::NumericLiteral: - case Tokens::StringLiteral: - case Tokens::NameToken: - filter.literal = ParsedLiteral{ - .text = std::string(literal.GetText()), - .token_type = literal.GetType(), - }; - cursor.Consume(literal.GetType(), "expected literal after comparison operator"); - break; - default: - throw Error::InvalidArgument("query_parser", "unsupported literal in WHERE"); - } + query_ast.filter = std::move(filter); + } + + if (cursor.TryConsume(Tokens::Group)) { + cursor.Consume(Tokens::By, "expected BY after GROUP"); + query_ast.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); + + while (cursor.TryConsume(Tokens::Comma)) { + query_ast.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); } + } - parsed.filter = std::move(filter); + if (cursor.TryConsume(Tokens::Order)) { + cursor.Consume(Tokens::By, "expected BY after ORDER"); + query_ast.order_by.push_back(ParseOrderByItem(cursor)); + + while (cursor.TryConsume(Tokens::Comma)) { + query_ast.order_by.push_back(ParseOrderByItem(cursor)); + } } + query_ast.limit = ParseOptionalLimit(cursor); cursor.TryConsume(Tokens::Semicolon); - cursor.Consume(Tokens::EOI, "unexpected trailing tokens"); + cursor.Consume(Tokens::EndOfInput, "unexpected trailing tokens"); - return parsed; + return query_ast; } diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index 80a245e..e5f7724 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -1,115 +1,244 @@ +#include "executor/query_planner.h" + #include -#include "ascii.h" -#include "columnar_batch_io.h" -#include "executor.h" +#include "common/ascii.h" +#include "common/error.h" +#include "executor/operator_utils.h" +#include "executor/planner_utils.h" +#include "io/columnar_batch.h" -static size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { - const std::string needle = ToLowerAscii(column_name); +static std::optional TryFindColumnIndex(const Schema& schema, const std::string_view column_name) { for (size_t i = 0; i < schema.columns.size(); ++i) { - if (ToLowerAscii(schema.columns[i].name) == needle) { + if (SameColumnName(schema.columns[i].name, column_name)) { return i; } } - throw Error::InvalidArgument("query_planner", "unknown column '" + std::string(column_name) + "'"); + return std::nullopt; } -static std::string UnescapeSqlString(const std::string_view text) { - if (text.size() < 2 || text.front() != '\'' || text.back() != '\'') { - throw Error::InvalidArgument("query_planner", "invalid string literal"); - } - - std::string result; - result.reserve(text.size() - 2); - - for (size_t i = 1; i + 1 < text.size(); ++i) { - if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { - result.push_back('\''); - ++i; - continue; - } - result.push_back(text[i]); - } - - return result; -} - -static std::string NormalizeLiteral(const ParsedLiteral& literal, const ColumnType type) { - switch (literal.token_type) { - case Tokens::StringLiteral: - return UnescapeSqlString(literal.text); - case Tokens::NumericLiteral: - case Tokens::NameToken: - if (type == ColumnType::String && literal.token_type == Tokens::NameToken) { - return literal.text; - } - return literal.text; - default: - throw Error::InvalidArgument("query_planner", "unsupported literal type"); - } -} - -PlannedQuery PlanQuery(const ParsedQuery& parsed, - const std::unordered_map& tables) { +PlannedQuery PlanQuery(const Query& query_ast, const std::unordered_map& tables) { PlannedQuery planned; - const auto table_it = tables.find(ToLowerAscii(parsed.table_name)); + const auto table_it = tables.find(ToLowerAscii(query_ast.table_name)); if (table_it == tables.end()) { - throw Error::NotFound("query_planner", "unknown table", parsed.table_name); + throw Error::NotFound("executor", "unknown table", query_ast.table_name); } planned.table_path = table_it->second; - ColumnarBatchReader reader(planned.table_path); + const ColumnarBatchReader reader(planned.table_path); const Schema& schema = reader.GetSchema(); - const auto register_projection = [&](const size_t source_index) -> size_t { + const auto ensure_projected_column = [&](const size_t source_index) -> size_t { const auto it = std::ranges::find(planned.projection_indexes, source_index); if (it != planned.projection_indexes.end()) { return static_cast(std::distance(planned.projection_indexes.begin(), it)); } + planned.projection_indexes.push_back(source_index); + return planned.projection_indexes.size() - 1; }; - if (parsed.filter.has_value()) { - const size_t source_index = FindColumnIndex(schema, parsed.filter->column_name); - const ColumnType column_type = schema.columns[source_index].type; + struct PlannedFilterValue { + PlannedValue value; + std::optional literal; + }; + + const auto plan_filter_value = [&](const QueryValue& value) -> PlannedFilterValue { + if (value.kind == QueryValueKind::Column) { + ValidateColumnQualifier(query_ast, value.column); + + const size_t source_index = FindColumnIndex(schema, value.column.name); + + return PlannedFilterValue{ + .value = + PlannedValue{ + .kind = PlannedValueKind::Column, + .column_type = schema.columns[source_index].type, + .column_index = ensure_projected_column(source_index), + .literal_text = {}, + }, + .literal = std::nullopt, + }; + } + + if (value.literal.kind == LiteralKind::Identifier) { + if (const std::optional source_index = TryFindColumnIndex(schema, value.literal.text); + source_index.has_value()) { + return PlannedFilterValue{ + .value = + PlannedValue{ + .kind = PlannedValueKind::Column, + .column_type = schema.columns[*source_index].type, + .column_index = ensure_projected_column(*source_index), + .literal_text = {}, + }, + .literal = std::nullopt, + }; + } + } + + return PlannedFilterValue{ + .value = + PlannedValue{ + .kind = PlannedValueKind::Literal, + .column_type = ColumnType::String, + .column_index = 0, + .literal_text = {}, + }, + .literal = value.literal, + }; + }; + + if (query_ast.filter.has_value()) { + PlannedFilterValue left = plan_filter_value(query_ast.filter->left); + PlannedFilterValue right = plan_filter_value(query_ast.filter->right); + + const ColumnType comparison_type = left.value.kind == PlannedValueKind::Column ? left.value.column_type + : right.value.kind == PlannedValueKind::Column ? right.value.column_type + : ColumnType::String; + if (left.literal.has_value()) { + left.value.column_type = comparison_type; + left.value.literal_text = NormalizeLiteral(*left.literal, comparison_type); + } + + if (right.literal.has_value()) { + right.value.column_type = comparison_type; + right.value.literal_text = NormalizeLiteral(*right.literal, comparison_type); + } planned.filter = PlannedFilter{ - .column_index = register_projection(source_index), - .column_type = column_type, - .comparison = parsed.filter->comparison, - .literal_text = NormalizeLiteral(parsed.filter->literal, column_type), + .left = std::move(left.value), + .comparison = query_ast.filter->comparison, + .right = std::move(right.value), + .comparison_type = comparison_type, }; } - planned.aggregates.reserve(parsed.aggregates.size()); + planned.group_keys.reserve(query_ast.group_by.size()); + + for (const ColumnRef& group_column : query_ast.group_by) { + ValidateColumnQualifier(query_ast, group_column); + + const size_t source_index = FindColumnIndex(schema, group_column.name); + + planned.group_keys.push_back(PlannedGroupKey{ + .column_index = ensure_projected_column(source_index), + .column_type = schema.columns[source_index].type, + .output_name = schema.columns[source_index].name, + }); + } + + bool has_aggregate = false; + for (const auto& item : query_ast.select_items) { + if (item.kind == SelectItemKind::Aggregate) { + has_aggregate = true; + break; + } + } + + planned.select_items.reserve(query_ast.select_items.size()); + planned.aggregates.reserve(query_ast.select_items.size()); + + for (const auto& item : query_ast.select_items) { + if (item.kind == SelectItemKind::GroupKey) { + if (query_ast.group_by.empty()) { + throw Error::Unsupported("executor", "plain column SELECT items require GROUP BY"); + } + + ValidateColumnQualifier(query_ast, item.column); + + const auto it = std::ranges::find_if(planned.group_keys, [&](const PlannedGroupKey& group_key) { + return SameColumnName(group_key.output_name, item.column.name); + }); + + if (it == planned.group_keys.end()) { + throw Error::InvalidArgument("executor", "column '" + item.column.name + "' must appear in GROUP BY"); + } + + planned.select_items.push_back(PlannedSelectItem{ + .kind = SelectItemKind::GroupKey, + .index = static_cast(std::distance(planned.group_keys.begin(), it)), + .output_type = it->column_type, + .output_name = item.output_name, + }); + + continue; + } + + const AggSpec& aggregate = item.aggregate; + const AggFuncDefinition& function = ResolveAggFunc(aggregate.function_name); + + if (aggregate.distinct && !HasAggCallFeature(function.call_features, AggCallFeature::Distinct)) { + throw Error::InvalidArgument("executor", + std::string(function.canonical_name) + " does not support DISTINCT"); + } + + if (aggregate.argument_kind == AggArgumentKind::Star && + !HasAggCallFeature(function.call_features, AggCallFeature::Star)) { + throw Error::InvalidArgument("executor", std::string(function.canonical_name) + " does not support '*'"); + } - for (const auto& aggregate : parsed.aggregates) { PlannedAgg planned_aggregate{ - .function = aggregate.function, + .function = &function, .distinct = aggregate.distinct, - .star = aggregate.star, - .column_index = std::nullopt, + .argument_kind = aggregate.argument_kind, + .column_index = 0, .input_type = ColumnType::String, - .output_name = aggregate.output_name, + .output_name = item.output_name, }; - if (!aggregate.star) { - const size_t source_index = FindColumnIndex(schema, aggregate.column_name); - planned_aggregate.column_index = register_projection(source_index); + if (aggregate.argument_kind == AggArgumentKind::Column) { + ValidateColumnQualifier(query_ast, aggregate.column); + + const size_t source_index = FindColumnIndex(schema, aggregate.column.name); + planned_aggregate.column_index = ensure_projected_column(source_index); planned_aggregate.input_type = schema.columns[source_index].type; - if (!AggSupportsInputType(*aggregate.function, planned_aggregate.input_type)) { - throw Error::Unsupported("query_planner", - "aggregate " + std::string(aggregate.function->canonical_name) + - " does not support column '" + aggregate.column_name + "'"); + + if (!AggSupportsInputType(function, planned_aggregate.input_type)) { + throw Error::Unsupported("executor", "aggregate " + std::string(function.canonical_name) + + " does not support column '" + aggregate.column.name + "'"); } } + const size_t aggregate_index = planned.aggregates.size(); planned.aggregates.push_back(std::move(planned_aggregate)); + planned.select_items.push_back(PlannedSelectItem{ + .kind = SelectItemKind::Aggregate, + .index = aggregate_index, + .output_type = AggregateOutputType(planned.aggregates.back()), + .output_name = item.output_name, + }); + } + + if (query_ast.group_by.empty() && !has_aggregate) { + throw Error::Unsupported("executor", "plain SELECT without aggregates is not supported"); } + planned.order_by.reserve(query_ast.order_by.size()); + + for (const auto& order_item : query_ast.order_by) { + const auto it = std::ranges::find_if(query_ast.select_items, [&](const SelectItemSpec& select_item) { + return SameSelectItem(select_item, order_item.item) || SameOutputName(select_item, order_item.item); + }); + + if (it == query_ast.select_items.end()) { + throw Error::InvalidArgument("executor", "ORDER BY expression must appear in SELECT"); + } + + const size_t result_column_index = static_cast(std::distance(query_ast.select_items.begin(), it)); + + planned.order_by.push_back(PlannedOrderBy{ + .result_column_index = result_column_index, + .descending = order_item.descending, + .value_type = planned.select_items[result_column_index].output_type, + }); + } + + planned.limit = query_ast.limit; + if (planned.projection_indexes.empty() && !schema.columns.empty()) { planned.projection_indexes.push_back(0); } diff --git a/src/io/batch.cpp b/src/io/batch.cpp new file mode 100644 index 0000000..cb2cf7a --- /dev/null +++ b/src/io/batch.cpp @@ -0,0 +1,37 @@ +#include "io/batch.h" + +#include +#include + +#include "common/error.h" + +static void ValidateOptionalLimit(const std::optional& value, const std::string_view name) { + if (value && *value == 0) { + throw Error::InvalidArgument("io", std::string(name) + " must be > 0"); + } +} + +bool BatchSizing::WouldExceed(const size_t next_rows, const size_t column_count, const uint64_t next_bytes) const { + ValidateOptionalLimit(max_rows, "max rows"); + ValidateOptionalLimit(max_values, "max values"); + + if (max_bytes && *max_bytes == 0) { + throw Error::InvalidArgument("io", "max bytes must be > 0"); + } + + if (max_rows && next_rows > *max_rows) { + return true; + } + + if (max_values && column_count > 0) { + if (next_rows > *max_values / column_count) { + return true; + } + } + + if (max_bytes && next_bytes > *max_bytes) { + return true; + } + + return false; +} diff --git a/src/io/batch_io.cpp b/src/io/batch_io.cpp deleted file mode 100644 index f66042a..0000000 --- a/src/io/batch_io.cpp +++ /dev/null @@ -1,328 +0,0 @@ -#include -#include -#include - -#include "columnar_batch_io.h" -#include "csv.h" -#include "csv_batch_io.h" -#include "error.h" -#include "metadata.h" -#include "stream.h" - -static constexpr std::string_view kColumnarMagic = "CLMN"; - -static void SeekRead(const std::filesystem::path& path, std::ifstream& in, const uint64_t pos) { - in.seekg(pos); - if (!in) { - throw Error::PathIo("batch_io", path, "seek file"); - } -} - -static uint64_t TellWrite(const std::filesystem::path& path, std::ofstream& out) { - const auto pos = out.tellp(); - if (pos == -1) { - throw Error::PathIo("batch_io", path, "tell file"); - } - return pos; -} - -static void FlushWrite(const std::filesystem::path& path, std::ofstream& out) { - out.flush(); - if (!out) { - throw Error::PathIo("batch_io", path, "write file"); - } -} - -static uint64_t AddChecked(const uint64_t current, const uint64_t add) { - if (add > std::numeric_limits::max() - current) { - throw Error::Overflow("batch_io", "batch byte size overflow"); - } - return current + add; -} - -static uint64_t EstimateRowBytes(const std::vector>& columns, - const std::vector& row) { - uint64_t bytes = 0; - for (size_t i = 0; i < columns.size(); ++i) { - bytes = AddChecked(bytes, columns[i]->EstimateSizeFromString(row[i])); - } - return bytes; -} - -static void ValidateSizing(const BatchSizing& sizing) { - if (sizing.max_rows && *sizing.max_rows == 0) { - throw Error::InvalidArgument("batch_io", "max rows must be > 0"); - } - if (sizing.max_values && *sizing.max_values == 0) { - throw Error::InvalidArgument("batch_io", "max values must be > 0"); - } - if (sizing.max_bytes && *sizing.max_bytes == 0) { - throw Error::InvalidArgument("batch_io", "max bytes must be > 0"); - } -} - -static void ReserveForSizing(const Batch& batch, const BatchSizing& sizing, const size_t column_count) { - if (sizing.max_rows) { - batch.Reserve(*sizing.max_rows); - return; - } - if (sizing.max_values && column_count > 0) { - batch.Reserve(std::max(1, *sizing.max_values / column_count)); - } -} - -bool BatchSizing::WouldExceed(const size_t next_rows, const size_t column_count, const uint64_t next_bytes) const { - if (max_rows && next_rows > *max_rows) { - return true; - } - if (max_values && column_count > 0) { - if (next_rows > *max_values / column_count) { - return true; - } - } - if (max_bytes && next_bytes > *max_bytes) { - return true; - } - return false; -} - -CsvBatchReader::CsvBatchReader(const std::filesystem::path& path, Schema schema, BatchSizing sizing) - : csv_reader_(path), schema_(std::move(schema)), sizing_(std::move(sizing)) { - if (schema_.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns"); - } - ValidateSizing(sizing_); -} - -std::optional CsvBatchReader::ReadNext() { - if (reached_eof_ && !pending_row_) { - return std::nullopt; - } - - Batch batch(schema_); - - const size_t column_count = schema_.columns.size(); - ReserveForSizing(batch, sizing_, column_count); - const auto& columns = batch.GetColumns(); - - std::vector row; - row.reserve(column_count); - - size_t rows = 0; - uint64_t bytes = 0; - - while (true) { - if (pending_row_) { - row = std::move(*pending_row_); - pending_row_.reset(); - } else { - if (!csv_reader_.ReadRow(row)) { - reached_eof_ = true; - break; - } - } - - if (row.size() != column_count) { - throw Error::Mismatch("batch_io", "data csv column count mismatch"); - } - - const size_t next_rows = rows + 1; - uint64_t next_bytes = bytes; - - if (sizing_.max_bytes) { - next_bytes = AddChecked(bytes, EstimateRowBytes(columns, row)); - } - - if (rows > 0 && sizing_.WouldExceed(next_rows, column_count, next_bytes)) { - pending_row_ = std::move(row); - break; - } - - for (size_t col = 0; col < column_count; ++col) { - columns[col]->AppendFromString(row[col]); - } - - rows = next_rows; - bytes = next_bytes; - } - - if (rows == 0 && reached_eof_ && !pending_row_) { - return std::nullopt; - } - - return batch; -} - -CsvBatchWriter::CsvBatchWriter(const std::filesystem::path& path, Schema schema) - : csv_writer_(path), schema_(std::move(schema)) { - if (schema_.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns"); - } -} - -void CsvBatchWriter::Write(const Batch& batch) { - batch.Validate(); - if (batch.GetSchema() != schema_) { - throw Error::Mismatch("batch_io", "batch schema mismatch"); - } - - const size_t column_count = schema_.columns.size(); - const size_t row_count = batch.RowsCount(); - - std::vector row(column_count); - - for (size_t row_index = 0; row_index < row_count; ++row_index) { - for (size_t col = 0; col < column_count; ++col) { - row[col] = batch.ColumnAt(col).ValueAsString(row_index); - } - csv_writer_.WriteRow(row); - } -} - -void CsvBatchWriter::Flush() { csv_writer_.Flush(); } - -void AppendBatchRows(const Batch& batch, std::vector>& rows) { - const size_t row_count = batch.RowsCount(); - const size_t column_count = batch.ColumnsCount(); - - for (size_t row = 0; row < row_count; ++row) { - std::vector values; - values.reserve(column_count); - - for (size_t col = 0; col < column_count; ++col) { - values.push_back(batch.ColumnAt(col).ValueAsString(row)); - } - - rows.push_back(std::move(values)); - } -} - -void WriteBatchCsv(const std::filesystem::path& path, const Batch& batch) { - CsvBatchWriter writer(path, batch.GetSchema()); - writer.Write(batch); - writer.Flush(); -} - -ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::path& path) { - const auto file_metadata = TryGetFileMetadata(path); - - if (!file_metadata || !file_metadata->is_regular) { - throw Error::NotFound("batch_io", "columnar file not found", path.string()); - } - - const uint64_t file_size = file_metadata->size; - constexpr uint64_t kFooterSize = sizeof(uint64_t) + kColumnarMagic.size(); - - if (file_size < kFooterSize) { - throw Error::InvalidData("batch_io", "columnar file is too small", path.string()); - } - - std::ifstream in = OpenInputFile(path); - SeekRead(path, in, file_size - kFooterSize); - - const uint64_t metadata_size = ReadStream(in); - std::string magic_read(kColumnarMagic.size(), '\0'); - ReadBytes(in, magic_read.data(), magic_read.size()); - - if (magic_read != kColumnarMagic) { - throw Error::InvalidData("batch_io", "invalid columnar magic", path.string()); - } - - if (metadata_size > file_size - kFooterSize) { - throw Error::InvalidData("batch_io", "metadata size exceeds file size", path.string()); - } - - SeekRead(path, in, file_size - kFooterSize - metadata_size); - - return ReadMetadata(in); -} - -ColumnarBatchReader::ColumnarBatchReader(const std::filesystem::path& path) - : path_(path), in_(OpenInputFile(path)), metadata_(ReadFileMetadata(path)) { - if (metadata_.schema.columns.empty()) { - throw Error::InvalidData("batch_io", "columnar schema is empty", path.string()); - } -} - -std::optional ColumnarBatchReader::ReadNext() { - if (next_group_ >= metadata_.row_groups.size()) { - return std::nullopt; - } - - const auto& [row_count, row_group_columns] = metadata_.row_groups[next_group_++]; - Batch batch(metadata_.schema, row_count); - - const auto& columns = batch.GetColumns(); - - if (row_group_columns.size() != columns.size()) { - throw Error::Mismatch("batch_io", "row group column count mismatch", path_.string()); - } - - for (size_t col = 0; col < columns.size(); ++col) { - const auto& [offset, size] = row_group_columns[col]; - SeekRead(path_, in_, offset); - columns[col]->ReadFrom(in_, row_count, size); - } - - return batch; -} - -ColumnarBatchWriter::ColumnarBatchWriter(const std::filesystem::path& path, Schema schema) - : path_(path), out_(OpenOutputFile(path)) { - if (schema.columns.empty()) { - throw Error::InvalidArgument("batch_io", "schema has no columns", path.string()); - } - metadata_.schema = std::move(schema); -} - -void ColumnarBatchWriter::Write(const Batch& batch) { - if (finalized_) { - throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); - } - - batch.Validate(); - if (batch.GetSchema() != metadata_.schema) { - throw Error::Mismatch("batch_io", "batch schema mismatch", path_.string()); - } - - const size_t row_count = batch.RowsCount(); - - if (row_count == 0) { - return; - } - if (row_count > std::numeric_limits::max()) { - throw Error::Overflow("batch_io", "row group exceeds supported size", path_.string()); - } - - RowGroupMetadata group; - group.row_count = row_count; - group.columns.reserve(batch.ColumnsCount()); - - for (const auto& column : batch.GetColumns()) { - const uint64_t offset = TellWrite(path_, out_); - column->WriteTo(out_); - const uint64_t end = TellWrite(path_, out_); - group.columns.push_back(ColumnChunkMetadata{offset, end - offset}); - } - - metadata_.row_groups.push_back(std::move(group)); -} - -void ColumnarBatchWriter::Flush() { FlushWrite(path_, out_); } - -void ColumnarBatchWriter::Finalize() { - if (finalized_) { - throw Error::InvalidState("batch_io", "writer already finalized", path_.string()); - } - - const uint64_t metadata_start = TellWrite(path_, out_); - WriteMetadata(out_, metadata_); - const uint64_t metadata_end = TellWrite(path_, out_); - const uint64_t metadata_size = metadata_end - metadata_start; - - WriteStream(out_, metadata_size); - WriteBytes(out_, kColumnarMagic); - - FlushWrite(path_, out_); - finalized_ = true; -} diff --git a/src/io/columnar_batch.cpp b/src/io/columnar_batch.cpp new file mode 100644 index 0000000..9b34359 --- /dev/null +++ b/src/io/columnar_batch.cpp @@ -0,0 +1,148 @@ +#include "io/columnar_batch.h" + +#include +#include +#include +#include +#include +#include + +#include "common/error.h" +#include "io/stream.h" + +static constexpr std::string_view ColumnarMagic = "CLMN"; + +static uint64_t TellWrite(const std::filesystem::path& path, std::ofstream& out) { + const auto pos = out.tellp(); + if (pos == -1) { + throw Error::PathIo("io", path, "tell file"); + } + return pos; +} + +static void FlushWrite(const std::filesystem::path& path, std::ofstream& out) { + out.flush(); + if (!out) { + throw Error::PathIo("io", path, "write file"); + } +} + +ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::path& path) { + const auto file_metadata = GetFileMetadata(path); + + if (!file_metadata || !file_metadata->is_regular) { + throw Error::NotFound("io", "columnar file not found", path.string()); + } + + const uint64_t file_size = file_metadata->size; + constexpr uint64_t FooterSize = sizeof(uint64_t) + ColumnarMagic.size(); + + if (file_size < FooterSize) { + throw Error::MalformedData("io", "columnar file is too small", path.string()); + } + + InputFile file(path); + const uint64_t metadata_size = file.ReadAt(file_size - FooterSize); + const std::string magic_read = file.ReadStringAt(file_size - ColumnarMagic.size(), ColumnarMagic.size()); + + if (magic_read != ColumnarMagic) { + throw Error::MalformedData("io", "invalid columnar magic", path.string()); + } + + if (metadata_size > file_size - FooterSize) { + throw Error::MalformedData("io", "metadata size exceeds file size", path.string()); + } + + return ReadMetadata(file.StreamAt(file_size - FooterSize - metadata_size)); +} + +ColumnarBatchReader::ColumnarBatchReader(const std::filesystem::path& path) + : path_(path), input_(path), metadata_(ReadFileMetadata(path)) { + if (metadata_.schema.columns.empty()) { + throw Error::MalformedData("io", "columnar schema is empty", path.string()); + } +} + +std::optional ColumnarBatchReader::ReadNext() { + if (next_group_ >= metadata_.row_groups.size()) { + return std::nullopt; + } + + const auto& [row_count, row_group_columns] = metadata_.row_groups[next_group_++]; + Batch batch(metadata_.schema, row_count); + + if (row_group_columns.size() != batch.ColumnsCount()) { + throw Error::InconsistentData("io", "row group column count mismatch", path_.string()); + } + + for (size_t col = 0; col < batch.ColumnsCount(); ++col) { + const auto& [offset, size] = row_group_columns[col]; + batch.ReadColumnFrom(col, input_.StreamAt(offset), row_count, size); + } + + return batch; +} + +ColumnarBatchWriter::ColumnarBatchWriter(const std::filesystem::path& path, Schema schema) + : path_(path), out_(OpenOutputFile(path)) { + if (schema.columns.empty()) { + throw Error::InvalidArgument("io", "schema has no columns", path.string()); + } + metadata_.schema = std::move(schema); +} + +void ColumnarBatchWriter::Write(const Batch& batch) { + if (finalized_) { + throw Error::InvalidState("io", "writer already finalized", path_.string()); + } + + batch.Validate(); + if (batch.GetSchema() != metadata_.schema) { + throw Error::InconsistentData("io", "batch schema mismatch", path_.string()); + } + + const size_t row_count = batch.RowsCount(); + + if (row_count == 0) { + return; + } + + if (row_count > std::numeric_limits::max()) { + throw Error::Overflow("io", "row group exceeds supported size", path_.string()); + } + + RowGroupMetadata group; + group.row_count = row_count; + group.columns.reserve(batch.ColumnsCount()); + + for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { + const uint64_t offset = TellWrite(path_, out_); + batch.ColumnAt(column_index).WriteTo(out_); + + const uint64_t end = TellWrite(path_, out_); + + group.columns.push_back(ColumnChunkMetadata{offset, end - offset}); + } + + metadata_.row_groups.push_back(std::move(group)); +} + +void ColumnarBatchWriter::Flush() { FlushWrite(path_, out_); } + +void ColumnarBatchWriter::Finalize() && { + if (finalized_) { + throw Error::InvalidState("io", "writer already finalized", path_.string()); + } + + const uint64_t metadata_start = TellWrite(path_, out_); + WriteMetadata(out_, metadata_); + const uint64_t metadata_end = TellWrite(path_, out_); + const uint64_t metadata_size = metadata_end - metadata_start; + + WriteStream(out_, metadata_size); + WriteBytes(out_, ColumnarMagic); + + FlushWrite(path_, out_); + + finalized_ = true; +} diff --git a/src/io/csv.cpp b/src/io/csv.cpp index 408d572..66819d6 100644 --- a/src/io/csv.cpp +++ b/src/io/csv.cpp @@ -1,9 +1,38 @@ -#include "csv.h" +#include "io/csv.h" +#include #include -#include "error.h" -#include "fileio.h" +#include "common/error.h" +#include "io/file.h" + +class CsvCharReader { + public: + explicit CsvCharReader(std::istream& in) : buffer_(in.rdbuf()) {} + + std::optional Read() const { + const Traits::int_type next = buffer_->sbumpc(); + if (Traits::eq_int_type(next, Traits::eof())) { + return std::nullopt; + } + return Traits::to_char_type(next); + } + + std::optional Peek() const { + const Traits::int_type next = buffer_->sgetc(); + if (Traits::eq_int_type(next, Traits::eof())) { + return std::nullopt; + } + return Traits::to_char_type(next); + } + + void Discard() const { static_cast(Read()); } + + private: + using Traits = std::streambuf::traits_type; + + std::streambuf* buffer_; +}; static bool NeedsCsvQuotes(const std::string_view value) { return value.find_first_of(",\"\n\r") != std::string_view::npos; @@ -39,39 +68,31 @@ bool CsvReader::ReadRow(std::vector& row) const { row.clear(); row.emplace_back(); - auto* buffer = in_->rdbuf(); - using Traits = std::streambuf::traits_type; - /* До этого чтение происходило через istream::get() и peek() - * Traits требуется только как вспомогательный тип для: - * - eof() - * - to_chat_type() - * - eq_int_type() - */ + const CsvCharReader input(*in_); bool in_quotes = false; bool saw_data = false; while (true) { - const Traits::int_type next = buffer->sbumpc(); + const std::optional next = input.Read(); - if (Traits::eq_int_type(next, Traits::eof())) { + if (!next.has_value()) { if (!saw_data) { return false; } if (in_quotes) { - throw Error::InvalidData("csv", "unexpected EOF inside quoted field"); + throw Error::MalformedData("io", "unexpected EOF inside quoted field"); } return true; } saw_data = true; - const char ch = Traits::to_char_type(next); + const char ch = *next; if (in_quotes) { if (ch == '"') { - const Traits::int_type peek = buffer->sgetc(); - if (Traits::to_char_type(peek) == '"') { - buffer->sbumpc(); + if (input.Peek() == '"') { + input.Discard(); row.back().push_back('"'); } else { in_quotes = false; @@ -101,8 +122,8 @@ bool CsvReader::ReadRow(std::vector& row) const { } if (ch == '\r') { - if (Traits::to_char_type(buffer->sgetc()) == '\n') { - buffer->sbumpc(); + if (input.Peek() == '\n') { + input.Discard(); } return true; } @@ -157,18 +178,21 @@ void CsvWriter::WriteRow(const std::vector& row) const { *out_ << ch; } } + *out_ << '"'; } + *out_ << '\n'; + if (!*out_) { - throw Error::Io("csv", "failed to write csv row"); + throw Error::Io("io", "failed to write csv row"); } } void CsvWriter::Flush() const { out_->flush(); if (!*out_) { - throw Error::Io("csv", "failed to write csv"); + throw Error::Io("io", "failed to write csv"); } } diff --git a/src/io/csv_batch.cpp b/src/io/csv_batch.cpp new file mode 100644 index 0000000..6e8d526 --- /dev/null +++ b/src/io/csv_batch.cpp @@ -0,0 +1,189 @@ +#include "io/csv_batch.h" + +#include +#include +#include +#include + +#include "common/error.h" +#include "common/int128.h" + +static uint64_t AddChecked(const uint64_t current, const uint64_t add) { + if (add > std::numeric_limits::max() - current) { + throw Error::Overflow("io", "batch byte size overflow"); + } + return current + add; +} + +static uint64_t EstimateValueBytes(const ColumnType type, const std::string_view value) { + switch (type) { + case ColumnType::Boolean: + return sizeof(uint8_t); + case ColumnType::Int16: + return sizeof(int16_t); + case ColumnType::Int32: + case ColumnType::Date: + return sizeof(int32_t); + case ColumnType::Int64: + case ColumnType::Timestamp: + return sizeof(int64_t); + case ColumnType::Int128: + return sizeof(Int128); + case ColumnType::Character: + return sizeof(char); + case ColumnType::String: + return value.size(); + } + throw Error::Unsupported("io", "unsupported column type"); +} + +static uint64_t EstimateRowBytes(const Schema& schema, const std::vector& row) { + uint64_t bytes = 0; + for (size_t i = 0; i < schema.columns.size(); ++i) { + bytes = AddChecked(bytes, EstimateValueBytes(schema.columns[i].type, row[i])); + } + return bytes; +} + +static void ValidateSizing(const BatchSizing& sizing) { + if (sizing.max_rows && *sizing.max_rows == 0) { + throw Error::InvalidArgument("io", "max rows must be > 0"); + } + + if (sizing.max_values && *sizing.max_values == 0) { + throw Error::InvalidArgument("io", "max values must be > 0"); + } + + if (sizing.max_bytes && *sizing.max_bytes == 0) { + throw Error::InvalidArgument("io", "max bytes must be > 0"); + } +} + +static void ReserveForSizing(Batch& batch, const BatchSizing& sizing, const size_t column_count) { + if (sizing.max_rows) { + batch.Reserve(*sizing.max_rows); + return; + } + + if (sizing.max_values && column_count > 0) { + batch.Reserve(std::max(1, *sizing.max_values / column_count)); + } +} + +CsvBatchReader::CsvBatchReader(const std::filesystem::path& path, Schema schema, BatchSizing sizing) + : csv_reader_(path), schema_(std::move(schema)), sizing_(std::move(sizing)) { + if (schema_.columns.empty()) { + throw Error::InvalidArgument("io", "schema has no columns"); + } + ValidateSizing(sizing_); +} + +std::optional CsvBatchReader::ReadNext() { + if (reached_eof_ && !pending_row_) { + return std::nullopt; + } + + Batch batch(schema_); + + const size_t column_count = schema_.columns.size(); + ReserveForSizing(batch, sizing_, column_count); + + std::vector row; + row.reserve(column_count); + + size_t rows = 0; + uint64_t bytes = 0; + + while (true) { + if (pending_row_) { + row = std::move(*pending_row_); + pending_row_.reset(); + } else { + if (!csv_reader_.ReadRow(row)) { + reached_eof_ = true; + break; + } + } + + if (row.size() != column_count) { + throw Error::InconsistentData("io", "data csv column count mismatch"); + } + + const size_t next_rows = rows + 1; + uint64_t next_bytes = bytes; + + if (sizing_.max_bytes) { + next_bytes = AddChecked(bytes, EstimateRowBytes(schema_, row)); + } + + if (rows > 0 && sizing_.WouldExceed(next_rows, column_count, next_bytes)) { + pending_row_ = std::move(row); + break; + } + + for (size_t col = 0; col < column_count; ++col) { + batch.AppendValueFromString(col, row[col]); + } + + rows = next_rows; + bytes = next_bytes; + } + + if (rows == 0 && reached_eof_ && !pending_row_) { + return std::nullopt; + } + + return batch; +} + +CsvBatchWriter::CsvBatchWriter(const std::filesystem::path& path, Schema schema) + : csv_writer_(path), schema_(std::move(schema)) { + if (schema_.columns.empty()) { + throw Error::InvalidArgument("io", "schema has no columns"); + } +} + +void CsvBatchWriter::Write(const Batch& batch) { + batch.Validate(); + if (batch.GetSchema() != schema_) { + throw Error::InconsistentData("io", "batch schema mismatch"); + } + + const size_t column_count = schema_.columns.size(); + const size_t row_count = batch.RowsCount(); + + std::vector row(column_count); + + for (size_t row_index = 0; row_index < row_count; ++row_index) { + for (size_t col = 0; col < column_count; ++col) { + row[col] = batch.ColumnAt(col).ValueAsString(row_index); + } + + csv_writer_.WriteRow(row); + } +} + +void CsvBatchWriter::Flush() { csv_writer_.Flush(); } + +void AppendBatchRows(const Batch& batch, std::vector>& rows) { + const size_t row_count = batch.RowsCount(); + const size_t column_count = batch.ColumnsCount(); + + for (size_t row = 0; row < row_count; ++row) { + std::vector values; + values.reserve(column_count); + + for (size_t col = 0; col < column_count; ++col) { + values.push_back(batch.ColumnAt(col).ValueAsString(row)); + } + + rows.push_back(std::move(values)); + } +} + +void WriteBatchCsv(const std::filesystem::path& path, const Batch& batch) { + CsvBatchWriter writer(path, batch.GetSchema()); + + writer.Write(batch); + writer.Flush(); +} diff --git a/src/io/fileio.cpp b/src/io/file.cpp similarity index 58% rename from src/io/fileio.cpp rename to src/io/file.cpp index a65ac38..7965ba0 100644 --- a/src/io/fileio.cpp +++ b/src/io/file.cpp @@ -1,14 +1,16 @@ -#include "fileio.h" +#include "io/file.h" #include +#include -#include "error.h" +#include "common/error.h" +#include "io/stream.h" -std::optional TryGetFileMetadata(const std::filesystem::path& path) { +std::optional GetFileMetadata(const std::filesystem::path& path) { std::error_code ec; if (!std::filesystem::exists(path, ec)) { if (ec) { - throw Error::PathIo("fileio", path, "check existence"); + throw Error::PathIo("io", path, "check existence"); } return std::nullopt; } @@ -16,7 +18,7 @@ std::optional TryGetFileMetadata(const std::filesystem::path& path FileMetadata metadata; const auto status = std::filesystem::status(path, ec); if (ec) { - throw Error::PathIo("fileio", path, "read status"); + throw Error::PathIo("io", path, "read status"); } metadata.is_regular = std::filesystem::is_regular_file(status); @@ -24,13 +26,13 @@ std::optional TryGetFileMetadata(const std::filesystem::path& path if (metadata.is_regular) { metadata.size = std::filesystem::file_size(path, ec); if (ec) { - throw Error::PathIo("fileio", path, "read file size"); + throw Error::PathIo("io", path, "read file size"); } } metadata.last_write_time = std::filesystem::last_write_time(path, ec); if (ec) { - throw Error::PathIo("fileio", path, "read last write time"); + throw Error::PathIo("io", path, "read last write time"); } return metadata; @@ -40,8 +42,9 @@ bool FileExists(const std::filesystem::path& path) { std::error_code ec; const bool exists = std::filesystem::exists(path, ec); if (ec) { - throw Error::PathIo("fileio", path, "check existence"); + throw Error::PathIo("io", path, "check existence"); } + return exists; } @@ -54,22 +57,51 @@ void EnsureParentDirectory(const std::filesystem::path& path) { std::error_code ec; std::filesystem::create_directories(parent, ec); if (ec) { - throw Error::PathIo("fileio", path, "create parent directories"); + throw Error::PathIo("io", path, "create parent directories"); } } std::ifstream OpenInputFile(const std::filesystem::path& path) { std::ifstream file(path, std::ios::binary); if (!file.is_open()) { - throw Error::PathIo("fileio", path, "open for read"); + throw Error::PathIo("io", path, "open for read"); } return file; } +void SeekInputFile(std::istream& in, const std::filesystem::path& path, const uint64_t offset) { + in.seekg(static_cast(offset)); + if (!in) { + throw Error::PathIo("io", path, "seek for read"); + } +} + +InputFile::InputFile(std::filesystem::path path) : path_(std::move(path)), in_(OpenInputFile(path_)) {} + +void InputFile::ReadAt(char* dst, const size_t size, const uint64_t offset) { + SeekInputFile(in_, path_, offset); + try { + ReadBytes(in_, dst, size); + } catch (const Error&) { + throw Error::PathIo("io", path_, "read file"); + } +} + +std::string InputFile::ReadStringAt(const uint64_t offset, const size_t size) { + std::string result(size, '\0'); + ReadAt(result.data(), result.size(), offset); + return result; +} + +std::istream& InputFile::StreamAt(const uint64_t offset) { + SeekInputFile(in_, path_, offset); + return in_; +} + std::ofstream OpenOutputFile(const std::filesystem::path& path, const FileOpenMode mode) { std::ofstream file(path, std::ios::binary | (mode == FileOpenMode::Append ? std::ios::app : std::ios::trunc)); if (!file.is_open()) { - throw Error::PathIo("fileio", path, mode == FileOpenMode::Append ? "open for append" : "open for write"); + throw Error::PathIo("io", path, mode == FileOpenMode::Append ? "open for append" : "open for write"); } return file; } @@ -82,13 +114,14 @@ std::string ReadTextFile(const std::filesystem::path& path) { std::vector ReadFileBytes(const std::filesystem::path& path) { std::ifstream file = OpenInputFile(path); file.seekg(0, std::ios::end); + if (!file) { - throw Error::PathIo("fileio", path, "read file size"); + throw Error::PathIo("io", path, "read file size"); } const auto end_pos = file.tellg(); if (end_pos < 0) { - throw Error::PathIo("fileio", path, "read file size"); + throw Error::PathIo("io", path, "read file size"); } std::vector data(end_pos); @@ -100,7 +133,7 @@ std::vector ReadFileBytes(const std::filesystem::path& path) { file.read(reinterpret_cast(data.data()), data.size()); if (!file) { - throw Error::PathIo("fileio", path, "read file"); + throw Error::PathIo("io", path, "read file"); } return data; @@ -111,13 +144,13 @@ void WriteFileBytes(const std::filesystem::path& path, const std::span(bytes.data()), bytes.size()); if (!file) { - throw Error::PathIo("fileio", path, "write file"); + throw Error::PathIo("io", path, "write file"); } } file.flush(); if (!file) { - throw Error::PathIo("fileio", path, "write file"); + throw Error::PathIo("io", path, "write file"); } } diff --git a/src/io/stream.cpp b/src/io/stream.cpp index 5193bfb..f2c372d 100644 --- a/src/io/stream.cpp +++ b/src/io/stream.cpp @@ -1,6 +1,6 @@ -#include "stream.h" +#include "io/stream.h" -#include "error.h" +#include "common/error.h" void ReadBytes(std::istream& in, char* dst, const size_t size) { if (size == 0) { @@ -8,7 +8,7 @@ void ReadBytes(std::istream& in, char* dst, const size_t size) { } in.read(dst, size); if (in.gcount() != static_cast(size) || !in) { - throw Error::Io("stream", "failed to read bytes"); + throw Error::Io("io", "failed to read bytes"); } } @@ -18,6 +18,6 @@ void WriteBytes(std::ostream& out, const std::string_view bytes) { } out.write(bytes.data(), bytes.size()); if (!out) { - throw Error::Io("stream", "failed to write bytes"); + throw Error::Io("io", "failed to write bytes"); } } diff --git a/src/model/batch.cpp b/src/model/batch.cpp index bad890d..b646c3a 100644 --- a/src/model/batch.cpp +++ b/src/model/batch.cpp @@ -1,15 +1,18 @@ -#include "batch.h" +#include "model/batch.h" #include -#include "error.h" +#include "common/error.h" -static std::vector> CloneColumns(const std::vector>& columns) { - std::vector> clones; +static std::vector> CloneColumns( + const std::vector>& columns) { + std::vector> clones; clones.reserve(columns.size()); + for (const auto& column : columns) { - clones.push_back(column->Clone()); + clones.push_back(column->CloneMutable()); } + return clones; } @@ -51,23 +54,31 @@ void Batch::Reserve(const size_t n) const { } } -const Column& Batch::ColumnAt(const size_t i) const { - if (i >= columns_.size()) { - throw Error::OutOfRange("batch", "column index out of range"); +void Batch::AppendValueFromString(const size_t column_index, const std::string& value) const { + if (column_index >= columns_.size()) { + throw Error::OutOfRange("model", "column index out of range"); } - return *columns_[i]; + columns_[column_index]->AppendFromString(value); } -Column& Batch::ColumnAt(const size_t i) { +void Batch::ReadColumnFrom(const size_t column_index, std::istream& in, const uint32_t row_count, + const uint64_t size) const { + if (column_index >= columns_.size()) { + throw Error::OutOfRange("model", "column index out of range"); + } + columns_[column_index]->ReadFrom(in, row_count, size); +} + +const Column& Batch::ColumnAt(const size_t i) const { if (i >= columns_.size()) { - throw Error::OutOfRange("batch", "column index out of range"); + throw Error::OutOfRange("model", "column index out of range"); } return *columns_[i]; } void Batch::Validate() const { if (schema_.columns.size() != columns_.size()) { - throw Error::Mismatch("batch", "column count mismatch"); + throw Error::InconsistentData("model", "column count mismatch"); } if (columns_.empty()) { @@ -77,10 +88,11 @@ void Batch::Validate() const { const size_t rows = columns_.front()->Size(); for (size_t i = 0; i < columns_.size(); ++i) { if (columns_[i]->Type() != schema_.columns[i].type) { - throw Error::Mismatch("batch", "column type mismatch"); + throw Error::InconsistentData("model", "column type mismatch"); } + if (columns_[i]->Size() != rows) { - throw Error::Mismatch("batch", "column row count mismatch"); + throw Error::InconsistentData("model", "column row count mismatch"); } } } diff --git a/src/model/column.cpp b/src/model/column.cpp index 7eded02..0a14447 100644 --- a/src/model/column.cpp +++ b/src/model/column.cpp @@ -1,19 +1,19 @@ -#include "column.h" +#include "model/column.h" #include -#include "column_boolean.h" -#include "column_character.h" -#include "column_date.h" -#include "column_int128.h" -#include "column_int16.h" -#include "column_int32.h" -#include "column_int64.h" -#include "column_string.h" -#include "column_timestamp.h" -#include "error.h" +#include "model/column_boolean.h" +#include "model/column_character.h" +#include "model/column_date.h" +#include "model/column_int128.h" +#include "model/column_int16.h" +#include "model/column_int32.h" +#include "model/column_int64.h" +#include "model/column_string.h" +#include "model/column_timestamp.h" +#include "common/error.h" -std::unique_ptr CreateColumn(const ColumnType type) { +std::unique_ptr CreateColumn(const ColumnType type) { switch (type) { case ColumnType::Int64: return std::make_unique(); @@ -34,7 +34,8 @@ std::unique_ptr CreateColumn(const ColumnType type) { case ColumnType::Character: return std::make_unique(); } - throw Error::Unsupported("column", "unsupported column type"); + + throw Error::Unsupported("model", "unsupported column type"); } Column::Column(const ColumnType type) : type_(type) {} diff --git a/src/model/column_boolean.cpp b/src/model/column_boolean.cpp deleted file mode 100644 index 4959ec5..0000000 --- a/src/model/column_boolean.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_boolean.h" - -#include "parsing.h" - -void BooleanColumn::AppendFromString(const std::string& value) { AppendValue(ParseBoolean(value) ? 1 : 0); } - -std::string BooleanColumn::ValueAsString(const size_t row) const { return BooleanToString(ValueAt(row) != 0); } diff --git a/src/model/column_character.cpp b/src/model/column_character.cpp deleted file mode 100644 index 5184b18..0000000 --- a/src/model/column_character.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_character.h" - -#include "parsing.h" - -void CharacterColumn::AppendFromString(const std::string& value) { AppendValue(ParseCharacter(value)); } - -std::string CharacterColumn::ValueAsString(const size_t row) const { return std::string(1, ValueAt(row)); } diff --git a/src/model/column_date.cpp b/src/model/column_date.cpp deleted file mode 100644 index 4bfcc9e..0000000 --- a/src/model/column_date.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_date.h" - -#include "parsing.h" - -void DateColumn::AppendFromString(const std::string& value) { AppendValue(ParseDate(value)); } - -std::string DateColumn::ValueAsString(const size_t row) const { return DateToString(ValueAt(row)); } diff --git a/src/model/column_int128.cpp b/src/model/column_int128.cpp deleted file mode 100644 index 161ab39..0000000 --- a/src/model/column_int128.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_int128.h" - -#include "parsing.h" - -void Int128Column::AppendFromString(const std::string& value) { AppendValue(ParseInt128(value)); } - -std::string Int128Column::ValueAsString(const size_t row) const { return Int128ToString(ValueAt(row)); } diff --git a/src/model/column_int16.cpp b/src/model/column_int16.cpp deleted file mode 100644 index 94415e8..0000000 --- a/src/model/column_int16.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_int16.h" - -#include "parsing.h" - -void Int16Column::AppendFromString(const std::string& value) { AppendValue(ParseInt16(value)); } - -std::string Int16Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_int32.cpp b/src/model/column_int32.cpp deleted file mode 100644 index b9353b9..0000000 --- a/src/model/column_int32.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_int32.h" - -#include "parsing.h" - -void Int32Column::AppendFromString(const std::string& value) { AppendValue(ParseInt32(value)); } - -std::string Int32Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_int64.cpp b/src/model/column_int64.cpp deleted file mode 100644 index f6c2399..0000000 --- a/src/model/column_int64.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_int64.h" - -#include "parsing.h" - -void Int64Column::AppendFromString(const std::string& value) { AppendValue(ParseInt64(value)); } - -std::string Int64Column::ValueAsString(const size_t row) const { return std::to_string(ValueAt(row)); } diff --git a/src/model/column_string.cpp b/src/model/column_string.cpp index 31cea45..807798a 100644 --- a/src/model/column_string.cpp +++ b/src/model/column_string.cpp @@ -1,5 +1,77 @@ -#include "column_string.h" +#include "model/column_string.h" -void StringColumn::AppendFromString(const std::string& value) { AppendValue(value); } +#include +#include +#include -std::string StringColumn::ValueAsString(const size_t row) const { return ValueAt(row); } +#include "io/stream.h" +#include "common/error.h" + +StringColumn::StringColumn() : MutableColumn(ColumnType::String) {} + +size_t StringColumn::Size() const { return values_.size(); } + +void StringColumn::Reserve(const size_t n) { values_.reserve(n); } + +void StringColumn::Clear() { values_.clear(); } + +void StringColumn::AppendFromString(const std::string& value) { values_.push_back(value); } + +std::string StringColumn::ValueAsString(const size_t row) const { + CheckRowIndex(ModuleName(), row, values_.size()); + return values_[row]; +} + +std::unique_ptr StringColumn::Clone() const { return std::make_unique(*this); } + +std::unique_ptr StringColumn::CloneMutable() const { return std::make_unique(*this); } + +void StringColumn::WriteTo(std::ostream& out) const { + size_t total_size = 0; + + for (const auto& value : values_) { + if (value.size() > std::numeric_limits::max()) { + throw Error::Overflow(ModuleName(), "value exceeds supported size"); + } + total_size += sizeof(uint32_t) + value.size(); + } + + std::string buffer(total_size, '\0'); + char* dst = buffer.data(); + + for (const auto& value : values_) { + const uint32_t length = value.size(); + + std::memcpy(dst, &length, sizeof(length)); + dst += sizeof(length); + + if (!value.empty()) { + std::memcpy(dst, value.data(), value.size()); + dst += value.size(); + } + } + + WriteBytes(out, buffer); +} + +void StringColumn::ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) { + values_.clear(); + values_.reserve(row_count); + + uint64_t consumed = 0; + + for (uint32_t row_index = 0; row_index < row_count; ++row_index) { + const uint32_t length = ReadStream(in); + consumed += sizeof(uint32_t); + + std::string value(length, '\0'); + ReadBytes(in, value.data(), value.size()); + + consumed += length; + values_.push_back(std::move(value)); + } + + if (consumed != size) { + throw Error::InconsistentData(ModuleName(), "column chunk size mismatch"); + } +} diff --git a/src/model/column_timestamp.cpp b/src/model/column_timestamp.cpp deleted file mode 100644 index 42867fe..0000000 --- a/src/model/column_timestamp.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "column_timestamp.h" - -#include "parsing.h" - -void TimestampColumn::AppendFromString(const std::string& value) { AppendValue(ParseTimestamp(value)); } - -std::string TimestampColumn::ValueAsString(const size_t row) const { return TimestampToString(ValueAt(row)); } diff --git a/src/model/metadata.cpp b/src/model/metadata.cpp index 33736bc..66765ab 100644 --- a/src/model/metadata.cpp +++ b/src/model/metadata.cpp @@ -1,10 +1,10 @@ -#include "metadata.h" +#include "model/metadata.h" #include #include -#include "error.h" -#include "stream.h" +#include "common/error.h" +#include "io/stream.h" static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { switch (static_cast(type_byte)) { @@ -20,7 +20,7 @@ static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { return static_cast(type_byte); } - throw Error::InvalidData("columnar", "unknown column type in metadata"); + throw Error::MalformedData("model", "unknown column type in metadata"); } ColumnarMetadata ReadMetadata(std::istream& in) { @@ -34,6 +34,7 @@ ColumnarMetadata ReadMetadata(std::istream& in) { std::string name(name_size, '\0'); ReadBytes(in, name.data(), name.size()); + const uint8_t type_byte = ReadStream(in); const ColumnType type = ColumnTypeFromByte(type_byte); @@ -49,7 +50,7 @@ ColumnarMetadata ReadMetadata(std::istream& in) { const uint32_t columns_in_group = ReadStream(in); if (columns_in_group != column_count) { - throw Error::Mismatch("columnar", "row group column count mismatch"); + throw Error::InconsistentData("model", "row group column count mismatch"); } group.columns.reserve(columns_in_group); @@ -58,6 +59,7 @@ ColumnarMetadata ReadMetadata(std::istream& in) { ColumnChunkMetadata chunk; chunk.offset = ReadStream(in); chunk.size = ReadStream(in); + group.columns.push_back(chunk); } @@ -69,22 +71,23 @@ ColumnarMetadata ReadMetadata(std::istream& in) { void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { if (metadata.schema.columns.size() > std::numeric_limits::max()) { - throw Error::Overflow("columnar", "too many columns for metadata"); + throw Error::Overflow("model", "too many columns for metadata"); } WriteStream(out, metadata.schema.columns.size()); for (const auto& [name, type] : metadata.schema.columns) { if (name.size() > std::numeric_limits::max()) { - throw Error::Overflow("columnar", "column name too long"); + throw Error::Overflow("model", "column name too long"); } + WriteStream(out, name.size()); WriteBytes(out, name); WriteStream(out, static_cast(type)); } if (metadata.row_groups.size() > std::numeric_limits::max()) { - throw Error::Overflow("columnar", "too many row groups for metadata"); + throw Error::Overflow("model", "too many row groups for metadata"); } WriteStream(out, metadata.row_groups.size()); @@ -92,6 +95,7 @@ void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { for (const auto& [row_count, columns] : metadata.row_groups) { WriteStream(out, row_count); WriteStream(out, columns.size()); + for (const auto& [offset, size] : columns) { WriteStream(out, offset); WriteStream(out, size); diff --git a/src/model/schema.cpp b/src/model/schema_csv.cpp similarity index 56% rename from src/model/schema.cpp rename to src/model/schema_csv.cpp index 3c0aae4..b9587c6 100644 --- a/src/model/schema.cpp +++ b/src/model/schema_csv.cpp @@ -1,63 +1,54 @@ -#include "schema.h" +#include "model/schema_csv.h" #include +#include +#include -#include "csv.h" -#include "error.h" -#include "fileio.h" -#include "parsing.h" +#include "common/error.h" +#include "common/parsing.h" +#include "io/csv.h" static bool CanParseAs(const ColumnType type, const std::string& value) { - try { - switch (type) { - case ColumnType::Boolean: - static_cast(ParseBoolean(value)); - return true; - case ColumnType::Int16: - static_cast(ParseInt16(value)); - return true; - case ColumnType::Int32: - static_cast(ParseInt32(value)); - return true; - case ColumnType::Int64: - static_cast(ParseInt64(value)); - return true; - case ColumnType::Int128: - static_cast(ParseInt128(value)); - return true; - case ColumnType::Date: - static_cast(ParseDate(value)); - return true; - case ColumnType::Timestamp: - static_cast(ParseTimestamp(value)); - return true; - case ColumnType::Character: - static_cast(ParseCharacter(value)); - return true; - case ColumnType::String: - return true; - } - } catch (const Error&) { - return false; + switch (type) { + case ColumnType::Boolean: + return TryParseBoolean(value).has_value(); + case ColumnType::Int16: + return TryParseInt16(value).has_value(); + case ColumnType::Int32: + return TryParseInt32(value).has_value(); + case ColumnType::Int64: + return TryParseInt64(value).has_value(); + case ColumnType::Int128: + return TryParseInt128(value).has_value(); + case ColumnType::Date: + return TryParseDate(value).has_value(); + case ColumnType::Timestamp: + return TryParseTimestamp(value).has_value(); + case ColumnType::Character: + return TryParseCharacter(value).has_value(); + case ColumnType::String: + return true; } - throw Error::Unsupported("schema", "unknown column type"); + throw Error::Unsupported("model", "unknown column type"); } static ColumnType InferColumnType(const std::vector& values) { - static constexpr std::array kInferenceOrder = { + static constexpr std::array InferenceOrder = { ColumnType::Boolean, ColumnType::Int16, ColumnType::Int32, ColumnType::Int64, ColumnType::Int128, ColumnType::Date, ColumnType::Timestamp, ColumnType::Character, ColumnType::String, }; - for (const ColumnType type : kInferenceOrder) { + for (const ColumnType type : InferenceOrder) { bool matches = true; + for (const auto& value : values) { if (!CanParseAs(type, value)) { matches = false; break; } } + if (matches) { return type; } @@ -76,9 +67,15 @@ Schema ReadSchemaCsv(const std::filesystem::path& path) { if (row.size() == 1 && row[0].empty()) { continue; } + + while (row.size() > 2 && row.back().empty()) { + row.pop_back(); + } + if (row.size() != 2) { - throw Error::InvalidData("schema", "schema csv must have 2 columns", path.string()); + throw Error::MalformedData("model", "schema csv must have 2 columns", path.string()); } + schema.columns.push_back(ColumnSchema{ row[0], ParseColumnType(row[1]), @@ -86,7 +83,7 @@ Schema ReadSchemaCsv(const std::filesystem::path& path) { } if (schema.columns.empty()) { - throw Error::InvalidData("schema", "schema csv is empty", path.string()); + throw Error::MalformedData("model", "schema csv is empty", path.string()); } return schema; @@ -102,7 +99,7 @@ Schema InferSchemaCsv(const std::filesystem::path& path) { if (values_by_column.empty()) { values_by_column.resize(row.size()); } else if (row.size() != values_by_column.size()) { - throw Error::InvalidData("schema", "csv rows have inconsistent column count", path.string()); + throw Error::MalformedData("model", "csv rows have inconsistent column count", path.string()); } for (size_t i = 0; i < row.size(); ++i) { @@ -111,7 +108,7 @@ Schema InferSchemaCsv(const std::filesystem::path& path) { } if (values_by_column.empty()) { - throw Error::InvalidData("schema", "csv is empty", path.string()); + throw Error::MalformedData("model", "csv is empty", path.string()); } Schema schema; diff --git a/src/sql_parser/tokenizer.cpp b/src/sql_parser/tokenizer.cpp index 04c5038..6e2cb46 100644 --- a/src/sql_parser/tokenizer.cpp +++ b/src/sql_parser/tokenizer.cpp @@ -1,4 +1,4 @@ -#include "tokenizer.h" +#include "sql_parser/tokenizer.h" #include #include @@ -24,7 +24,7 @@ Expected Tokenizer::GetNext() { } if (pos_ >= query_.size()) { - return MakeToken(Tokens::EOI, {}, query_.size()); + return MakeToken(Tokens::EndOfInput, {}, query_.size()); } const char ch = query_[pos_]; @@ -129,23 +129,28 @@ Expected Tokenizer::GetNext() { ++pos_; return MakeToken(Tokens::LessOrEqual, "<=", start); } + if (pos_ < query_.size() && query_[pos_] == '>') { ++pos_; return MakeToken(Tokens::NotEqual, "<>", start); } + return MakeToken(Tokens::Less, "<", start); case '>': ++pos_; + if (pos_ < query_.size() && query_[pos_] == '=') { ++pos_; return MakeToken(Tokens::GreaterOrEqual, ">=", start); } + return MakeToken(Tokens::Greater, ">", start); case '!': if (pos_ + 1 < query_.size() && query_[pos_ + 1] == '=') { pos_ += 2; return MakeToken(Tokens::NotEqual, "!=", start); } + return MakeUnexpectedCharacter(ch, start); default: return MakeUnexpectedCharacter(ch, start); @@ -165,7 +170,8 @@ Expected> TokenizeSql(const std::string_view input) { } tokens.push_back(token.value()); - if (token.value()->GetType() == Tokens::EOI) { + + if (token.value()->GetType() == Tokens::EndOfInput) { break; } } diff --git a/src/sql_parser/tokenizer_factory.cpp b/src/sql_parser/tokenizer_factory.cpp index 7ee46eb..aec0708 100644 --- a/src/sql_parser/tokenizer_factory.cpp +++ b/src/sql_parser/tokenizer_factory.cpp @@ -4,10 +4,10 @@ #include #include -#include "ascii.h" -#include "tokenizer.h" +#include "common/ascii.h" +#include "sql_parser/tokenizer.h" -static const std::unordered_map kKeywords = { +static const std::unordered_map Keywords = { {"AS", Tokens::As}, {"AND", Tokens::And}, {"AVG", Tokens::Avg}, {"BY", Tokens::By}, {"COUNT", Tokens::Count}, {"CREATE", Tokens::Create}, {"DISTINCT", Tokens::Distinct}, {"FROM", Tokens::From}, {"GROUP", Tokens::Group}, {"LENGTH", Tokens::Length}, {"LIMIT", Tokens::Limit}, {"MAX", Tokens::Max}, @@ -36,10 +36,12 @@ static std::string DescribeCharacter(const char ch) { Tokens ResolveIdentifierType(const std::string_view text) { const std::string upper = ToUpperAscii(text); - const auto it = kKeywords.find(upper); - if (it != kKeywords.end()) { + const auto it = Keywords.find(upper); + + if (it != Keywords.end()) { return it->second; } + return Tokens::NameToken; } @@ -113,18 +115,19 @@ TokenPtr MakeToken(const Tokens type, std::string text, const size_t offset) { return std::make_shared(std::move(text), offset); case Tokens::GreaterOrEqual: return std::make_shared(std::move(text), offset); - case Tokens::EOI: + case Tokens::EndOfInput: return std::make_shared(std::move(text), offset); } + return nullptr; } Expected MakeUnexpectedCharacter(const char ch, const size_t offset) { - return tl::unexpected(Error::InvalidData("tokenizer_factory", "unexpected character '" + DescribeCharacter(ch) + - "' at position " + std::to_string(offset))); + return tl::unexpected(Error::MalformedData( + "sql_parser", "unexpected character '" + DescribeCharacter(ch) + "' at position " + std::to_string(offset))); } Expected MakeUnterminatedString(const size_t offset) { return tl::unexpected( - Error::InvalidData("tokenizer_factory", "unterminated string literal at position " + std::to_string(offset))); + Error::MalformedData("sql_parser", "unterminated string literal at position " + std::to_string(offset))); } diff --git a/src/sql_parser/tokenizer_tokens.cpp b/src/sql_parser/tokenizer_tokens.cpp index b1bc6b5..04b8552 100644 --- a/src/sql_parser/tokenizer_tokens.cpp +++ b/src/sql_parser/tokenizer_tokens.cpp @@ -1,6 +1,6 @@ #include -#include "tokenizer.h" +#include "sql_parser/tokenizer.h" std::string_view TokenTypeToString(const Tokens type) noexcept { switch (type) { @@ -72,9 +72,10 @@ std::string_view TokenTypeToString(const Tokens type) noexcept { return "greater"; case Tokens::GreaterOrEqual: return "greater_or_equal"; - case Tokens::EOI: + case Tokens::EndOfInput: return "end_of_input"; } + return "unknown"; } @@ -88,46 +89,4 @@ void Command::AddArg(TokenPtr arg) { args_.push_back(std::move(arg)); } const std::vector& Command::GetArgs() const noexcept { return args_; } -// alert: не типобезопасная вещь, лучше переписать -#define COLUMNAR_DEFINE_TOKEN_TYPE(token_class, token_value) \ - Tokens token_class::GetType() const noexcept { return Tokens::token_value; } - -COLUMNAR_DEFINE_TOKEN_TYPE(TSelectToken, Select) -COLUMNAR_DEFINE_TOKEN_TYPE(TCreateToken, Create) -COLUMNAR_DEFINE_TOKEN_TYPE(TFromToken, From) -COLUMNAR_DEFINE_TOKEN_TYPE(TLimitToken, Limit) -COLUMNAR_DEFINE_TOKEN_TYPE(TOrderToken, Order) -COLUMNAR_DEFINE_TOKEN_TYPE(TWhereToken, Where) -COLUMNAR_DEFINE_TOKEN_TYPE(TGroupToken, Group) -COLUMNAR_DEFINE_TOKEN_TYPE(TSumToken, Sum) -COLUMNAR_DEFINE_TOKEN_TYPE(TMinToken, Min) -COLUMNAR_DEFINE_TOKEN_TYPE(TMaxToken, Max) -COLUMNAR_DEFINE_TOKEN_TYPE(TCountToken, Count) -COLUMNAR_DEFINE_TOKEN_TYPE(TDistinctToken, Distinct) -COLUMNAR_DEFINE_TOKEN_TYPE(TAvgToken, Avg) -COLUMNAR_DEFINE_TOKEN_TYPE(TLengthToken, Length) -COLUMNAR_DEFINE_TOKEN_TYPE(TPlusToken, Plus) -COLUMNAR_DEFINE_TOKEN_TYPE(TMinusToken, Minus) -COLUMNAR_DEFINE_TOKEN_TYPE(TNameToken, NameToken) -COLUMNAR_DEFINE_TOKEN_TYPE(TNumericLiteralToken, NumericLiteral) -COLUMNAR_DEFINE_TOKEN_TYPE(TStringLiteralToken, StringLiteral) -COLUMNAR_DEFINE_TOKEN_TYPE(TAsToken, As) -COLUMNAR_DEFINE_TOKEN_TYPE(TAndToken, And) -COLUMNAR_DEFINE_TOKEN_TYPE(TByToken, By) -COLUMNAR_DEFINE_TOKEN_TYPE(TOpenBracketToken, OpenBracket) -COLUMNAR_DEFINE_TOKEN_TYPE(TCommaToken, Comma) -COLUMNAR_DEFINE_TOKEN_TYPE(TCloseBracketToken, CloseBracket) -COLUMNAR_DEFINE_TOKEN_TYPE(TAsteriskToken, Asterisk) -COLUMNAR_DEFINE_TOKEN_TYPE(TDotToken, Dot) -COLUMNAR_DEFINE_TOKEN_TYPE(TSemicolonToken, Semicolon) -COLUMNAR_DEFINE_TOKEN_TYPE(TEqualToken, Equal) -COLUMNAR_DEFINE_TOKEN_TYPE(TNotEqualToken, NotEqual) -COLUMNAR_DEFINE_TOKEN_TYPE(TLessToken, Less) -COLUMNAR_DEFINE_TOKEN_TYPE(TLessOrEqualToken, LessOrEqual) -COLUMNAR_DEFINE_TOKEN_TYPE(TGreaterToken, Greater) -COLUMNAR_DEFINE_TOKEN_TYPE(TGreaterOrEqualToken, GreaterOrEqual) -COLUMNAR_DEFINE_TOKEN_TYPE(TEndOfInputToken, EOI) - -#undef COLUMNAR_DEFINE_TOKEN_TYPE - std::string_view TNameToken::GetName() const noexcept { return GetText(); } diff --git a/tests/test_batch.cpp b/tests/test_batch.cpp index 90ddefa..eb5f93d 100644 --- a/tests/test_batch.cpp +++ b/tests/test_batch.cpp @@ -1,19 +1,51 @@ #include #include +#include #include -#include "columnar_batch_io.h" -#include "csv.h" -#include "csv_batch_io.h" -#include "error.h" #include "gtest/gtest.h" -#include "temp_file.h" +#include "io/columnar_batch.h" +#include "io/csv.h" +#include "io/csv_batch.h" +#include "common/error.h" +#include "testing/temp_file.h" static_assert(std::is_copy_constructible_v); static_assert(std::is_copy_assignable_v); static_assert(std::is_move_constructible_v); static_assert(std::is_move_assignable_v); +TEST(batch, usage_example_csv_to_columnar_roundtrip) { + Schema schema; + schema.columns = { + {"id", ColumnType::Int64}, + {"name", ColumnType::String}, + }; + + const TempFile csv_file("batch_usage_csv"); + const TempFile columnar_file("batch_usage_columnar"); + WriteRows(csv_file.Path(), { + {"1", "alpha"}, + {"2", "beta"}, + }); + + CsvBatchReader csv_reader(csv_file.Path(), schema, {}); + ColumnarBatchWriter columnar_writer(columnar_file.Path(), schema); + + while (auto batch = csv_reader.ReadNext()) { + columnar_writer.Write(*batch); + } + std::move(columnar_writer).Finalize(); + + ColumnarBatchReader columnar_reader(columnar_file.Path()); + auto batch = columnar_reader.ReadNext(); + ASSERT_TRUE(batch.has_value()); + EXPECT_EQ(batch->RowsCount(), 2u); + EXPECT_EQ(batch->ColumnAt(0).ValueAsString(0), "1"); + EXPECT_EQ(batch->ColumnAt(1).ValueAsString(1), "beta"); + EXPECT_FALSE(columnar_reader.ReadNext().has_value()); +} + TEST(batch, csv_reader_respects_max_rows) { Schema schema; schema.columns = { @@ -57,10 +89,10 @@ TEST(batch, write_batch_csv_writes_single_batch) { }; Batch batch(schema); - batch.ColumnAt(0).AppendFromString("1"); - batch.ColumnAt(1).AppendFromString("alpha"); - batch.ColumnAt(0).AppendFromString("2"); - batch.ColumnAt(1).AppendFromString("be,ta"); + batch.AppendValueFromString(0, "1"); + batch.AppendValueFromString(1, "alpha"); + batch.AppendValueFromString(0, "2"); + batch.AppendValueFromString(1, "be,ta"); const TempFile data_out("batch_write_csv"); WriteBatchCsv(data_out.Path(), batch); @@ -97,7 +129,7 @@ TEST(batch, columnar_roundtrip) { while (auto batch = csv_reader.ReadNext()) { columnar_writer.Write(*batch); } - columnar_writer.Finalize(); + std::move(columnar_writer).Finalize(); ColumnarBatchReader columnar_reader(columnar_file.Path()); EXPECT_EQ(columnar_reader.GetMetadata().row_groups.size(), 2u); @@ -118,9 +150,9 @@ TEST(batch, validate_detects_row_count_mismatch) { }; Batch batch(schema); - batch.ColumnAt(0).AppendFromString("1"); - batch.ColumnAt(0).AppendFromString("2"); - batch.ColumnAt(1).AppendFromString("alpha"); + batch.AppendValueFromString(0, "1"); + batch.AppendValueFromString(0, "2"); + batch.AppendValueFromString(1, "alpha"); EXPECT_THROW(batch.Validate(), Error); } @@ -133,12 +165,12 @@ TEST(batch, copy_is_deep) { }; Batch original(schema); - original.ColumnAt(0).AppendFromString("1"); - original.ColumnAt(1).AppendFromString("alpha"); + original.AppendValueFromString(0, "1"); + original.AppendValueFromString(1, "alpha"); Batch copied = original; - copied.ColumnAt(0).AppendFromString("2"); - copied.ColumnAt(1).AppendFromString("beta"); + copied.AppendValueFromString(0, "2"); + copied.AppendValueFromString(1, "beta"); EXPECT_EQ(original.RowsCount(), 1u); EXPECT_EQ(original.ColumnAt(0).ValueAsString(0), "1"); @@ -237,9 +269,9 @@ TEST(batch, columnar_writer_rejects_after_finalize) { const TempFile columnar_file("batch_finalize_columnar"); ColumnarBatchWriter writer(columnar_file.Path(), schema); - writer.Finalize(); + std::move(writer).Finalize(); Batch batch(schema); EXPECT_THROW(writer.Write(batch), Error); - EXPECT_THROW(writer.Finalize(), Error); + EXPECT_THROW(std::move(writer).Finalize(), Error); } diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp index 4132268..172754b 100644 --- a/tests/test_columnar.cpp +++ b/tests/test_columnar.cpp @@ -2,15 +2,16 @@ #include #include -#include "columnar_batch_io.h" -#include "csv_columnar.h" -#include "csv.h" -#include "error.h" -#include "fileio.h" +#include "convert/csv_columnar.h" #include "gtest/gtest.h" -#include "parsing.h" -#include "schema.h" -#include "temp_file.h" +#include "io/columnar_batch.h" +#include "io/csv.h" +#include "io/file.h" +#include "model/schema.h" +#include "model/schema_csv.h" +#include "common/error.h" +#include "common/parsing.h" +#include "testing/temp_file.h" TEST(columnar, csv_to_columnar_and_back) { const TempFile schema_in("schema_in"); @@ -108,7 +109,7 @@ TEST(columnar, metadata_offsets_and_sizes) { } } - const auto file_info = TryGetFileMetadata(columnar_file.Path()); + const auto file_info = GetFileMetadata(columnar_file.Path()); ASSERT_TRUE(file_info.has_value()); EXPECT_TRUE(file_info->is_regular); EXPECT_LT(expected_offset, file_info->size); @@ -195,3 +196,14 @@ TEST(columnar, infer_schema_from_csv_is_convert_compatible) { EXPECT_EQ(ReadRows(data_out.Path()), data_rows); } + +TEST(columnar, schema_rows_with_trailing_empty_columns_are_accepted) { + const TempFile schema_in("schema_trailing_empty"); + const TempFile data_in("data_trailing_empty"); + const TempFile columnar_file("columnar_trailing_empty"); + + WriteRows(schema_in.Path(), {{"id", "int64", ""}, {"name", "string", ""}}); + WriteRows(data_in.Path(), {{"1", "alpha"}, {"2", "beta"}}); + + EXPECT_NO_THROW(ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 2)); +} diff --git a/tests/test_columns.cpp b/tests/test_columns.cpp index b83e667..56a895f 100644 --- a/tests/test_columns.cpp +++ b/tests/test_columns.cpp @@ -2,12 +2,12 @@ #include #include -#include "column.h" -#include "column_int64.h" -#include "column_string.h" -#include "error.h" +#include "model/column.h" +#include "model/column_int64.h" +#include "model/column_string.h" +#include "common/error.h" #include "gtest/gtest.h" -#include "parsing.h" +#include "common/parsing.h" static void ExpectColumnRoundtrip(const ColumnType type, const std::vector& values, const std::vector& expected_values = {}) { diff --git a/tests/test_csv.cpp b/tests/test_csv.cpp index 7e9c4c6..c892da5 100644 --- a/tests/test_csv.cpp +++ b/tests/test_csv.cpp @@ -3,7 +3,7 @@ #include #include -#include "csv.h" +#include "io/csv.h" #include "gtest/gtest.h" static_assert(!std::is_copy_constructible_v); diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index e44a044..e0b9241 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -1,12 +1,13 @@ +#include #include #include -#include "csv.h" -#include "csv_columnar.h" -#include "executor.h" -#include "executor_test_utils.h" +#include "convert/csv_columnar.h" +#include "executor/executor.h" #include "gtest/gtest.h" -#include "temp_file.h" +#include "io/csv.h" +#include "testing/executor_test_utils.h" +#include "testing/temp_file.h" static Batch BuildHitsTable(const std::string_view query) { const TempFile schema_file("executor_schema"); @@ -17,16 +18,20 @@ static Batch BuildHitsTable(const std::string_view query) { {"AdvEngineID", "int64"}, {"ResolutionWidth", "int32"}, {"UserID", "int64"}, + {"RegionID", "int32"}, + {"MobilePhone", "bool"}, + {"MobilePhoneModel", "string"}, + {"SearchEngineID", "int32"}, {"SearchPhrase", "string"}, {"EventDate", "date"}, }); WriteRows(data_file.Path(), { - {"0", "100", "1", "alpha", "2024-01-05"}, - {"7", "200", "2", "beta", "2024-01-03"}, - {"3", "301", "2", "alpha", "2024-01-07"}, - {"7", "500", "5", "gamma", "2024-01-01"}, - {"1", "400", "11", "delta", "2024-01-09"}, + {"0", "100", "1", "10", "false", "", "1", "alpha", "2024-01-05"}, + {"7", "200", "2", "20", "true", "iPhone", "1", "beta", "2024-01-03"}, + {"3", "301", "2", "10", "true", "Pixel", "2", "alpha", "2024-01-07"}, + {"7", "500", "5", "20", "true", "iPhone", "1", "gamma", "2024-01-01"}, + {"1", "400", "11", "30", "false", "Nokia", "2", "delta", "2024-01-09"}, }); ConvertCsvToColumnar(schema_file.Path(), data_file.Path(), columnar_file.Path(), 2); @@ -38,45 +43,183 @@ static Batch BuildHitsTable(const std::string_view query) { if (!result.has_value()) { throw result.error(); } + return std::move(result.value()); } -TEST(executor, executes_basic_aggregate_queries) { - { - const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(*)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"5"})); - } +TEST(executor, usage_example_register_table_and_execute_query) { + const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"5"}); +} + +TEST(executor, groups_rows_and_orders_by_aggregate_descending) { + const Batch batch = BuildHitsTable( + "SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"7", "2"}, + {"1", "1"}, + {"3", "1"}, + }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); + std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, encodes_multicolumn_group_keys_with_length_prefix) { + const TempFile schema_file("executor_schema"); + const TempFile data_file("executor_data"); + const TempFile columnar_file("executor_columnar"); + + WriteRows(schema_file.Path(), { + {"KeyLeft", "string"}, + {"KeyRight", "string"}, + {"Value", "int64"}, + }); + WriteRows(data_file.Path(), { + {"ab", "c", "1"}, + {"a", "bc", "2"}, + {"ab", "c", "3"}, + {"a|", "bc", "4"}, + {"a|", "bc", "5"}, + }); + ConvertCsvToColumnar(schema_file.Path(), data_file.Path(), columnar_file.Path(), 2); + + Executor executor; + executor.RegisterTable("events", columnar_file.Path()); + + auto result = executor.Execute("SELECT KeyLeft, KeyRight, COUNT(*) FROM events GROUP BY KeyLeft, KeyRight;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); + EXPECT_EQ(BatchColumnNames(result.value()), (std::vector{"KeyLeft", "KeyRight", "COUNT(*)"})); + auto actual_rows = BatchRows(result.value()); + auto expected_rows = std::vector>{ + {"a", "bc", "1"}, + {"a|", "bc", "2"}, + {"ab", "c", "2"}, + }; + std::ranges::sort(actual_rows); + std::ranges::sort(expected_rows); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_from_with_as_alias_and_qualified_columns) { + const Batch batch = BuildHitsTable( + "SELECT h.AdvEngineID, COUNT(*) FROM hits AS h WHERE h.AdvEngineID <> 0 GROUP BY h.AdvEngineID ORDER BY " + "COUNT(*) DESC;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"7", "2"}, + {"1", "1"}, + {"3", "1"}, + }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); + std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_from_with_bare_alias) { + const Batch batch = BuildHitsTable( + "SELECT h.AdvEngineID, COUNT(*) FROM hits h WHERE h.AdvEngineID <> 0 GROUP BY h.AdvEngineID ORDER BY " + "COUNT(*) DESC;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"7", "2"}, + {"1", "1"}, + {"3", "1"}, + }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); + std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_column_to_column_filter) { + const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits h WHERE h.AdvEngineID > h.SearchEngineID;"); + + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"3"}); +} + +TEST(executor, supports_select_alias_order_by_alias_and_limit) { + const Batch batch = BuildHitsTable( + "SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 2;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"RegionID", "u"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"10", "2"}, + {"20", "2"}, + }; + std::ranges::sort(actual_rows); + std::ranges::sort(expected_rows); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, supports_order_by_limit_top_k_in_result_order) { + const Batch batch = + BuildHitsTable("SELECT RegionID, COUNT(*) FROM hits GROUP BY RegionID ORDER BY RegionID ASC LIMIT 2;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"RegionID", "COUNT(*)"})); + EXPECT_EQ(BatchRows(batch), (std::vector>{ + {"10", "2"}, + {"20", "2"}, + })); +} + +TEST(executor, supports_multiple_aggregates_with_alias_and_limit) { + const Batch batch = BuildHitsTable( + "SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) " + "FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 2;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"RegionID", "SUM(AdvEngineID)", "c", + "AVG(ResolutionWidth)", "COUNT(DISTINCT UserID)"})); + auto actual_rows = BatchRows(batch); + auto expected_rows = std::vector>{ + {"10", "3", "2", "200", "2"}, + {"20", "14", "2", "350", "2"}, + }; + std::ranges::sort(actual_rows); + std::ranges::sort(expected_rows); + EXPECT_EQ(actual_rows, expected_rows); +} + +TEST(executor, executes_basic_aggregate_queries) { { const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(*)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4"})); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { const Batch batch = BuildHitsTable("SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;"); EXPECT_EQ(BatchColumnNames(batch), (std::vector{"SUM(AdvEngineID)", "COUNT(*)", "AVG(ResolutionWidth)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"18", "5", "300.2"})); + EXPECT_EQ(SingleRowValues(batch), (std::vector{"18", "5", "300"})); } { const Batch batch = BuildHitsTable("SELECT AVG(UserID) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AVG(UserID)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4.2"})); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"AVG(UserID)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { const Batch batch = BuildHitsTable("SELECT COUNT(DISTINCT UserID) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(DISTINCT UserID)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4"})); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(DISTINCT UserID)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { const Batch batch = BuildHitsTable("SELECT COUNT(DISTINCT SearchPhrase) FROM hits;"); - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"COUNT(DISTINCT SearchPhrase)"})); - EXPECT_EQ(SingleRowValues(batch), (std::vector{"4"})); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(DISTINCT SearchPhrase)"}); + EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { diff --git a/tests/test_fileio.cpp b/tests/test_fileio.cpp index c919fcd..d45683e 100644 --- a/tests/test_fileio.cpp +++ b/tests/test_fileio.cpp @@ -3,10 +3,10 @@ #include #include -#include "error.h" -#include "fileio.h" #include "gtest/gtest.h" -#include "temp_file.h" +#include "io/file.h" +#include "common/error.h" +#include "testing/temp_file.h" TEST(fileio, write_and_read_roundtrip) { const TempFile temp("fileio_roundtrip"); @@ -41,7 +41,7 @@ TEST(fileio, metadata_for_file) { WriteFileBytes(temp.Path(), payload); - const auto metadata = TryGetFileMetadata(temp.Path()); + const auto metadata = GetFileMetadata(temp.Path()); ASSERT_TRUE(metadata.has_value()); EXPECT_TRUE(metadata->is_regular); EXPECT_FALSE(metadata->is_directory); @@ -50,7 +50,7 @@ TEST(fileio, metadata_for_file) { TEST(fileio, metadata_for_missing_file) { const auto missing = UniqueTempPath("fileio_missing"); - const auto metadata = TryGetFileMetadata(missing); + const auto metadata = GetFileMetadata(missing); EXPECT_FALSE(metadata.has_value()); } @@ -146,6 +146,17 @@ TEST(fileio, stream_seek_and_tell) { EXPECT_EQ(value, payload[2]); } +TEST(fileio, input_file_reads_at_offsets) { + const TempFile temp("fileio_input_file_read_at"); + const std::vector payload = {10, 11, 12, 13, 14}; + + WriteFileBytes(temp.Path(), payload); + + InputFile file(temp.Path()); + EXPECT_EQ(file.ReadAt(3), payload[3]); + EXPECT_EQ(file.ReadStringAt(1, 3), std::string("\x0B\x0C\x0D", 3)); +} + TEST(fileio, stream_open_missing_throws) { const auto missing = UniqueTempPath("fileio_stream_missing"); diff --git a/tests/test_stream.cpp b/tests/test_stream.cpp index a3617c8..89a7d80 100644 --- a/tests/test_stream.cpp +++ b/tests/test_stream.cpp @@ -3,7 +3,7 @@ #include #include "gtest/gtest.h" -#include "stream.h" +#include "io/stream.h" TEST(stream, roundtrip_signed_integrals) { std::stringstream buffer; diff --git a/tests/test_tokenizer.cpp b/tests/test_tokenizer.cpp index ed9f019..0e9d919 100644 --- a/tests/test_tokenizer.cpp +++ b/tests/test_tokenizer.cpp @@ -3,7 +3,7 @@ #include #include "gtest/gtest.h" -#include "tokenizer.h" +#include "sql_parser/tokenizer.h" static std::vector CollectTypes(const std::vector& tokens) { std::vector types; @@ -30,7 +30,7 @@ TEST(sql_tokenizer, tokenizes_select_query) { Tokens::OpenBracket, Tokens::Asterisk, Tokens::CloseBracket, Tokens::Comma, Tokens::Length, Tokens::OpenBracket, Tokens::NameToken, Tokens::CloseBracket, Tokens::From, Tokens::NameToken, Tokens::Group, Tokens::By, Tokens::NameToken, Tokens::Order, Tokens::By, - Tokens::NameToken, Tokens::Limit, Tokens::NumericLiteral, Tokens::Semicolon, Tokens::EOI, + Tokens::NameToken, Tokens::Limit, Tokens::NumericLiteral, Tokens::Semicolon, Tokens::EndOfInput, })); ASSERT_EQ(tokens.size(), 25U); @@ -63,7 +63,7 @@ TEST(sql_tokenizer, tokenizes_create_with_literals_and_case_insensitive_keywords Tokens::NumericLiteral, Tokens::From, Tokens::NameToken, - Tokens::EOI, + Tokens::EndOfInput, })); ASSERT_EQ(tokens.size(), 12U); @@ -84,7 +84,7 @@ TEST(sql_tokenizer, tokenizer_returns_tokens_one_by_one) { ASSERT_TRUE(token.has_value()); types.push_back(token.value()->GetType()); - if (token.value()->GetType() == Tokens::EOI) { + if (token.value()->GetType() == Tokens::EndOfInput) { break; } } @@ -102,16 +102,16 @@ TEST(sql_tokenizer, tokenizer_returns_tokens_one_by_one) { Tokens::NameToken, Tokens::NotEqual, Tokens::NumericLiteral, - Tokens::EOI, + Tokens::EndOfInput, })); } TEST(sql_tokenizer, rejects_invalid_input) { const auto unterminated = TokenizeSql("SELECT 'oops"); ASSERT_FALSE(unterminated.has_value()); - EXPECT_EQ(unterminated.error().GetCode(), Error::Code::InvalidData); + EXPECT_EQ(unterminated.error().GetCode(), Error::Code::MalformedData); const auto invalid_character = TokenizeSql("SELECT @oops"); ASSERT_FALSE(invalid_character.has_value()); - EXPECT_EQ(invalid_character.error().GetCode(), Error::Code::InvalidData); + EXPECT_EQ(invalid_character.error().GetCode(), Error::Code::MalformedData); } diff --git a/include/tl/expected.hpp b/third-party/tl/expected.hpp similarity index 100% rename from include/tl/expected.hpp rename to third-party/tl/expected.hpp