diff --git a/.github/apt-packages.txt b/.github/apt-packages.txt index cad3f63..1e00ca6 100644 --- a/.github/apt-packages.txt +++ b/.github/apt-packages.txt @@ -1,5 +1,6 @@ ninja-build cmake +liblz4-dev libgtest-dev clang-18 libclang-rt-18-dev diff --git a/README.md b/README.md index 8875e93..7f4084c 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ ## Benchmark Dashboard -`generated 2026-05-31 17:18:21` · `queries 43/43` · `warm runs 4` · `cache drop per-query` · `query mode hardcoded` · [`csv`](benchmarks/results/readme_benchmarks.csv) +`generated 2026-06-04 16:20:50` · `queries 43/43` · `warm runs 4` · `cache drop per-query` · `query mode hardcoded` · [`csv`](benchmarks/results/readme_benchmarks.csv) @@ -25,14 +25,14 @@ - - - - + + + + - +
queries ok43 / 43
queries failed0
median warm time149.50 ms
average warm time190.25 ms
p95 warm time521.35 ms
cold/warm delta17.8%
median warm time105.00 ms
average warm time147.59 ms
p95 warm time389.85 ms
cold/warm delta20.3%
total output size0 B
max output size0 B
fastest queryQ00 · <1 ms
slowest queryQ28 · 940.00 ms
slowest queryQ28 · 873.00 ms
@@ -49,10 +49,10 @@ roundtrip csv size743.0 MB compression ratio6.71x columnar / csv14.9% -csv -> columnar5.71s -columnar -> csv9.46s -convert throughput140.5 MB/s -roundtrip throughput12.6 MB/s +csv -> columnar5.87s +columnar -> csv9.11s +convert throughput136.6 MB/s +roundtrip throughput13.1 MB/s @@ -65,60 +65,60 @@ | slot 1 | slot 2 | slot 3 | slot 4 | slot 5 | slot 6 | | ---: | ---: | ---: | ---: | ---: | ---: | -| Q00 ⬜ `<1` | Q01 🟩 `4.00` | Q02 🟩 `7.00` | Q03 🟩 `6.00` | Q04 🟩 `34.50` | Q05 🟩 `36.50` | -| Q06 ⬜ `<1` | Q07 🟩 `8.00` | Q08 🟥 `189.50` | Q09 🟥 `205.00` | Q10 🟥 `260.50` | Q11 🟥 `271.00` | -| Q12 🟥 `223.00` | Q13 🟥 `252.50` | Q14 🟥 `226.00` | Q15 🟩 `89.50` | Q16 🟦 `143.00` | Q17 🟦 `157.00` | -| Q18 🟥 `516.00` | Q19 ⬜ `<1` | Q20 🟦 `136.50` | Q21 🟦 `156.00` | Q22 🟥 `280.00` | Q23 🟥 `239.00` | -| Q24 🟩 `39.00` | Q25 🟩 `36.50` | Q26 🟩 `39.50` | Q27 🟥 `450.50` | Q28 🟥 `940.00` | Q29 🟩 `5.00` | -| Q30 🟩 `37.50` | Q31 🟩 `52.50` | Q32 🟥 `623.00` | Q33 🟥 `322.00` | Q34 🟥 `323.00` | Q35 🟩 `20.00` | -| Q36 🟥 `358.50` | Q37 🟥 `242.50` | Q38 🟩 `70.50` | Q39 🟥 `462.00` | Q40 🟩 `53.00` | Q41 🟩 `30.00` | -| Q42 🟩 `64.50` | | | | | | +| Q00 ⬜ `<1` | Q01 🟩 `4.00` | Q02 🟩 `6.50` | Q03 🟩 `6.00` | Q04 🟩 `35.50` | Q05 🟩 `25.00` | +| Q06 ⬜ `<1` | Q07 🟩 `7.00` | Q08 🟥 `170.00` | Q09 🟥 `187.50` | Q10 🟥 `258.00` | Q11 🟥 `257.00` | +| Q12 🟥 `210.00` | Q13 🟥 `243.00` | Q14 🟥 `211.50` | Q15 🟩 `54.00` | Q16 🟩 `84.00` | Q17 🟥 `123.00` | +| Q18 🟥 `346.00` | Q19 ⬜ `<1` | Q20 🟦 `98.50` | Q21 🟦 `111.50` | Q22 🟥 `178.00` | Q23 🟥 `156.00` | +| Q24 🟩 `16.00` | Q25 🟩 `12.00` | Q26 🟩 `16.00` | Q27 🟥 `389.50` | Q28 🟥 `873.00` | Q29 🟩 `4.00` | +| Q30 🟩 `24.00` | Q31 🟩 `36.00` | Q32 🟥 `396.50` | Q33 🟥 `195.50` | Q34 🟥 `189.00` | Q35 🟩 `19.50` | +| Q36 🟥 `272.50` | Q37 🟥 `197.50` | Q38 🟩 `46.00` | Q39 🟥 `309.00` | Q40 🟩 `38.50` | Q41 🟩 `27.00` | +| Q42 🟩 `70.00` | | | | | | ### Query Table | Query | Output CSV | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | | [Q00](benchmarks/queries/query_0.sql) | 0 B | <1 | <1 | <1 | <1 | <1 | ok | -| [Q01](benchmarks/queries/query_1.sql) | 0 B | 16.00 | 4.00 | 4.00 | 4.00 | 4.00 | ok | -| [Q02](benchmarks/queries/query_2.sql) | 0 B | 31.00 | 7.00 | 7.00 | 7.00 | 7.00 | ok | -| [Q03](benchmarks/queries/query_3.sql) | 0 B | 24.00 | 6.00 | 6.00 | 6.00 | 6.00 | ok | -| [Q04](benchmarks/queries/query_4.sql) | 0 B | 52.00 | 34.50 | 34.50 | 34.00 | 35.00 | ok | -| [Q05](benchmarks/queries/query_5.sql) | 0 B | 65.00 | 36.75 | 36.50 | 36.00 | 38.00 | ok | +| [Q01](benchmarks/queries/query_1.sql) | 0 B | 17.00 | 4.00 | 4.00 | 4.00 | 4.00 | ok | +| [Q02](benchmarks/queries/query_2.sql) | 0 B | 28.00 | 6.50 | 6.50 | 6.00 | 7.00 | ok | +| [Q03](benchmarks/queries/query_3.sql) | 0 B | 21.00 | 6.00 | 6.00 | 6.00 | 6.00 | ok | +| [Q04](benchmarks/queries/query_4.sql) | 0 B | 50.00 | 35.50 | 35.50 | 35.00 | 36.00 | ok | +| [Q05](benchmarks/queries/query_5.sql) | 0 B | 46.00 | 25.25 | 25.00 | 25.00 | 26.00 | ok | | [Q06](benchmarks/queries/query_6.sql) | 0 B | <1 | <1 | <1 | <1 | <1 | ok | -| [Q07](benchmarks/queries/query_7.sql) | 0 B | 18.00 | 7.75 | 8.00 | 7.00 | 8.00 | ok | -| [Q08](benchmarks/queries/query_8.sql) | 0 B | 213.00 | 189.25 | 189.50 | 186.00 | 192.00 | ok | -| [Q09](benchmarks/queries/query_9.sql) | 0 B | 257.00 | 205.25 | 205.00 | 204.00 | 207.00 | ok | -| [Q10](benchmarks/queries/query_10.sql) | 0 B | 296.00 | 262.25 | 260.50 | 260.00 | 268.00 | ok | -| [Q11](benchmarks/queries/query_11.sql) | 0 B | 306.00 | 269.50 | 271.00 | 262.00 | 274.00 | ok | -| [Q12](benchmarks/queries/query_12.sql) | 0 B | 260.00 | 224.25 | 223.00 | 221.00 | 230.00 | ok | -| [Q13](benchmarks/queries/query_13.sql) | 0 B | 298.00 | 253.00 | 252.50 | 251.00 | 256.00 | ok | -| [Q14](benchmarks/queries/query_14.sql) | 0 B | 257.00 | 226.00 | 226.00 | 225.00 | 227.00 | ok | -| [Q15](benchmarks/queries/query_15.sql) | 0 B | 108.00 | 89.75 | 89.50 | 88.00 | 92.00 | ok | -| [Q16](benchmarks/queries/query_16.sql) | 0 B | 193.00 | 143.00 | 143.00 | 142.00 | 144.00 | ok | -| [Q17](benchmarks/queries/query_17.sql) | 0 B | 216.00 | 157.75 | 157.00 | 154.00 | 163.00 | ok | -| [Q18](benchmarks/queries/query_18.sql) | 0 B | 589.00 | 517.50 | 516.00 | 509.00 | 529.00 | ok | -| [Q19](benchmarks/queries/query_19.sql) | 0 B | 3.00 | <1 | <1 | <1 | <1 | ok | -| [Q20](benchmarks/queries/query_20.sql) | 0 B | 202.00 | 136.25 | 136.50 | 135.00 | 137.00 | ok | -| [Q21](benchmarks/queries/query_21.sql) | 0 B | 245.00 | 156.75 | 156.00 | 156.00 | 159.00 | ok | -| [Q22](benchmarks/queries/query_22.sql) | 0 B | 383.00 | 284.50 | 280.00 | 272.00 | 306.00 | ok | -| [Q23](benchmarks/queries/query_23.sql) | 0 B | 365.00 | 241.25 | 239.00 | 238.00 | 249.00 | ok | -| [Q24](benchmarks/queries/query_24.sql) | 0 B | 101.00 | 38.75 | 39.00 | 38.00 | 39.00 | ok | -| [Q25](benchmarks/queries/query_25.sql) | 0 B | 65.00 | 36.50 | 36.50 | 36.00 | 37.00 | ok | -| [Q26](benchmarks/queries/query_26.sql) | 0 B | 103.00 | 39.75 | 39.50 | 39.00 | 41.00 | ok | -| [Q27](benchmarks/queries/query_27.sql) | 0 B | 534.00 | 454.75 | 450.50 | 447.00 | 471.00 | ok | -| [Q28](benchmarks/queries/query_28.sql) | 0 B | 993.00 | 940.75 | 940.00 | 934.00 | 949.00 | ok | -| [Q29](benchmarks/queries/query_29.sql) | 0 B | 17.00 | 4.75 | 5.00 | 4.00 | 5.00 | ok | -| [Q30](benchmarks/queries/query_30.sql) | 0 B | 99.00 | 37.75 | 37.50 | 37.00 | 39.00 | ok | -| [Q31](benchmarks/queries/query_31.sql) | 0 B | 161.00 | 53.50 | 52.50 | 51.00 | 58.00 | ok | -| [Q32](benchmarks/queries/query_32.sql) | 0 B | 744.00 | 633.25 | 623.00 | 614.00 | 673.00 | ok | -| [Q33](benchmarks/queries/query_33.sql) | 0 B | 381.00 | 323.00 | 322.00 | 319.00 | 329.00 | ok | -| [Q34](benchmarks/queries/query_34.sql) | 0 B | 386.00 | 323.75 | 323.00 | 319.00 | 330.00 | ok | -| [Q35](benchmarks/queries/query_35.sql) | 0 B | 43.00 | 20.00 | 20.00 | 20.00 | 20.00 | ok | -| [Q36](benchmarks/queries/query_36.sql) | 0 B | 414.00 | 358.25 | 358.50 | 354.00 | 362.00 | ok | -| [Q37](benchmarks/queries/query_37.sql) | 0 B | 284.00 | 242.25 | 242.50 | 240.00 | 244.00 | ok | -| [Q38](benchmarks/queries/query_38.sql) | 0 B | 118.00 | 70.25 | 70.50 | 68.00 | 72.00 | ok | -| [Q39](benchmarks/queries/query_39.sql) | 0 B | 528.00 | 461.00 | 462.00 | 455.00 | 465.00 | ok | -| [Q40](benchmarks/queries/query_40.sql) | 0 B | 105.00 | 53.00 | 53.00 | 52.00 | 54.00 | ok | -| [Q41](benchmarks/queries/query_41.sql) | 0 B | 70.00 | 30.00 | 30.00 | 30.00 | 30.00 | ok | -| [Q42](benchmarks/queries/query_42.sql) | 0 B | 95.00 | 64.50 | 64.50 | 64.00 | 65.00 | ok | +| [Q07](benchmarks/queries/query_7.sql) | 0 B | 17.00 | 7.00 | 7.00 | 7.00 | 7.00 | ok | +| [Q08](benchmarks/queries/query_8.sql) | 0 B | 190.00 | 170.25 | 170.00 | 169.00 | 172.00 | ok | +| [Q09](benchmarks/queries/query_9.sql) | 0 B | 231.00 | 188.25 | 187.50 | 186.00 | 192.00 | ok | +| [Q10](benchmarks/queries/query_10.sql) | 0 B | 296.00 | 258.00 | 258.00 | 256.00 | 260.00 | ok | +| [Q11](benchmarks/queries/query_11.sql) | 0 B | 293.00 | 257.50 | 257.00 | 257.00 | 259.00 | ok | +| [Q12](benchmarks/queries/query_12.sql) | 0 B | 236.00 | 209.50 | 210.00 | 206.00 | 212.00 | ok | +| [Q13](benchmarks/queries/query_13.sql) | 0 B | 271.00 | 248.75 | 243.00 | 239.00 | 270.00 | ok | +| [Q14](benchmarks/queries/query_14.sql) | 0 B | 251.00 | 211.25 | 211.50 | 209.00 | 213.00 | ok | +| [Q15](benchmarks/queries/query_15.sql) | 0 B | 70.00 | 54.25 | 54.00 | 52.00 | 57.00 | ok | +| [Q16](benchmarks/queries/query_16.sql) | 0 B | 120.00 | 84.75 | 84.00 | 83.00 | 88.00 | ok | +| [Q17](benchmarks/queries/query_17.sql) | 0 B | 166.00 | 123.00 | 123.00 | 122.00 | 124.00 | ok | +| [Q18](benchmarks/queries/query_18.sql) | 0 B | 398.00 | 347.50 | 346.00 | 334.00 | 364.00 | ok | +| [Q19](benchmarks/queries/query_19.sql) | 0 B | 4.00 | <1 | <1 | <1 | <1 | ok | +| [Q20](benchmarks/queries/query_20.sql) | 0 B | 157.00 | 98.50 | 98.50 | 98.00 | 99.00 | ok | +| [Q21](benchmarks/queries/query_21.sql) | 0 B | 181.00 | 111.25 | 111.50 | 109.00 | 113.00 | ok | +| [Q22](benchmarks/queries/query_22.sql) | 0 B | 273.00 | 178.50 | 178.00 | 176.00 | 182.00 | ok | +| [Q23](benchmarks/queries/query_23.sql) | 0 B | 284.00 | 155.75 | 156.00 | 155.00 | 156.00 | ok | +| [Q24](benchmarks/queries/query_24.sql) | 0 B | 64.00 | 16.00 | 16.00 | 16.00 | 16.00 | ok | +| [Q25](benchmarks/queries/query_25.sql) | 0 B | 29.00 | 12.00 | 12.00 | 12.00 | 12.00 | ok | +| [Q26](benchmarks/queries/query_26.sql) | 0 B | 68.00 | 16.00 | 16.00 | 16.00 | 16.00 | ok | +| [Q27](benchmarks/queries/query_27.sql) | 0 B | 450.00 | 389.00 | 389.50 | 382.00 | 395.00 | ok | +| [Q28](benchmarks/queries/query_28.sql) | 0 B | 929.00 | 875.50 | 873.00 | 865.00 | 891.00 | ok | +| [Q29](benchmarks/queries/query_29.sql) | 0 B | 16.00 | 4.00 | 4.00 | 4.00 | 4.00 | ok | +| [Q30](benchmarks/queries/query_30.sql) | 0 B | 72.00 | 24.00 | 24.00 | 24.00 | 24.00 | ok | +| [Q31](benchmarks/queries/query_31.sql) | 0 B | 131.00 | 36.25 | 36.00 | 36.00 | 37.00 | ok | +| [Q32](benchmarks/queries/query_32.sql) | 0 B | 484.00 | 398.75 | 396.50 | 383.00 | 419.00 | ok | +| [Q33](benchmarks/queries/query_33.sql) | 0 B | 253.00 | 196.75 | 195.50 | 186.00 | 210.00 | ok | +| [Q34](benchmarks/queries/query_34.sql) | 0 B | 248.00 | 188.25 | 189.00 | 184.00 | 191.00 | ok | +| [Q35](benchmarks/queries/query_35.sql) | 0 B | 34.00 | 19.75 | 19.50 | 19.00 | 21.00 | ok | +| [Q36](benchmarks/queries/query_36.sql) | 0 B | 315.00 | 273.25 | 272.50 | 271.00 | 277.00 | ok | +| [Q37](benchmarks/queries/query_37.sql) | 0 B | 235.00 | 197.25 | 197.50 | 195.00 | 199.00 | ok | +| [Q38](benchmarks/queries/query_38.sql) | 0 B | 86.00 | 45.75 | 46.00 | 45.00 | 46.00 | ok | +| [Q39](benchmarks/queries/query_39.sql) | 0 B | 378.00 | 308.75 | 309.00 | 304.00 | 313.00 | ok | +| [Q40](benchmarks/queries/query_40.sql) | 0 B | 76.00 | 38.75 | 38.50 | 38.00 | 40.00 | ok | +| [Q41](benchmarks/queries/query_41.sql) | 0 B | 64.00 | 26.75 | 27.00 | 26.00 | 27.00 | ok | +| [Q42](benchmarks/queries/query_42.sql) | 0 B | 100.00 | 70.25 | 70.00 | 70.00 | 71.00 | ok | diff --git a/benchmarks/benchmark_csv_to_columnar.cpp b/benchmarks/benchmark_csv_to_columnar.cpp index d8e8d82..2e47a17 100644 --- a/benchmarks/benchmark_csv_to_columnar.cpp +++ b/benchmarks/benchmark_csv_to_columnar.cpp @@ -28,7 +28,6 @@ #define COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA "benchmarks/schema_sample_new.csv" #endif -static constexpr size_t RowsPerGroup = 1 << 14; static constexpr auto DefaultCompression = Compression::Lz4; struct CsvCompareResult { @@ -66,6 +65,7 @@ static CsvCompareResult CompareCsvFiles(const std::filesystem::path& lhs_path, c int main(const int argc, char** argv) { try { size_t rows_per_group = RowsPerGroup; + if (argc == 2) { rows_per_group = std::stoull(argv[1]); } diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index 66e29b0..81a4807 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -23,19 +23,51 @@ static constexpr int LastQueryId = 42; enum class QueryRunStatus { Passed, + Executed, Failed, Skipped, Recorded, NoReference, }; +enum class ReferenceCheckMode { + Auto, + Always, + Never, +}; + static std::string_view ModeName(const QueryExecutionMode mode) { return mode == QueryExecutionMode::Parser ? "parser" : "hardcoded"; } +static std::string_view ReferenceCheckModeName(const ReferenceCheckMode mode) { + switch (mode) { + case ReferenceCheckMode::Auto: + return "auto"; + case ReferenceCheckMode::Always: + return "always"; + case ReferenceCheckMode::Never: + return "never"; + } + + return "unknown"; +} + +static bool LooksLikeSampleInput(const std::filesystem::path& data_path, const std::filesystem::path& queries_dir) { + const std::filesystem::path sample_path = queries_dir.parent_path() / "hits_sample.columnar"; + std::error_code error; + + if (std::filesystem::equivalent(data_path, sample_path, error)) { + return true; + } + + return data_path.filename() == sample_path.filename(); +} + QueryRunStatus RunQuery(const Executor& executor, const std::filesystem::path& path, const int query_id, const QueryExecutionMode mode, const PlannedQuery* planned_query, bool record = false, - const std::optional& output_csv = std::nullopt) { + const std::optional& output_csv = std::nullopt, + const bool check_reference = true) { const std::string query_sql = std::filesystem::exists(path) ? ReadTextFile(path) : ""; const bool compare_unordered = QueryUsesGroupBy(query_sql); @@ -69,6 +101,7 @@ QueryRunStatus RunQuery(const Executor& executor, const std::filesystem::path& p } Batch result_batch = std::move(*result); + if (output_csv.has_value()) { WriteBatchCsv(*output_csv, result_batch); } @@ -86,7 +119,16 @@ QueryRunStatus RunQuery(const Executor& executor, const std::filesystem::path& p std::cout << "query_" << query_id << ": rows=" << result_batch.RowsCount() << ", time=" << duration_ms << "ms, status=" << status << std::endl; return QueryRunStatus::Recorded; - } else if (std::filesystem::exists(ref_path)) { + } + + if (!check_reference) { + status = "executed"; + std::cout << "query_" << query_id << ": rows=" << result_batch.RowsCount() << ", time=" << duration_ms + << "ms, status=" << status << std::endl; + return QueryRunStatus::Executed; + } + + if (std::filesystem::exists(ref_path)) { const std::string expected_result = ReadTextFile(ref_path); if (actual_result == expected_result || EqualWithLimitTies(query_sql, result_batch.GetSchema(), actual_result, expected_result) || @@ -123,6 +165,7 @@ int main(const int argc, char** argv) { try { bool record = false; bool strict_status = false; + auto reference_check_mode = ReferenceCheckMode::Auto; auto mode = QueryExecutionMode::Hardcoded; @@ -138,6 +181,10 @@ int main(const int argc, char** argv) { record = true; } else if (arg == "--strict-status") { strict_status = true; + } else if (arg == "--check-reference") { + reference_check_mode = ReferenceCheckMode::Always; + } else if (arg == "--skip-reference-check") { + reference_check_mode = ReferenceCheckMode::Never; } else if (arg == "--parser") { mode = QueryExecutionMode::Parser; } else if (arg == "--hardcoded") { @@ -182,12 +229,17 @@ int main(const int argc, char** argv) { const std::filesystem::path queries_dir = COLUMNAR_BENCHMARK_QUERIES_DIR; const std::filesystem::path data_path = input_path.empty() ? queries_dir.parent_path() / "hits_sample.columnar" : input_path; + const bool check_reference = reference_check_mode == ReferenceCheckMode::Always ? true + : reference_check_mode == ReferenceCheckMode::Never + ? false + : LooksLikeSampleInput(data_path, queries_dir); Executor executor; executor.SetUnsupportedFallbackEnabled(true); executor.RegisterTable("hits", data_path); std::vector> hardcoded_plans(LastQueryId + 1); + if (mode == QueryExecutionMode::Hardcoded) { for (int query_id = FirstQueryId; query_id <= LastQueryId; ++query_id) { if (HasHardcodedClickBenchQuery(query_id)) { @@ -202,6 +254,8 @@ int main(const int argc, char** argv) { std::cout << "queries_dir=" << queries_dir << std::endl; std::cout << "input=" << data_path << std::endl; std::cout << "mode=" << ModeName(mode) << std::endl; + std::cout << "reference_check=" << (check_reference ? "enabled" : "disabled") + << " (mode=" << ReferenceCheckModeName(reference_check_mode) << ")" << std::endl; for (int query_id = from_query; query_id <= to_query; ++query_id) { const std::filesystem::path query_path = queries_dir / ("query_" + std::to_string(query_id) + ".sql"); @@ -212,7 +266,7 @@ int main(const int argc, char** argv) { const PlannedQuery* planned_query = hardcoded_plans[query_id].has_value() ? &*hardcoded_plans[query_id] : nullptr; const QueryRunStatus status = - RunQuery(executor, query_path, query_id, mode, planned_query, record, output_csv); + RunQuery(executor, query_path, query_id, mode, planned_query, record, output_csv, check_reference); if (status == QueryRunStatus::Failed) { saw_failure = true; } diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index ca8927c..d7b34de 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,44 +1,44 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,output_csv_bytes,status,error 0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,0.00,0.00,0.00,0.00,0.00,0,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,16.00,4.00,4.00,4.00,4.00,0,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",31.00,7.00,7.00,7.00,7.00,0,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,24.00,6.00,6.00,6.00,6.00,0,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,52.00,34.50,34.50,34.00,35.00,0,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,65.00,36.75,36.50,36.00,38.00,0,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,17.00,4.00,4.00,4.00,4.00,0,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",28.00,6.50,6.50,6.00,7.00,0,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,21.00,6.00,6.00,6.00,6.00,0,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,50.00,35.50,35.50,35.00,36.00,0,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,46.00,25.25,25.00,25.00,26.00,0,ok, 6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",0.00,0.00,0.00,0.00,0.00,0,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",18.00,7.75,8.00,7.00,8.00,0,ok, -8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",213.00,189.25,189.50,186.00,192.00,0,ok, -9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",257.00,205.25,205.00,204.00,207.00,0,ok, -10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",296.00,262.25,260.50,260.00,268.00,0,ok, -11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",306.00,269.50,271.00,262.00,274.00,0,ok, -12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",260.00,224.25,223.00,221.00,230.00,0,ok, -13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",298.00,253.00,252.50,251.00,256.00,0,ok, -14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",257.00,226.00,226.00,225.00,227.00,0,ok, -15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",108.00,89.75,89.50,88.00,92.00,0,ok, -16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",193.00,143.00,143.00,142.00,144.00,0,ok, -17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",216.00,157.75,157.00,154.00,163.00,0,ok, -18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",589.00,517.50,516.00,509.00,529.00,0,ok, -19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,3.00,0.00,0.00,0.00,0.00,0,ok, -20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,202.00,136.25,136.50,135.00,137.00,0,ok, -21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",245.00,156.75,156.00,156.00,159.00,0,ok, -22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",383.00,284.50,280.00,272.00,306.00,0,ok, -23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,365.00,241.25,239.00,238.00,249.00,0,ok, -24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,101.00,38.75,39.00,38.00,39.00,0,ok, -25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,65.00,36.50,36.50,36.00,37.00,0,ok, -26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",103.00,39.75,39.50,39.00,41.00,0,ok, -27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",534.00,454.75,450.50,447.00,471.00,0,ok, -28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",993.00,940.75,940.00,934.00,949.00,0,ok, -29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",17.00,4.75,5.00,4.00,5.00,0,ok, -30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",99.00,37.75,37.50,37.00,39.00,0,ok, -31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",161.00,53.50,52.50,51.00,58.00,0,ok, -32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",744.00,633.25,623.00,614.00,673.00,0,ok, -33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",381.00,323.00,322.00,319.00,329.00,0,ok, -34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",386.00,323.75,323.00,319.00,330.00,0,ok, -35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",43.00,20.00,20.00,20.00,20.00,0,ok, -36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",414.00,358.25,358.50,354.00,362.00,0,ok, -37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",284.00,242.25,242.50,240.00,244.00,0,ok, -38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",118.00,70.25,70.50,68.00,72.00,0,ok, -39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",528.00,461.00,462.00,455.00,465.00,0,ok, -40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",105.00,53.00,53.00,52.00,54.00,0,ok, -41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",70.00,30.00,30.00,30.00,30.00,0,ok, -42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",95.00,64.50,64.50,64.00,65.00,0,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",17.00,7.00,7.00,7.00,7.00,0,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",190.00,170.25,170.00,169.00,172.00,0,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",231.00,188.25,187.50,186.00,192.00,0,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",296.00,258.00,258.00,256.00,260.00,0,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",293.00,257.50,257.00,257.00,259.00,0,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",236.00,209.50,210.00,206.00,212.00,0,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",271.00,248.75,243.00,239.00,270.00,0,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",251.00,211.25,211.50,209.00,213.00,0,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",70.00,54.25,54.00,52.00,57.00,0,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",120.00,84.75,84.00,83.00,88.00,0,ok, +17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",166.00,123.00,123.00,122.00,124.00,0,ok, +18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",398.00,347.50,346.00,334.00,364.00,0,ok, +19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,4.00,0.00,0.00,0.00,0.00,0,ok, +20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,157.00,98.50,98.50,98.00,99.00,0,ok, +21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",181.00,111.25,111.50,109.00,113.00,0,ok, +22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",273.00,178.50,178.00,176.00,182.00,0,ok, +23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,284.00,155.75,156.00,155.00,156.00,0,ok, +24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,64.00,16.00,16.00,16.00,16.00,0,ok, +25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,29.00,12.00,12.00,12.00,12.00,0,ok, +26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",68.00,16.00,16.00,16.00,16.00,0,ok, +27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",450.00,389.00,389.50,382.00,395.00,0,ok, +28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",929.00,875.50,873.00,865.00,891.00,0,ok, +29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",16.00,4.00,4.00,4.00,4.00,0,ok, +30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",72.00,24.00,24.00,24.00,24.00,0,ok, +31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",131.00,36.25,36.00,36.00,37.00,0,ok, +32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",484.00,398.75,396.50,383.00,419.00,0,ok, +33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",253.00,196.75,195.50,186.00,210.00,0,ok, +34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",248.00,188.25,189.00,184.00,191.00,0,ok, +35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",34.00,19.75,19.50,19.00,21.00,0,ok, +36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",315.00,273.25,272.50,271.00,277.00,0,ok, +37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",235.00,197.25,197.50,195.00,199.00,0,ok, +38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",86.00,45.75,46.00,45.00,46.00,0,ok, +39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",378.00,308.75,309.00,304.00,313.00,0,ok, +40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",76.00,38.75,38.50,38.00,40.00,0,ok, +41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",64.00,26.75,27.00,26.00,27.00,0,ok, +42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",100.00,70.25,70.00,70.00,71.00,0,ok, diff --git a/benchmarks/support/query_result_checks.cpp b/benchmarks/support/query_result_checks.cpp index 78549a1..aac0539 100644 --- a/benchmarks/support/query_result_checks.cpp +++ b/benchmarks/support/query_result_checks.cpp @@ -107,12 +107,14 @@ static std::optional ParseSingleOrderBy(const std::string_view sq std::string item = TrimCopy(sql.substr(order_pos + std::string_view("ORDER BY").size(), limit_pos - order_pos - std::string_view("ORDER BY").size())); + if (const size_t comma = item.find(','); comma != std::string::npos) { item = TrimCopy(std::string_view(item).substr(0, comma)); } bool descending = false; const std::string upper_item = ToUpperAscii(item); + if (upper_item.ends_with(" DESC")) { descending = true; item = TrimCopy(std::string_view(item).substr(0, item.size() - std::string_view(" DESC").size())); @@ -136,16 +138,19 @@ static std::optional FindOutputColumn(const Schema& schema, const std::s return i; } } + return std::nullopt; } static std::string EncodeRowForCompare(const std::vector& row) { std::string encoded; + for (const auto& value : row) { encoded += std::to_string(value.size()); encoded.push_back(':'); encoded += value; } + return encoded; } @@ -153,6 +158,7 @@ static bool EqualRowMultisets(const std::vector>& lhs, const std::vector>& rhs) { std::vector lhs_encoded; std::vector rhs_encoded; + lhs_encoded.reserve(lhs.size()); rhs_encoded.reserve(rhs.size()); @@ -182,6 +188,7 @@ static std::optional RawFieldFromRight(const std::string& row, cons const size_t comma = row.rfind(',', end == 0 ? 0 : end - 1); const size_t begin = comma == std::string::npos ? 0 : comma + 1; + return row.substr(begin, end - begin); } @@ -195,12 +202,14 @@ static bool EqualWithLimitTiesRawFallback(const Schema& schema, const ParsedOrde const std::string_view expected) { const std::vector actual_rows = SplitRows(actual); const std::vector expected_rows = SplitRows(expected); + if (actual_rows.size() != expected_rows.size() || expected_rows.empty() || order_column >= schema.columns.size()) { return false; } const size_t offset_from_right = schema.columns.size() - 1 - order_column; const std::optional cutoff = RawFieldFromRight(expected_rows.back(), offset_from_right); + if (!cutoff.has_value()) { return false; } @@ -249,11 +258,13 @@ static bool EqualWithLimitTiesRawFallback(const Schema& schema, const ParsedOrde bool EqualWithLimitTies(const std::string_view sql, const Schema& schema, const std::string_view actual, const std::string_view expected) { const std::optional order_by = ParseSingleOrderBy(sql); + if (!order_by.has_value()) { return false; } const std::optional order_column = FindOutputColumn(schema, order_by->output_name); + if (!order_column.has_value()) { return false; } diff --git a/benchmarks/support/query_result_checks.h b/benchmarks/support/query_result_checks.h index 82409e7..bfb2a78 100644 --- a/benchmarks/support/query_result_checks.h +++ b/benchmarks/support/query_result_checks.h @@ -6,6 +6,8 @@ #include "model/batch.h" std::string BatchToString(const Batch& batch); + bool QueryUsesGroupBy(std::string_view sql); + bool EqualAsRowMultisets(std::string_view lhs, std::string_view rhs); bool EqualWithLimitTies(std::string_view sql, const Schema& schema, std::string_view actual, std::string_view expected); diff --git a/benchmarks/support/query_runners.cpp b/benchmarks/support/query_runners.cpp index 61dbbb8..b08e40d 100644 --- a/benchmarks/support/query_runners.cpp +++ b/benchmarks/support/query_runners.cpp @@ -157,6 +157,7 @@ static SelectItemSpec Agg(std::string function, ExprPtr argument, const bool dis aggregate.argument_kind = argument->kind == ExprKind::Star ? AggArgumentKind::Star : AggArgumentKind::Column; const std::string argument_name = argument->output_name; + if (aggregate.argument_kind == AggArgumentKind::Column) { aggregate.argument = std::move(argument); if (aggregate.argument->kind == ExprKind::Column) { diff --git a/benchmarks/support/query_runners_fast.cpp b/benchmarks/support/query_runners_fast.cpp index 866252c..0d2f965 100644 --- a/benchmarks/support/query_runners_fast.cpp +++ b/benchmarks/support/query_runners_fast.cpp @@ -134,6 +134,66 @@ static std::vector OrderGroupsByCount(const Ma return ordered; } +constexpr uint64_t max_group_reserve = 1 << 27; + +static uint64_t TotalRowCount(const ColumnarBatchReader& reader) { + uint64_t rows = 0; + + for (const auto& row_group : reader.GetMetadata().row_groups) { + rows += row_group.row_count; + } + + return rows; +} + +namespace detail { + +struct OrderRow { + Int128 event_time = 0; + std::string phrase; + size_t ordinal = 0; +}; + +struct OrderRowView { + Int128 event_time = 0; + std::string_view phrase; + size_t ordinal = 0; +}; + +inline OrderRowView ViewOf(const OrderRow& row) { return {row.event_time, row.phrase, row.ordinal}; } + +} // namespace detail + +using detail::OrderRow; +using detail::OrderRowView; + +template +static void OfferTopK(std::vector& heap, const size_t limit, const Better& better, const Int128 event_time, + const std::string_view phrase, const size_t ordinal) { + const auto worst_on_top = [&](const OrderRow& lhs, const OrderRow& rhs) { + return better(detail::ViewOf(lhs), detail::ViewOf(rhs)); + }; + + if (heap.size() < limit) { + heap.push_back(OrderRow{event_time, std::string(phrase), ordinal}); + std::push_heap(heap.begin(), heap.end(), worst_on_top); + return; + } + + if (better(OrderRowView{event_time, phrase, ordinal}, detail::ViewOf(heap.front()))) { + std::pop_heap(heap.begin(), heap.end(), worst_on_top); + heap.back() = OrderRow{event_time, std::string(phrase), ordinal}; + std::push_heap(heap.begin(), heap.end(), worst_on_top); + } +} + +template +static void SortTopK(std::vector& heap, const Better& better) { + std::ranges::sort(heap, [&](const OrderRow& lhs, const OrderRow& rhs) { + return better(detail::ViewOf(lhs), detail::ViewOf(rhs)); + }); +} + static ExecuteExpected CountAll(const std::filesystem::path& path) { const ColumnarBatchReader reader(path); uint64_t rows = 0; @@ -336,21 +396,25 @@ static ExecuteExpected SearchPhraseGroupByTitleGoogle(const std::filesystem::pat for (size_t row = 0; row < batch->RowsCount(); ++row) { const std::string_view title_value = arena.Store(title.ValueAsString(row)); + if (!Contains(title_value, "Google")) { continue; } const std::string_view url_value = arena.Store(url.ValueAsString(row)); + if (Contains(url_value, ".google.")) { continue; } const std::string_view key = arena.Store(search_phrase.ValueAsString(row)); + if (key.empty()) { continue; } auto& group = groups[key]; + if (group.count == 0) { group.key = key; group.ordinal = ordinal++; @@ -430,6 +494,10 @@ static ExecuteExpected GroupByIntPairWithMetrics(const std::filesystem::path& pa std::unordered_map, IntGroupStats, Int128PairHash> groups; size_t ordinal = 0; + if (!filter_search_phrase) { + groups.reserve(std::min(TotalRowCount(reader), max_group_reserve)); + } + while (const auto batch = scan->Next()) { const Column& key1 = batch->ColumnAt(0); const Column& key2 = batch->ColumnAt(1); @@ -443,8 +511,8 @@ static ExecuteExpected GroupByIntPairWithMetrics(const std::filesystem::path& pa } const std::pair key{key1.ValueAsInt128(row), key2.ValueAsInt128(row)}; - auto& group = groups[key]; + if (group.count == 0) { group.key1 = key.first; group.key2 = key.second; @@ -484,10 +552,14 @@ static ExecuteExpected GroupByURLCount(const std::filesystem::path& path, const std::unordered_map groups; size_t ordinal = 0; + groups.reserve(std::min(TotalRowCount(reader), max_group_reserve)); + + std::string scratch; + while (const auto batch = scan->Next()) { const Column& url = batch->ColumnAt(0); for (size_t row = 0; row < batch->RowsCount(); ++row) { - const std::string_view key = arena.Store(url.ValueAsString(row)); + const std::string_view key = arena.Store(url.ValueAsStringView(row, scratch)); auto& group = groups[key]; if (group.count == 0) { group.key = key; @@ -525,6 +597,8 @@ static ExecuteExpected GroupByClientIPDerived(const std::filesystem::path& path) std::unordered_map groups; size_t ordinal = 0; + groups.reserve(std::min(TotalRowCount(reader), max_group_reserve)); + while (const auto batch = scan->Next()) { const Column& client_ip = batch->ColumnAt(0); for (size_t row = 0; row < batch->RowsCount(); ++row) { @@ -557,12 +631,21 @@ static ExecuteExpected GroupByClientIPDerived(const std::filesystem::path& path) return result; } -static std::vector CollectSearchPhraseOrderRows(const std::filesystem::path& path) { +constexpr size_t order_by_limit = 10; + +static ExecuteExpected SelectSearchPhraseOrderByEventTime(const std::filesystem::path& path) { const ColumnarBatchReader reader(path); const auto scan = ScanColumns(path, reader.GetSchema(), {"EventTime", "SearchPhrase"}); - StringArena arena; - std::vector rows; + const auto better = [](const OrderRowView& lhs, const OrderRowView& rhs) { + if (lhs.event_time != rhs.event_time) { + return lhs.event_time < rhs.event_time; + } + return lhs.ordinal < rhs.ordinal; + }; + + std::vector heap; + std::string scratch; size_t ordinal = 0; while (const auto batch = scan->Next()) { @@ -570,38 +653,24 @@ static std::vector CollectSearchPhraseOrderRows(const std: const Column& search_phrase = batch->ColumnAt(1); for (size_t row = 0; row < batch->RowsCount(); ++row) { - const std::string_view phrase = arena.Store(search_phrase.ValueAsString(row)); + const std::string_view phrase = search_phrase.ValueAsStringView(row, scratch); + if (phrase.empty()) { continue; } - rows.push_back(SearchPhraseOrderRow{ - .event_time = event_time.ValueAsInt128(row), .search_phrase = phrase, .ordinal = ordinal++}); - } - } - return rows; -} - -static ExecuteExpected SelectSearchPhraseOrderByEventTime(const std::filesystem::path& path) { - auto rows = CollectSearchPhraseOrderRows(path); - - std::ranges::sort(rows, [](const SearchPhraseOrderRow& lhs, const SearchPhraseOrderRow& rhs) { - if (lhs.event_time != rhs.event_time) { - return lhs.event_time < rhs.event_time; + OfferTopK(heap, order_by_limit, better, event_time.ValueAsInt128(row), phrase, ordinal++); } - return lhs.ordinal < rhs.ordinal; - }); - - if (rows.size() > 10) { - rows.resize(10); } + SortTopK(heap, better); + Batch result( Schema{{ColumnSchema("SearchPhrase", ColumnType::String), ColumnSchema("EventTime", ColumnType::Timestamp)}}, - rows.size()); + heap.size()); - for (const auto& row : rows) { - result.AppendValueFromString(0, row.search_phrase); + for (const auto& row : heap) { + result.AppendValueFromString(0, row.phrase); result.AppendValueFromString(1, TimestampToString(static_cast(row.event_time))); } @@ -612,57 +681,79 @@ static ExecuteExpected SelectSearchPhraseOrderByPhrase(const std::filesystem::pa const ColumnarBatchReader reader(path); const auto scan = ScanColumns(path, reader.GetSchema(), {"SearchPhrase"}); - StringArena arena; - std::vector rows; + const auto better = [](const OrderRowView& lhs, const OrderRowView& rhs) { + if (lhs.phrase != rhs.phrase) { + return lhs.phrase < rhs.phrase; + } + return lhs.ordinal < rhs.ordinal; + }; + + std::vector heap; + std::string scratch; + size_t ordinal = 0; while (const auto batch = scan->Next()) { const Column& search_phrase = batch->ColumnAt(0); for (size_t row = 0; row < batch->RowsCount(); ++row) { - const std::string_view phrase = arena.Store(search_phrase.ValueAsString(row)); + const std::string_view phrase = search_phrase.ValueAsStringView(row, scratch); if (phrase.empty()) { continue; } - rows.push_back(phrase); + OfferTopK(heap, order_by_limit, better, 0, phrase, ordinal++); } } - std::ranges::sort(rows); + SortTopK(heap, better); - if (rows.size() > 10) { - rows.resize(10); - } - - Batch result(Schema{{ColumnSchema("SearchPhrase", ColumnType::String)}}, rows.size()); - for (const std::string_view row : rows) { - result.AppendValueFromString(0, row); + Batch result(Schema{{ColumnSchema("SearchPhrase", ColumnType::String)}}, heap.size()); + for (const auto& row : heap) { + result.AppendValueFromString(0, row.phrase); } return result; } static ExecuteExpected SelectSearchPhraseOrderByEventTimeThenPhrase(const std::filesystem::path& path) { - auto rows = CollectSearchPhraseOrderRows(path); + const ColumnarBatchReader reader(path); + const auto scan = ScanColumns(path, reader.GetSchema(), {"EventTime", "SearchPhrase"}); - std::ranges::sort(rows, [](const SearchPhraseOrderRow& lhs, const SearchPhraseOrderRow& rhs) { + const auto better = [](const OrderRowView& lhs, const OrderRowView& rhs) { if (lhs.event_time != rhs.event_time) { return lhs.event_time < rhs.event_time; } - if (lhs.search_phrase != rhs.search_phrase) { - return lhs.search_phrase < rhs.search_phrase; + if (lhs.phrase != rhs.phrase) { + return lhs.phrase < rhs.phrase; } return lhs.ordinal < rhs.ordinal; - }); + }; + + std::vector heap; + std::string scratch; + size_t ordinal = 0; + + while (const auto batch = scan->Next()) { + const Column& event_time = batch->ColumnAt(0); + const Column& search_phrase = batch->ColumnAt(1); - if (rows.size() > 10) { - rows.resize(10); + for (size_t row = 0; row < batch->RowsCount(); ++row) { + const std::string_view phrase = search_phrase.ValueAsStringView(row, scratch); + + if (phrase.empty()) { + continue; + } + + OfferTopK(heap, order_by_limit, better, event_time.ValueAsInt128(row), phrase, ordinal++); + } } + SortTopK(heap, better); + Batch result( Schema{{ColumnSchema("SearchPhrase", ColumnType::String), ColumnSchema("EventTime", ColumnType::Timestamp)}}, - rows.size()); + heap.size()); - for (const auto& row : rows) { - result.AppendValueFromString(0, row.search_phrase); + for (const auto& row : heap) { + result.AppendValueFromString(0, row.phrase); result.AppendValueFromString(1, TimestampToString(static_cast(row.event_time))); } diff --git a/include/convert/csv_columnar.h b/include/convert/csv_columnar.h index a069e1e..9e63bb5 100644 --- a/include/convert/csv_columnar.h +++ b/include/convert/csv_columnar.h @@ -5,10 +5,10 @@ #include "io/compression.h" -inline constexpr size_t DefaultMaxRowsPerGroup = 1 << 14; +inline constexpr size_t RowsPerGroup = 1 << 17; void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, - const std::filesystem::path& output_path, size_t max_rows_per_group = DefaultMaxRowsPerGroup, + const std::filesystem::path& output_path, size_t max_rows_per_group = RowsPerGroup, Compression compression = Compression::None); void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, const std::filesystem::path& data_path); diff --git a/include/executor/typed_value_utils.h b/include/executor/typed_value_utils.h index 95aa863..78fb20f 100644 --- a/include/executor/typed_value_utils.h +++ b/include/executor/typed_value_utils.h @@ -8,7 +8,9 @@ #include "model/schema.h" bool IsNumericColumnType(ColumnType type); + Int128 ParseColumnValueAsInt128(ColumnType type, std::string_view value); std::optional TryParseLiteralValueAsInt128(const QueryLiteral& literal, ColumnType type); std::string FormatInt128Value(ColumnType type, Int128 value); + ComparisonKind FlipComparison(ComparisonKind comparison); diff --git a/include/io/columnar_batch.h b/include/io/columnar_batch.h index ae8c0b8..ee792b6 100644 --- a/include/io/columnar_batch.h +++ b/include/io/columnar_batch.h @@ -32,6 +32,9 @@ class ColumnarBatchReader final : public BatchReader { ColumnarMetadata metadata_; size_t next_group_ = 0; + + std::vector read_buffer_; + std::vector decompression_buffer_; }; class ColumnarBatchWriter final : public BatchWriter { @@ -63,4 +66,5 @@ class ColumnarBatchWriter final : public BatchWriter { std::vector ReadColumnChunk(const std::filesystem::path& path, InputFile& input, const ColumnChunkMetadata& chunk); void ReadBatchColumnChunk(const std::filesystem::path& path, InputFile& input, const ColumnChunkMetadata& chunk, - uint32_t row_count, const Batch& batch, size_t column_index); + uint32_t row_count, const Batch& batch, size_t column_index, + std::vector& read_buffer, std::vector& decompression_buffer); diff --git a/include/io/compression.h b/include/io/compression.h index 4f9d0f1..6dc5a71 100644 --- a/include/io/compression.h +++ b/include/io/compression.h @@ -15,3 +15,5 @@ Compression CompressionFromName(std::string_view name); std::vector Compress(std::span input, Compression compression); std::vector Decompress(std::span input, Compression compression, uint64_t uncompressed_size); +void DecompressInto(std::span input, Compression compression, uint64_t uncompressed_size, + std::vector& output); diff --git a/include/model/batch.h b/include/model/batch.h index 606dfae..282c2e3 100644 --- a/include/model/batch.h +++ b/include/model/batch.h @@ -34,7 +34,7 @@ class Batch { void AppendColumnSelected(size_t column_index, const Column& source, std::span rows) const; void AppendRowsRangeFromBatch(const Batch& source, size_t begin, size_t count) const; void AppendRowsSelectedFromBatch(const Batch& source, std::span rows) const; - void ReadColumnFrom(size_t column_index, std::istream& in, uint32_t row_count, uint64_t size) const; + void ReadColumnFrom(size_t column_index, std::span data, uint32_t row_count, uint64_t size) const; const Column& ColumnAt(size_t i) const; diff --git a/include/model/column.h b/include/model/column.h index a1e86d4..9d248c1 100644 --- a/include/model/column.h +++ b/include/model/column.h @@ -35,6 +35,7 @@ class Column { virtual size_t Size() const = 0; virtual std::string ValueAsString(size_t row) const = 0; + virtual std::string_view ValueAsStringView(size_t row, std::string& scratch) const; virtual Int128 ValueAsInt128(size_t row) const; virtual void SelectRowsByInt128Comparison(Int128 rhs, ValueComparison comparison, std::vector& rows) const; @@ -66,7 +67,7 @@ class MutableColumn : public Column { virtual void AppendRangeFromColumn(const Column& source, size_t begin, size_t count) = 0; virtual void AppendSelectedFromColumn(const Column& source, std::span rows) = 0; - virtual void ReadFrom(std::istream& in, uint32_t row_count, uint64_t size) = 0; + virtual void ReadFrom(std::span data, uint32_t row_count, uint64_t size) = 0; virtual std::unique_ptr CloneMutable() const = 0; }; diff --git a/include/model/column_string.h b/include/model/column_string.h index 209569e..8b02f42 100644 --- a/include/model/column_string.h +++ b/include/model/column_string.h @@ -28,6 +28,7 @@ class StringColumn final : public MutableColumn { void AppendRangeFromColumn(const Column& source, size_t begin, size_t count) override; void AppendSelectedFromColumn(const Column& source, std::span rows) override; std::string ValueAsString(size_t row) const override; + std::string_view ValueAsStringView(size_t row, std::string& scratch) const override; void SelectRowsByStringSet(const std::unordered_set& values, std::vector& rows) const override; void SelectRowsByLikePattern(std::string_view pattern, bool negated, std::vector& rows) const override; void AppendEncodedValue(size_t row, std::string& out) const override; @@ -36,8 +37,13 @@ class StringColumn final : public MutableColumn { std::unique_ptr CloneMutable() const override; void WriteTo(std::ostream& out) const override; - void ReadFrom(std::istream& in, uint32_t row_count, uint64_t size) override; + void ReadFrom(std::span data, uint32_t row_count, uint64_t size) override; private: - std::vector values_; + std::string_view StringAt(size_t row) const; + void AppendValue(std::string_view value); + void CheckAppendSize(size_t value_size) const; + + std::vector blob_; + std::vector offsets_; }; diff --git a/include/model/fixed_column.h b/include/model/fixed_column.h index 40f3581..e29fce6 100644 --- a/include/model/fixed_column.h +++ b/include/model/fixed_column.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -30,7 +31,9 @@ class FixedColumn : public MutableColumn { if (source.Type() != TypeValue) { throw Error::InconsistentData(ColumnImpl::ModuleName(), "column type mismatch"); } + const auto& typed_source = static_cast(source); + AppendValue(typed_source.ValueAt(row)); } @@ -38,10 +41,21 @@ class FixedColumn : public MutableColumn { if (source.Type() != TypeValue) { throw Error::InconsistentData(ColumnImpl::ModuleName(), "column type mismatch"); } + const auto& typed_source = static_cast(source); + if (begin > typed_source.values_.size() || count > typed_source.values_.size() - begin) { throw Error::OutOfRange(ColumnImpl::ModuleName(), "row range out of range"); } + + if (&typed_source == this) { + const std::vector slice(values_.begin() + static_cast(begin), + values_.begin() + static_cast(begin + count)); + values_.insert(values_.end(), slice.begin(), slice.end()); + + return; + } + values_.insert(values_.end(), typed_source.values_.begin() + static_cast(begin), typed_source.values_.begin() + static_cast(begin + count)); } @@ -50,8 +64,10 @@ class FixedColumn : public MutableColumn { if (source.Type() != TypeValue) { throw Error::InconsistentData(ColumnImpl::ModuleName(), "column type mismatch"); } + const auto& typed_source = static_cast(source); values_.reserve(values_.size() + rows.size()); + for (const size_t row : rows) { values_.push_back(typed_source.ValueAt(row)); } @@ -89,13 +105,18 @@ class FixedColumn : public MutableColumn { WriteBytes(out, {reinterpret_cast(values_.data()), values_.size() * sizeof(T)}); } - void ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) override { + void ReadFrom(const std::span data, const uint32_t row_count, const uint64_t size) override { const uint64_t expected = static_cast(row_count) * sizeof(T); - if (size != expected) { + + if (size != expected || data.size() != expected) { throw Error::InconsistentData(ColumnImpl::ModuleName(), "column chunk size mismatch"); } + values_.resize(row_count); - ReadBytes(in, reinterpret_cast(values_.data()), values_.size() * sizeof(T)); + + if (!values_.empty()) { + std::memcpy(values_.data(), data.data(), values_.size() * sizeof(T)); + } } protected: diff --git a/include/testing/temp_file.h b/include/testing/temp_file.h index 2eb9307..d4c85d1 100644 --- a/include/testing/temp_file.h +++ b/include/testing/temp_file.h @@ -29,6 +29,7 @@ class TempFile { path_ = std::move(other.path_); other.path_.clear(); } + return *this; } ~TempFile() { Remove(); } @@ -40,6 +41,7 @@ class TempFile { if (path_.empty()) { return; } + std::error_code ec; std::filesystem::remove(path_, ec); } diff --git a/script/setup.sh b/script/setup.sh index 087e96a..9887312 100755 --- a/script/setup.sh +++ b/script/setup.sh @@ -7,6 +7,7 @@ packages=( ninja-build git ca-certificates + liblz4-dev ) if ! command -v apt-get >/dev/null 2>&1; then diff --git a/scripts/action.py b/scripts/action.py index a6b61a5..55bf588 100644 --- a/scripts/action.py +++ b/scripts/action.py @@ -1,13 +1,5 @@ #!/usr/bin/env python3 -""" -action.py — starts Docker (if not running) and then runs act -for a GitHub Actions workflow. -Examples: - python3 scripts/action.py - python3 scripts/action.py --workflow .github/workflows/cmake-single-platform.yml - python3 scripts/action.py -- --verbose -""" from __future__ import annotations @@ -19,8 +11,7 @@ import sys import time from pathlib import Path -from typing import List, Optional - +from typing import List DEFAULT_WORKFLOW = ".github/workflows/cmake-single-platform.yml" DEFAULT_PLATFORM_MAP = "ubuntu-latest=ghcr.io/catthehacker/ubuntu:act-latest" @@ -41,7 +32,6 @@ APT_PACKAGES_FILE = ".github/apt-packages.txt" - ERR_DOCKER_NOT_FOUND = ( "Docker executable not found in PATH. Install Docker Desktop (macOS/Windows) " "or Docker Engine (Linux)." @@ -75,7 +65,7 @@ def run( - cmd: List[str], check: bool = False, capture: bool = False + cmd: List[str], check: bool = False, capture: bool = False ) -> subprocess.CompletedProcess: return subprocess.run( cmd, @@ -102,13 +92,16 @@ def detect_repo_root() -> Path: return Path(p.stdout.strip()).resolve() print(INFO_AUTO_DETECT_REPO_ROOT_FAILED, flush=True) + return Path(__file__).resolve().parent.parent def docker_daemon_ready() -> bool: if not have_exe(DOCKER_EXE): return False + p = run([DOCKER_EXE, "info"], check=False, capture=True) + return p.returncode == 0 @@ -138,7 +131,7 @@ def start_docker_linux() -> None: def ensure_docker_running( - timeout_seconds: int = DEFAULT_DOCKER_TIMEOUT_SECONDS, + timeout_seconds: int = DEFAULT_DOCKER_TIMEOUT_SECONDS, ) -> None: if not have_exe(DOCKER_EXE): raise RuntimeError(ERR_DOCKER_NOT_FOUND) @@ -185,6 +178,7 @@ def run_act(workflow: str, arch: str, platform_map: str, extra_args: List[str]) print(f"\n{INFO_RUNNING_ACT} {' '.join(cmd)}\n", flush=True) p = subprocess.run(cmd) + return p.returncode @@ -221,6 +215,7 @@ def build_arg_parser() -> argparse.ArgumentParser: nargs=argparse.REMAINDER, help="Extra args passed to act (prefix with --), e.g. -- -v", ) + return parser diff --git a/scripts/dashboard.py b/scripts/dashboard.py index 94efe57..5813e4f 100644 --- a/scripts/dashboard.py +++ b/scripts/dashboard.py @@ -118,6 +118,7 @@ def parse_args() -> argparse.Namespace: "--cache-drop-command", help="Override the OS-specific cache drop command, for example: 'sudo purge'.", ) + return parser.parse_args() @@ -146,8 +147,10 @@ def render_ms(value: float) -> str: def render_ms_text(value: str) -> str: parsed = parse_ms(value) + if parsed is None: return value or "—" + return render_ms(parsed) @@ -179,8 +182,10 @@ def format_size(num_bytes: int) -> str: def compact_sql(sql: str, max_len: int = 88) -> str: normalized = " ".join(sql.split()) + if len(normalized) <= max_len: return normalized + return normalized[: max_len - 1] + "…" @@ -204,11 +209,13 @@ def render_html_metric_table(title: str, rows: list[tuple[str, str]]) -> list[st 'MetricValue', "", ] + for metric, value in rows: lines.append( f"{html_escape(str(metric))}{html_escape(str(value))}" ) lines.extend(["", ""]) + return lines @@ -235,6 +242,7 @@ def percentile(values: list[float], ratio: float) -> float: lower_value = ordered[lower_index] upper_value = ordered[upper_index] fraction = position - lower_index + return lower_value + (upper_value - lower_value) * fraction @@ -329,6 +337,7 @@ def collect_conversion_stats( def default_cache_drop_command() -> list[str] | None: system = platform.system() + if system == "Darwin": purge = shutil.which("purge") return [purge] if purge else None @@ -374,6 +383,7 @@ def run_cache_drop_command(command: list[str]) -> None: def drop_file_system_cache(cache_drop_command: str | None) -> None: command = shlex.split(cache_drop_command) if cache_drop_command else default_cache_drop_command() + if command is None: raise RuntimeError( "cache drop is not supported on this OS; pass --cache-drop-mode none or --cache-drop-command" @@ -619,7 +629,7 @@ def collect_hardcoded_rows( row["status"] = "failed" row["error"] = "missing benchmark result" continue - if result["status"] != "passed": + if result["status"] not in {"passed", "executed"}: row["status"] = "failed" row["error"] = f"benchmark status={result['status']}" continue @@ -628,6 +638,7 @@ def collect_hardcoded_rows( warm_values = row["warm_values"] assert isinstance(warm_values, list) + for _ in range(warm_runs): try: warm_run_result = run_hardcoded_benchmark(benchmark_binary, input_path, query_id, query_id) @@ -641,7 +652,7 @@ def collect_hardcoded_rows( row["status"] = "failed" row["error"] = "missing benchmark result" break - if result["status"] != "passed": + if result["status"] not in {"passed", "executed"}: row["status"] = "failed" row["error"] = f"benchmark status={result['status']}" break @@ -694,6 +705,7 @@ def query_metric(row: dict[str, str]) -> float: return warm_median first_run = parse_ms(row["first_run_ms"]) + return first_run or 0.0 @@ -756,7 +768,8 @@ def render_markdown( median_value = statistics.median(warm_values) if warm_values else 0.0 avg_value = statistics.fmean(warm_values) if warm_values else 0.0 p95_value = percentile(warm_values, 0.95) if warm_values else 0.0 - cold_penalty = ((statistics.fmean(first_run_values) / avg_value) - 1.0) * 100.0 if avg_value and first_run_values else 0.0 + cold_penalty = ((statistics.fmean( + first_run_values) / avg_value) - 1.0) * 100.0 if avg_value and first_run_values else 0.0 fastest = min(ok_rows, key=query_metric) if ok_rows else None slowest = max(ok_rows, key=query_metric) if ok_rows else None summary_rows = [ @@ -974,7 +987,9 @@ def main() -> int: conversion_stats, ) update_readme(readme_path, markdown_block) + print(f"updated {display_path(readme_path, root)} from {display_path(csv_path, root)} with {len(rows)} row(s)") + return 0 diff --git a/src/app/main.cpp b/src/app/main.cpp index 75ef337..fe98547 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -96,6 +96,7 @@ int RunQuery(const argparse::ArgumentParser& command) { std::filesystem::path(command.get("--input"))); const ExecuteExpected result = executor.Execute(query); + if (!result.has_value()) { throw result.error(); } diff --git a/src/common/parsing.cpp b/src/common/parsing.cpp index 4193529..9dc0ea1 100644 --- a/src/common/parsing.cpp +++ b/src/common/parsing.cpp @@ -64,6 +64,7 @@ T ParseInteger(const std::string_view value, const std::string_view type_name) { } T result = 0; + if (!TryParseInteger(value, result)) { throw Error::MalformedData("common", "invalid " + std::string(type_name) + " value"); } @@ -76,6 +77,7 @@ static bool TryParseUnsignedPart(const std::string_view input, const size_t offs const char* begin = input.data() + offset; const char* end = begin + length; const auto [ptr, ec] = std::from_chars(begin, end, value); + return ec == std::errc() && ptr == end; } @@ -112,9 +114,11 @@ static bool TryParseDatePoint(const std::string_view value, std::chrono::sys_day static std::chrono::sys_days ParseDatePoint(const std::string_view value, const std::string_view type_name) { std::chrono::sys_days result{}; + if (!TryParseDatePoint(value, result)) { throw Error::MalformedData("common", "invalid " + std::string(type_name) + " value"); } + return result; } @@ -122,33 +126,40 @@ static std::string FormatDateParts(const std::chrono::year_month_day ymd) { std::ostringstream out; out << std::setfill('0') << std::setw(4) << static_cast(ymd.year()) << '-' << std::setw(2) << static_cast(ymd.month()) << '-' << std::setw(2) << static_cast(ymd.day()); + return out.str(); } std::optional TryParseBoolean(const std::string_view value) { const std::string lowered = ToLowerAscii(value); + if (lowered == "true" || lowered == "1") { return true; } if (lowered == "false" || lowered == "0") { return false; } + return std::nullopt; } bool ParseBoolean(const std::string_view value) { const std::optional result = TryParseBoolean(value); + if (!result.has_value()) { throw Error::MalformedData("common", "invalid bool value"); } + return *result; } std::optional TryParseInt16(const std::string_view value) { int16_t result = 0; + if (!TryParseInteger(value, result)) { return std::nullopt; } + return result; } @@ -156,9 +167,11 @@ int16_t ParseInt16(const std::string_view value) { return ParseInteger( std::optional TryParseInt32(const std::string_view value) { int32_t result = 0; + if (!TryParseInteger(value, result)) { return std::nullopt; } + return result; } @@ -166,9 +179,11 @@ int32_t ParseInt32(const std::string_view value) { return ParseInteger( std::optional TryParseInt64(const std::string_view value) { int64_t result = 0; + if (!TryParseInteger(value, result)) { return std::nullopt; } + return result; } @@ -199,11 +214,13 @@ std::optional TryParseInt128(const std::string_view value) { for (; pos < value.size(); ++pos) { const char ch = value[pos]; + if (ch < '0' || ch > '9') { return std::nullopt; } const auto digit = static_cast(ch - '0'); + if (magnitude > (limit - digit) / 10) { return std::nullopt; } @@ -228,6 +245,7 @@ Int128 ParseInt128(const std::string_view value) { } const std::optional result = TryParseInt128(value); + if (!result.has_value()) { throw Error::MalformedData("common", "invalid int128 value"); } @@ -237,6 +255,7 @@ Int128 ParseInt128(const std::string_view value) { std::optional TryParseDate(const std::string_view value) { std::chrono::sys_days day_point{}; + if (!TryParseDatePoint(value, day_point)) { return std::nullopt; } @@ -269,6 +288,7 @@ std::optional TryParseTimestamp(const std::string_view value) { } std::chrono::sys_days day_point{}; + if (!TryParseDatePoint(value.substr(0, TimestampDateLength), day_point)) { return std::nullopt; } @@ -305,11 +325,13 @@ std::optional TryParseTimestamp(const std::string_view value) { } const size_t digits = value.size() - TimestampFractionDigitsOffset; + if (digits == 0 || digits > TimestampMaxFractionDigits) { return std::nullopt; } unsigned fractional = 0; + if (!TryParseUnsignedPart(value, TimestampFractionDigitsOffset, digits, fractional)) { return std::nullopt; } @@ -328,9 +350,11 @@ std::optional TryParseTimestamp(const std::string_view value) { int64_t ParseTimestamp(const std::string_view value) { const std::optional result = TryParseTimestamp(value); + if (!result.has_value()) { throw Error::MalformedData("common", "invalid timestamp value"); } + return *result; } @@ -338,14 +362,17 @@ std::optional TryParseCharacter(const std::string_view value) { if (value.size() != 1) { return std::nullopt; } + return value[0]; } char ParseCharacter(const std::string_view value) { const std::optional result = TryParseCharacter(value); + if (!result.has_value()) { throw Error::MalformedData("common", "invalid char value"); } + return *result; } @@ -384,6 +411,7 @@ std::string Int128ToString(const Int128 value) { std::string DateToString(const int32_t value) { const auto day_point = std::chrono::sys_days{} + std::chrono::days{value}; + return FormatDateParts(std::chrono::year_month_day{day_point}); } @@ -393,18 +421,20 @@ std::string TimestampToString(const int64_t value) { const std::chrono::hh_mm_ss tod{timestamp - day_point}; std::ostringstream out; - out << FormatDateParts(std::chrono::year_month_day{day_point}) << TimestampDateTimeSeparator - << std::setfill('0') << std::setw(2) << tod.hours().count() << TimeSeparator << std::setw(2) - << tod.minutes().count() << TimeSeparator << std::setw(2) << tod.seconds().count(); + out << FormatDateParts(std::chrono::year_month_day{day_point}) << TimestampDateTimeSeparator << std::setfill('0') + << std::setw(2) << tod.hours().count() << TimeSeparator << std::setw(2) << tod.minutes().count() + << TimeSeparator << std::setw(2) << tod.seconds().count(); const int64_t fractional = tod.subseconds().count(); if (fractional != 0) { std::string frac = std::to_string(fractional); frac.insert(frac.begin(), TimestampMaxFractionDigits - static_cast(frac.size()), '0'); + while (!frac.empty() && frac.back() == '0') { frac.pop_back(); } + out << FractionSeparator << frac; } diff --git a/src/common/string_arena.cpp b/src/common/string_arena.cpp index b1165dd..d09284c 100644 --- a/src/common/string_arena.cpp +++ b/src/common/string_arena.cpp @@ -15,6 +15,7 @@ std::string_view StringArena::Store(const std::string_view value) { const size_t offset = chunk.used; std::memcpy(chunk.data.get() + static_cast(offset), value.data(), value.size()); chunk.used = offset + value.size(); + return {chunk.data.get() + static_cast(offset), value.size()}; } diff --git a/src/common/string_pattern_utils.cpp b/src/common/string_pattern_utils.cpp index 48252cd..52e2f02 100644 --- a/src/common/string_pattern_utils.cpp +++ b/src/common/string_pattern_utils.cpp @@ -42,9 +42,11 @@ static bool LikeMatchesPercentOnly(const std::string_view value, const std::stri for (; segment_index < segments.size(); ++segment_index) { const std::string_view segment = segments[segment_index]; const size_t found = value.find(segment, value_pos); + if (found == std::string_view::npos) { return false; } + value_pos = found + segment.size(); } diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index d4cd162..37746ad 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -36,6 +36,7 @@ static std::string FormatAverage(const Int128 sum, const uint64_t count) { if (count == 0) { return "0"; } + return Int128ToString(sum / static_cast(count)); } @@ -120,6 +121,7 @@ class ExtremumAS final : public AggState { if (typed_extremum_.has_value()) { return FormatInt128Value(type_, *typed_extremum_); } + return extremum_.value_or(""); } @@ -181,9 +183,11 @@ void AggRegistry::Register(const AggFuncDefinition& def) { registry_[ToUpperAsci const AggFuncDefinition* AggRegistry::Find(const std::string_view name) const { const auto it = registry_.find(ToUpperAscii(name)); + if (it == registry_.end()) { return nullptr; } + return &it->second; } @@ -222,9 +226,11 @@ AggRegistrar::AggRegistrar(const AggFuncDefinition& def) { AggRegistry::Instance const AggFuncDefinition& ResolveAggFunc(const std::string_view name) { const AggFuncDefinition* definition = AggRegistry::Instance().Find(name); + if (definition == nullptr) { throw Error::Unsupported("executor", "aggregate '" + std::string(name) + "' is not registered"); } + return *definition; } @@ -232,6 +238,7 @@ bool AggSupportsInputType(const AggFuncDefinition& definition, const ColumnType if (definition.supports_type == nullptr) { return false; } + return definition.supports_type(type); } @@ -252,6 +259,7 @@ std::unique_ptr CreateAggState(const PlannedAgg& aggregate) { } std::unique_ptr state = aggregate.function->factory(aggregate); + if (aggregate.distinct) { state = std::make_unique(std::move(state)); } diff --git a/src/executor/executor.cpp b/src/executor/executor.cpp index 3ee4cf5..2aabc52 100644 --- a/src/executor/executor.cpp +++ b/src/executor/executor.cpp @@ -55,6 +55,7 @@ ExecuteExpected Executor::ExecutePlanned(const PlannedQuery& planned) { const std::unique_ptr root = BuildPlan(planned); auto batch = root->Next(); + if (!batch.has_value()) { return Batch{}; } diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index df2c298..2e5481b 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -29,6 +29,14 @@ std::unique_ptr BuildPlan(const PlannedQuery& planned) { bool limit_applied_by_top_k = false; + if (!planned.group_keys.empty() && !planned.order_by.empty() && planned.limit.has_value() && planned.offset == 0 && + !planned.having) { + root = CreateGroupAggTopKOperator(std::move(root), planned.group_keys, planned.aggregates, planned.select_items, + planned.filter, planned.order_by, *planned.limit); + root = CreateEnsureSchemaOperator(std::move(root), BuildSelectOutputSchema(planned.select_items)); + return root; + } + if (!planned.group_keys.empty()) { root = CreateGroupAggOperator(std::move(root), planned.group_keys, planned.aggregates, planned.select_items, planned.filter, planned.having, planned.order_by.empty()); diff --git a/src/executor/operators_aggregate.cpp b/src/executor/operators_aggregate.cpp index 99664ce..997fc72 100644 --- a/src/executor/operators_aggregate.cpp +++ b/src/executor/operators_aggregate.cpp @@ -20,6 +20,7 @@ constexpr std::string_view ExtractMinutePart = "MINUTE"; constexpr std::string_view ExtractHourPart = "HOUR"; constexpr int64_t MinuteMicros = 60'000'000; +constexpr size_t InitialGroupReserve = 1U << 14; template Schema BuildAggregateOutputSchema(const std::vector& bindings) { @@ -54,7 +55,9 @@ void ConsumeAggRow(const PlannedAgg& aggregate, const Batch& batch, const size_t return; } } + state.ConsumeValue(EvalExpr(aggregate.argument, batch, row)); + return; } @@ -88,6 +91,7 @@ class CompactAggState { } const std::string name = ToUpperAscii(aggregate.function->canonical_name); + if (name == "COUNT") { kind_ = Kind::Count; } else if (name == "SUM") { @@ -266,7 +270,9 @@ void ConsumeCompactAggRow(const PlannedAgg& aggregate, const Batch& batch, const return; } } + state.ConsumeValue(EvalExpr(aggregate.argument, batch, row)); + return; } @@ -318,6 +324,7 @@ std::optional TryEvalTypedGroupKeyInt(const ExprPtr& expr, const Batch& if (name == "EXTRACT" && expr->arguments.size() == 2 && expr->arguments[0] && expr->arguments[0]->kind == ExprKind::Literal) { const auto ts = TryEvalTypedGroupKeyInt(expr->arguments[1], batch, row); + if (!ts.has_value()) { return std::nullopt; } @@ -339,11 +346,13 @@ std::optional TryEvalTypedGroupKeyInt(const ExprPtr& expr, const Batch& if (name == "DATE_TRUNC" && expr->arguments.size() == 2 && expr->arguments[0] && expr->arguments[0]->kind == ExprKind::Literal) { const auto ts = TryEvalTypedGroupKeyInt(expr->arguments[1], batch, row); + if (!ts.has_value()) { return std::nullopt; } const std::string part = ToUpperAscii(NormalizeLiteralForEval(expr->arguments[0]->literal)); + if (part == ExtractMinutePart) { const int64_t micros = static_cast(*ts); return micros / MinuteMicros * MinuteMicros; @@ -406,26 +415,32 @@ size_t HashInt128(const Int128 value) { return HashCombine(std::hash{}(high), std::hash{}(low)); } +size_t ComputeGroupKeyHash(const std::span values) { + size_t hash = 0; + + for (const GroupKeyComponent& value : values) { + hash = HashCombine(hash, std::hash{}(static_cast(value.type))); + hash = HashCombine(hash, value.type == ColumnType::String ? StringViewHash{}(value.string_value) + : HashInt128(value.int_value)); + } + + return hash; +} + struct TypedGroupKey { std::vector values; + size_t precomputed_hash = 0; }; struct TypedGroupKeyHash { - size_t operator()(const TypedGroupKey& key) const { - size_t hash = 0; - - for (const GroupKeyComponent& value : key.values) { - hash = HashCombine(hash, std::hash{}(static_cast(value.type))); - hash = HashCombine(hash, value.type == ColumnType::String ? StringViewHash{}(value.string_value) - : HashInt128(value.int_value)); - } - - return hash; - } + size_t operator()(const TypedGroupKey& key) const { return key.precomputed_hash; } }; struct TypedGroupKeyEqual { bool operator()(const TypedGroupKey& lhs, const TypedGroupKey& rhs) const { + if (lhs.precomputed_hash != rhs.precomputed_hash) { + return false; + } if (lhs.values.size() != rhs.values.size()) { return false; } @@ -442,17 +457,22 @@ struct TypedGroupKeyEqual { class GroupKeyMaterializer { public: - explicit GroupKeyMaterializer(std::vector group_keys) : group_keys_(std::move(group_keys)) {} + explicit GroupKeyMaterializer(std::vector group_keys) : group_keys_(std::move(group_keys)) { + scratch_.values.reserve(group_keys_.size()); + scratch_strings_.reserve(group_keys_.size()); + } + + const TypedGroupKey& MaterializeScratch(const Batch& batch, const size_t row) const { + scratch_.values.clear(); + scratch_strings_.clear(); - TypedGroupKey Materialize(const Batch& batch, const size_t row, StringArena& arena) const { - TypedGroupKey key; - key.values.reserve(group_keys_.size()); + std::string scratch; for (const auto& group_key : group_keys_) { if (group_key.column_type != ColumnType::String) { if (const auto typed_value = TryEvalTypedGroupKeyInt(group_key.expression, batch, row); typed_value.has_value()) { - key.values.push_back(GroupKeyComponent{ + scratch_.values.push_back(GroupKeyComponent{ .type = group_key.column_type, .int_value = *typed_value, .string_value = {}, @@ -462,23 +482,25 @@ class GroupKeyMaterializer { } else if (group_key.expression && group_key.expression->kind == ExprKind::Column && group_key.expression->column_index_bound && group_key.expression->column_index < batch.ColumnsCount()) { - key.values.push_back(GroupKeyComponent{ + scratch_.values.push_back(GroupKeyComponent{ .type = group_key.column_type, .int_value = 0, - .string_value = arena.Store(batch.ColumnAt(group_key.expression->column_index).ValueAsString(row)), + .string_value = batch.ColumnAt(group_key.expression->column_index).ValueAsStringView(row, scratch), }); continue; } std::string value = EvalExpr(group_key.expression, batch, row); + if (group_key.column_type == ColumnType::String) { - key.values.push_back(GroupKeyComponent{ + scratch_strings_.push_back(std::move(value)); + scratch_.values.push_back(GroupKeyComponent{ .type = group_key.column_type, .int_value = 0, - .string_value = arena.Store(value), + .string_value = scratch_strings_.back(), }); } else { - key.values.push_back(GroupKeyComponent{ + scratch_.values.push_back(GroupKeyComponent{ .type = group_key.column_type, .int_value = ParseColumnValueAsInt128(group_key.column_type, value), .string_value = {}, @@ -486,11 +508,31 @@ class GroupKeyMaterializer { } } - return key; + scratch_.precomputed_hash = ComputeGroupKeyHash(scratch_.values); + + return scratch_; + } + + static TypedGroupKey CopyToArena(const TypedGroupKey& key, StringArena& arena) { + TypedGroupKey owned; + owned.values.reserve(key.values.size()); + owned.precomputed_hash = key.precomputed_hash; + + for (const GroupKeyComponent& value : key.values) { + owned.values.push_back(GroupKeyComponent{ + .type = value.type, + .int_value = value.int_value, + .string_value = value.type == ColumnType::String ? arena.Store(value.string_value) : std::string_view{}, + }); + } + + return owned; } private: std::vector group_keys_; + mutable TypedGroupKey scratch_; + mutable std::vector scratch_strings_; }; class AggOperator final : public Operator { @@ -562,6 +604,8 @@ class GroupAggOperator final : public Operator { .aggregate = aggregate, }); } + + groups_.reserve(InitialGroupReserve); } std::optional Next() override { @@ -577,16 +621,18 @@ class GroupAggOperator final : public Operator { continue; } - TypedGroupKey key = group_key_materializer_.Materialize(*batch, row, string_arena_); + const TypedGroupKey& key = group_key_materializer_.MaterializeScratch(*batch, row); auto it = groups_.find(key); + if (it == groups_.end()) { + TypedGroupKey owned_key = group_key_materializer_.CopyToArena(key, string_arena_); GroupState group{ .states = CreateStates(), .finalized_aggregates = nullptr, .ordinal = next_group_ordinal_++, }; - it = groups_.emplace(std::move(key), std::move(group)).first; + it = groups_.emplace(std::move(owned_key), std::move(group)).first; } GroupState& group = it->second; @@ -647,6 +693,7 @@ class GroupAggOperator final : public Operator { for (size_t i = 0; i < group.states.size(); ++i) { const ColumnType type = AggregateOutputType(bindings_[i].aggregate); + if (type == ColumnType::String) { finalized->int_values.push_back(0); finalized->values.push_back(group.states[i].Finalize()); @@ -659,6 +706,7 @@ class GroupAggOperator final : public Operator { } group.finalized_aggregates = std::move(finalized); + return *group.finalized_aggregates; } @@ -764,6 +812,8 @@ class GroupAggTopKOperator final : public Operator { .aggregate = aggregate, }); } + + groups_.reserve(InitialGroupReserve); } std::optional Next() override { @@ -779,16 +829,18 @@ class GroupAggTopKOperator final : public Operator { continue; } - TypedGroupKey key = group_key_materializer_.Materialize(*batch, row, string_arena_); + const TypedGroupKey& key = group_key_materializer_.MaterializeScratch(*batch, row); auto it = groups_.find(key); + if (it == groups_.end()) { + TypedGroupKey owned_key = group_key_materializer_.CopyToArena(key, string_arena_); GroupState group{ .states = CreateStates(), .finalized_aggregates = nullptr, .ordinal = next_group_ordinal_++, }; - it = groups_.emplace(std::move(key), std::move(group)).first; + it = groups_.emplace(std::move(owned_key), std::move(group)).first; } GroupState& group = it->second; @@ -846,6 +898,7 @@ class GroupAggTopKOperator final : public Operator { for (size_t i = 0; i < group.states.size(); ++i) { const ColumnType type = AggregateOutputType(bindings_[i].aggregate); + if (type == ColumnType::String) { finalized->int_values.push_back(0); finalized->values.push_back(group.states[i].Finalize()); @@ -858,6 +911,7 @@ class GroupAggTopKOperator final : public Operator { } group.finalized_aggregates = std::move(finalized); + return *group.finalized_aggregates; } diff --git a/src/executor/operators_runtime.cpp b/src/executor/operators_runtime.cpp index ecffa19..67a3a0e 100644 --- a/src/executor/operators_runtime.cpp +++ b/src/executor/operators_runtime.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -37,19 +38,18 @@ bool IsAggregateLookupFunction(const std::string_view name) { std::string FormatAggregateExprName(const ExprSpec& expr) { const std::string name = ToUpperAscii(expr.function_name); + if (name == "COUNT" && !expr.arguments.empty() && expr.arguments.front()->kind == ExprKind::Star) { return std::string(CountStarName); } std::string out = name + "("; - for (size_t i = 0; i < expr.arguments.size(); ++i) { if (i > 0) { out += ", "; } out += expr.arguments[i]->output_name; } - out += ")"; return out; @@ -57,6 +57,7 @@ std::string FormatAggregateExprName(const ExprSpec& expr) { std::optional TryResolveAggregateValue(const ExprSpec& expr, const Batch& batch, const size_t row) { const std::string name = ToUpperAscii(expr.function_name); + if (!IsAggregateLookupFunction(name)) { return std::nullopt; } @@ -72,6 +73,27 @@ std::optional TryResolveAggregateValue(const ExprSpec& expr, const return std::nullopt; } +std::optional TryReadTimestampMicros(const ExprPtr& expr, const Batch& batch, const size_t row) { + if (expr && expr->kind == ExprKind::Column && expr->column_index_bound && + expr->column_index < batch.ColumnsCount()) { + const Column& column = batch.ColumnAt(expr->column_index); + + if (column.Type() == ColumnType::Timestamp) { + return static_cast(column.ValueAsInt128(row)); + } + } + + return std::nullopt; +} + +int64_t EvalTimestampMicros(const ExprPtr& expr, const Batch& batch, const size_t row) { + if (const auto micros = TryReadTimestampMicros(expr, batch, row); micros.has_value()) { + return *micros; + } + + return ParseTimestamp(EvalExpr(expr, batch, row)); +} + std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t row) { if (const auto aggregate_value = TryResolveAggregateValue(expr, batch, row); aggregate_value.has_value()) { return *aggregate_value; @@ -85,7 +107,7 @@ std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t if (name == "EXTRACT") { const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); - const int64_t ts = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); + const int64_t ts = EvalTimestampMicros(expr.arguments.at(1), batch, row); const std::chrono::sys_time time{std::chrono::microseconds{ts}}; const auto day = std::chrono::floor(time); @@ -104,7 +126,7 @@ std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t if (name == "DATE_TRUNC") { const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); - int64_t micros = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); + int64_t micros = EvalTimestampMicros(expr.arguments.at(1), batch, row); if (part == ExtractMinutePart) { micros = micros / MinuteMicros * MinuteMicros; @@ -121,6 +143,7 @@ std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t if (pattern_text == R"(^https?://(?:www\.)?([^/]+)/.*$)" && replacement == R"(\1)") { std::string_view rest = source; + if (rest.starts_with(HttpScheme)) { rest.remove_prefix(HttpScheme.size()); } else if (rest.starts_with(HttpsScheme)) { @@ -134,11 +157,13 @@ std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t } const size_t slash = rest.find('/'); + if (slash == std::string_view::npos || slash == 0) { return source; } const std::string_view suffix = rest.substr(slash + 1); + if (suffix.find('\n') != std::string_view::npos || suffix.find('\r') != std::string_view::npos) { return source; } @@ -215,16 +240,19 @@ std::optional TryEvaluateTypedComparison(const ExprPtr& left, const ExprPt } const auto column_index = TryFindBatchColumn(batch.GetSchema(), left->column.name); + if (!column_index.has_value()) { return std::nullopt; } const Column& column = batch.ColumnAt(*column_index); + if (column.Type() == ColumnType::String) { return std::nullopt; } const std::optional rhs = TryParseLiteralValueAsInt128(right->literal, column.Type()); + if (!rhs.has_value()) { return std::nullopt; } @@ -257,16 +285,20 @@ struct RowRef { std::strong_ordering CompareColumnRows(const Column& lhs_column, const size_t lhs_row, const Column& rhs_column, const size_t rhs_row, const ColumnType type) { - if (type != ColumnType::String && lhs_column.Type() == ColumnType::String && rhs_column.Type() == ColumnType::String) { - return ParseColumnValueAsInt128(type, lhs_column.ValueAsString(lhs_row)) <=> - ParseColumnValueAsInt128(type, rhs_column.ValueAsString(rhs_row)); + std::string lhs_scratch; + std::string rhs_scratch; + + if (type != ColumnType::String && lhs_column.Type() == ColumnType::String && + rhs_column.Type() == ColumnType::String) { + return ParseColumnValueAsInt128(type, lhs_column.ValueAsStringView(lhs_row, lhs_scratch)) <=> + ParseColumnValueAsInt128(type, rhs_column.ValueAsStringView(rhs_row, rhs_scratch)); } if (type != ColumnType::String) { return lhs_column.ValueAsInt128(lhs_row) <=> rhs_column.ValueAsInt128(rhs_row); } - return lhs_column.ValueAsString(lhs_row) <=> rhs_column.ValueAsString(rhs_row); + return lhs_column.ValueAsStringView(lhs_row, lhs_scratch) <=> rhs_column.ValueAsStringView(rhs_row, rhs_scratch); } class RowOrdering { @@ -617,6 +649,7 @@ std::string EvalExpr(const ExprPtr& expr, const Batch& batch, const size_t row) } const auto column = TryFindBatchColumn(batch.GetSchema(), expr->column.name); + if (!column.has_value()) { throw Error::InvalidArgument("executor", "unknown column '" + expr->column.name + "'"); } @@ -639,6 +672,7 @@ std::string EvalExpr(const ExprPtr& expr, const Batch& batch, const size_t row) case ExprKind::Star: return "*"; } + return {}; } diff --git a/src/executor/operators_scan.cpp b/src/executor/operators_scan.cpp index 0e89bc3..293d492 100644 --- a/src/executor/operators_scan.cpp +++ b/src/executor/operators_scan.cpp @@ -132,6 +132,7 @@ bool MayMatchRowGroup(const PredicatePtr& predicate, const RowGroupMetadata& row predicate->metadata_typed_column_index >= row_group.columns.size()) { return true; } + return MatchesRowGroupComparison(row_group.columns[predicate->metadata_typed_column_index], predicate->metadata_typed_literal_value, predicate->metadata_typed_comparison); @@ -200,7 +201,8 @@ class ScanOperator final : public Operator { const size_t source_index = projection_indexes_[projected_index]; const auto& chunk = row_group->columns[source_index]; - ReadBatchColumnChunk(path_, input_, chunk, row_group->row_count, batch, projected_index); + ReadBatchColumnChunk(path_, input_, chunk, row_group->row_count, batch, projected_index, read_buffer_, + decompression_buffer_); } return batch; @@ -217,6 +219,9 @@ class ScanOperator final : public Operator { PredicatePtr filter_; size_t next_group_ = 0; + + std::vector read_buffer_; + std::vector decompression_buffer_; }; std::unique_ptr CreateMetadataCountOperator(std::filesystem::path path, std::string output_name) { diff --git a/src/executor/query_parser.cpp b/src/executor/query_parser.cpp index 04bcfd5..4ad8e40 100644 --- a/src/executor/query_parser.cpp +++ b/src/executor/query_parser.cpp @@ -16,6 +16,7 @@ class TokenCursor { if (pos_ + lookahead >= tokens_.size()) { throw Error::InvalidArgument("executor", "unexpected end of query"); } + return *tokens_[pos_ + lookahead]; } @@ -27,6 +28,7 @@ class TokenCursor { if (!Match(type)) { throw Error::InvalidArgument("executor", std::string(message)); } + return *tokens_[pos_++]; } @@ -34,7 +36,9 @@ class TokenCursor { if (!Match(type)) { return false; } + ++pos_; + return true; } @@ -44,6 +48,7 @@ class TokenCursor { } const Tokens type = tokens_[pos_]->GetType(); + switch (type) { case Tokens::NumericLiteral: case Tokens::StringLiteral: @@ -74,6 +79,7 @@ class TokenCursor { static bool IsAggregateName(const std::string_view name) { const std::string upper = ToUpperAscii(name); + return upper == "COUNT" || upper == "SUM" || upper == "AVG" || upper == "MIN" || upper == "MAX"; } @@ -100,6 +106,7 @@ static std::string FormatColumnRef(const ColumnRef& column) { if (column.qualifier.empty()) { return column.name; } + return column.qualifier + "." + column.name; } @@ -207,6 +214,7 @@ static ExprPtr ParsePrimary(TokenCursor& cursor) { expr->left = std::move(zero); expr->right = ParsePrimary(cursor); expr->output_name = FormatExpression(*expr); + return expr; } @@ -224,8 +232,10 @@ static ExprPtr ParsePrimary(TokenCursor& cursor) { if (cursor.Match(Tokens::Select)) { throw Error::Unsupported("executor", "subqueries are not supported"); } + auto expr = ParseExpression(cursor); cursor.Consume(Tokens::CloseBracket, "expected ')' after expression"); + return expr; } @@ -235,6 +245,7 @@ static ExprPtr ParsePrimary(TokenCursor& cursor) { QueryLiteral{std::string(cursor.Consume(Tokens::NumericLiteral, "expected numeric literal").GetText()), LiteralKind::Numeric}; expr->output_name = expr->literal.text; + return expr; } @@ -244,6 +255,7 @@ static ExprPtr ParsePrimary(TokenCursor& cursor) { QueryLiteral{std::string(cursor.Consume(Tokens::StringLiteral, "expected string literal").GetText()), LiteralKind::String}; expr->output_name = expr->literal.text; + return expr; } @@ -279,6 +291,7 @@ static ExprPtr ParseExpression(TokenCursor& cursor) { while (cursor.Match(Tokens::Plus) || cursor.Match(Tokens::Minus)) { const bool add = cursor.TryConsume(Tokens::Plus); + if (!add) { cursor.Consume(Tokens::Minus, "expected '-'"); } @@ -288,6 +301,7 @@ static ExprPtr ParseExpression(TokenCursor& cursor) { binary->left = std::move(expr); binary->right = ParsePrimary(cursor); binary->output_name = FormatExpression(*binary); + expr = std::move(binary); } @@ -323,6 +337,7 @@ static PredicatePtr ParsePredicateAtom(TokenCursor& cursor) { auto predicate = MakePredicate(PredicateKind::Like); predicate->left = std::move(left); predicate->right = ParseExpression(cursor); + return predicate; } @@ -376,6 +391,7 @@ static std::optional ParseOptionalSize(TokenCursor& cursor, const Tokens const char* begin = text.data(); const char* end = begin + text.size(); + if (const auto [ptr, ec] = std::from_chars(begin, end, value); ec != std::errc() || ptr != end) { throw Error::InvalidArgument("executor", "invalid " + std::string(name) + " value"); } @@ -498,6 +514,7 @@ static OrderBySpec ParseOrderByItem(TokenCursor& cursor) { Query ParseQuery(const std::string_view query) { auto tokenized = TokenizeSql(query); + if (!tokenized.has_value()) { throw tokenized.error(); } @@ -546,6 +563,7 @@ Query ParseQuery(const std::string_view query) { } query_ast.limit = ParseOptionalSize(cursor, Tokens::Limit, "LIMIT"); + if (const auto offset = ParseOptionalSize(cursor, Tokens::Offset, "OFFSET"); offset.has_value()) { query_ast.offset = *offset; } diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index 619d6e6..22065b0 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -17,6 +17,7 @@ static ColumnType ColumnTypeOf(const Query& query, const Schema& schema, const E static ColumnType ColumnTypeOfColumn(const Query& query, const Schema& schema, const ColumnRef& column) { ValidateColumnQualifier(query, column); + return schema.columns[FindColumnIndex(schema, column.name)].type; } @@ -34,6 +35,7 @@ static ColumnType ColumnTypeOf(const Query& query, const Schema& schema, const E return ColumnType::Int64; case ExprKind::Function: { const std::string name = ToUpperAscii(expr->function_name); + if (name == "STRLEN" || name == "LENGTH" || name == "EXTRACT") { return ColumnType::Int64; } @@ -52,6 +54,7 @@ static ColumnType ColumnTypeOf(const Query& query, const Schema& schema, const E if ((name == "MIN" || name == "MAX") && !expr->arguments.empty()) { return ColumnTypeOf(query, schema, expr->arguments.front()); } + return ColumnType::String; } case ExprKind::Case: @@ -71,9 +74,11 @@ static bool HasAggregate(const std::vector& items) { static std::optional FindSelectItem(const Query& query_ast, const SelectItemSpec& item) { for (size_t i = 0; i < query_ast.select_items.size(); ++i) { const SelectItemSpec& candidate = query_ast.select_items[i]; + if (SameOutputName(candidate, item)) { return i; } + if (candidate.kind == item.kind) { if (candidate.kind == SelectItemKind::Aggregate && SameColumnName(candidate.aggregate.output_name, item.aggregate.output_name)) { @@ -102,6 +107,7 @@ static ExprPtr ResolveGroupByExpression(const Query& query_ast, const ExprPtr& e if (expr->kind == ExprKind::Literal && expr->literal.kind == LiteralKind::Numeric) { try { const size_t ordinal = std::stoull(expr->literal.text); + if (ordinal >= SqlOrdinalBase && ordinal <= query_ast.select_items.size()) { return query_ast.select_items[ordinal - SqlOrdinalBase].expression; } @@ -210,6 +216,7 @@ static std::optional ProjectedIndexOf(const std::vector& project return i; } } + return std::nullopt; } @@ -263,12 +270,15 @@ static void BindExprColumnIndexes(const Query& query_ast, const Schema& schema, switch (expr->kind) { case ExprKind::Column: { ValidateColumnQualifier(query_ast, expr->column); + const size_t source_index = FindColumnIndex(schema, expr->column.name); const std::optional projected_index = ProjectedIndexOf(projection_indexes, source_index); + if (projected_index.has_value()) { expr->column_index_bound = true; expr->column_index = *projected_index; } + return; } case ExprKind::Literal: @@ -282,7 +292,9 @@ static void BindExprColumnIndexes(const Query& query_ast, const Schema& schema, for (const ExprPtr& argument : expr->arguments) { BindExprColumnIndexes(query_ast, schema, projection_indexes, argument); } + BindRegexReplace(expr); + return; case ExprKind::Case: BindPredicateColumnIndexes(query_ast, schema, projection_indexes, expr->case_spec.condition); @@ -302,6 +314,7 @@ static bool IsSimpleCountStar(const Query& query_ast, const PlannedQuery& planne } const PlannedAgg& aggregate = planned.aggregates.front(); + return aggregate.function != nullptr && SameColumnName(aggregate.function->canonical_name, "COUNT") && aggregate.argument_kind == AggArgumentKind::Star && !aggregate.distinct; } @@ -344,8 +357,10 @@ static void BindLiteralInSet(const Query& query_ast, const Schema& schema, } ValidateColumnQualifier(query_ast, predicate->left->column); + const size_t source_index = FindColumnIndex(schema, predicate->left->column.name); const std::optional projected_index = ProjectedIndexOf(projection_indexes, source_index); + if (!projected_index.has_value()) { return; } @@ -374,9 +389,11 @@ static void BindLiteralInSet(const Query& query_ast, const Schema& schema, for (const ExprPtr& value : predicate->values) { const std::optional typed_value = TryParseLiteralValueAsInt128(value->literal, schema.columns[source_index].type); + if (!typed_value.has_value()) { return; } + typed_values.push_back(*typed_value); } @@ -395,11 +412,13 @@ static void BindLiteralLikePattern(const Query& query_ast, const Schema& schema, ValidateColumnQualifier(query_ast, predicate->left->column); const size_t source_index = FindColumnIndex(schema, predicate->left->column.name); + if (schema.columns[source_index].type != ColumnType::String) { return; } const std::optional projected_index = ProjectedIndexOf(projection_indexes, source_index); + if (!projected_index.has_value()) { return; } @@ -434,12 +453,14 @@ static void BindTypedLiteralComparison(const Query& query_ast, const Schema& sch ValidateColumnQualifier(query_ast, (*column_expr)->column); const size_t source_index = FindColumnIndex(schema, (*column_expr)->column.name); const std::optional projected_index = ProjectedIndexOf(projection_indexes, source_index); + if (!projected_index.has_value()) { return; } const ColumnType type = schema.columns[source_index].type; const std::optional literal = TryParseLiteralValueAsInt128((*literal_expr)->literal, type); + if (!literal.has_value()) { return; } @@ -469,6 +490,7 @@ static std::optional> TryLinearColumnExpr(const Exp } const std::optional literal = TryNumericLiteralValue(expr->right); + if (!literal.has_value()) { return std::nullopt; } @@ -482,7 +504,9 @@ static bool SupportsDirectTypedAggInput(const AggFuncDefinition& function, const if (IsNumericColumnType(type)) { return true; } + const std::string name = ToUpperAscii(function.canonical_name); + return (name == "MIN" || name == "MAX") && (type == ColumnType::Date || type == ColumnType::Timestamp); } @@ -494,6 +518,7 @@ static bool IsSimpleMetadataExtrema(const Query& query, const PlannedQuery& plan for (const PlannedAgg& aggregate : planned.aggregates) { const std::string name = ToUpperAscii(aggregate.function->canonical_name); + if ((name != "MIN" && name != "MAX") || aggregate.distinct || aggregate.argument_kind != AggArgumentKind::Column || !aggregate.direct_numeric_argument || aggregate.direct_numeric_offset != 0) { @@ -551,8 +576,10 @@ PlannedQuery PlanQuery(const Query& query, const std::unordered_mapkind == ExprKind::Star) { for (const auto& column : schema.columns) { planned.select_items.push_back(PlannedSelectItem{ @@ -564,6 +591,7 @@ PlannedQuery PlanQuery(const Query& query, const std::unordered_map result_column = FindSelectItem(query, order_item.item); + if (!result_column.has_value()) { throw Error::InvalidArgument("executor", "ORDER BY expression must appear in SELECT"); } @@ -743,6 +773,7 @@ PlannedQuery PlanQuery(const Query& query, const std::unordered_map projected_index = ProjectedIndexOf(planned.projection_indexes, aggregate.column_index); + if (projected_index.has_value()) { aggregate.column_index = *projected_index; } else { diff --git a/src/executor/query_utils.cpp b/src/executor/query_utils.cpp index 710db9f..6797ad8 100644 --- a/src/executor/query_utils.cpp +++ b/src/executor/query_utils.cpp @@ -144,11 +144,13 @@ bool SameExpr(const ExprPtr& lhs, const ExprPtr& rhs) { lhs->arguments.size() != rhs->arguments.size()) { return false; } + for (size_t i = 0; i < lhs->arguments.size(); ++i) { if (!SameExpr(lhs->arguments[i], rhs->arguments[i])) { return false; } } + return true; case ExprKind::Case: return lhs->output_name == rhs->output_name; @@ -189,5 +191,6 @@ std::optional TryFindBatchColumn(const Schema& schema, const std::string return i; } } + return std::nullopt; } diff --git a/src/io/columnar_batch.cpp b/src/io/columnar_batch.cpp index 25c7f6b..7087483 100644 --- a/src/io/columnar_batch.cpp +++ b/src/io/columnar_batch.cpp @@ -18,9 +18,11 @@ static constexpr std::string_view ColumnarMagic = "CLMN"; static uint64_t TellWrite(const std::filesystem::path& path, std::ofstream& out) { const auto pos = out.tellp(); + if (pos == -1) { throw Error::PathIo("io", path, "tell file"); } + return pos; } @@ -33,6 +35,14 @@ static void FlushWrite(const std::filesystem::path& path, std::ofstream& out) { static bool SupportsChunkMinMax(const ColumnType type) { return type != ColumnType::String; } +static size_t CheckedChunkSize(const std::filesystem::path& path, const uint64_t size) { + if (size > static_cast(std::numeric_limits::max())) { + throw Error::Overflow("io", "column chunk exceeds addressable size", path.string()); + } + + return size; +} + static void PopulateChunkMinMax(const Column& column, ColumnChunkMetadata& chunk) { if (!SupportsChunkMinMax(column.Type()) || column.Size() == 0) { return; @@ -43,9 +53,11 @@ static void PopulateChunkMinMax(const Column& column, ColumnChunkMetadata& chunk for (size_t row = 1; row < column.Size(); ++row) { const Int128 value = column.ValueAsInt128(row); + if (value < min_value) { min_value = value; } + if (value > max_value) { max_value = value; } @@ -59,7 +71,9 @@ static void PopulateChunkMinMax(const Column& column, ColumnChunkMetadata& chunk static std::vector SerializeColumn(const Column& column) { std::ostringstream buffer(std::ios::binary); column.WriteTo(buffer); + const std::string bytes = std::move(buffer).str(); + return std::vector(bytes.begin(), bytes.end()); } @@ -145,7 +159,7 @@ std::optional ColumnarBatchReader::ReadNext() { for (size_t col = 0; col < batch.ColumnsCount(); ++col) { const auto& chunk = row_group_columns[col]; - ReadBatchColumnChunk(path_, input_, chunk, row_count, batch, col); + ReadBatchColumnChunk(path_, input_, chunk, row_count, batch, col, read_buffer_, decompression_buffer_); } return batch; @@ -166,6 +180,7 @@ void ColumnarBatchWriter::Write(const Batch& batch) { } batch.Validate(); + if (batch.GetSchema() != metadata_.schema) { throw Error::InconsistentData("io", "batch schema mismatch", path_.string()); } @@ -216,23 +231,44 @@ void ColumnarBatchWriter::Finalize() && { std::vector ReadColumnChunk(const std::filesystem::path& path, InputFile& input, const ColumnChunkMetadata& chunk) { - const std::string raw = input.ReadStringAt(chunk.offset, chunk.compressed_size); - const std::span raw_bytes(reinterpret_cast(raw.data()), raw.size()); + std::vector read_buffer; + std::vector decompression_buffer; + + read_buffer.resize(CheckedChunkSize(path, chunk.compressed_size)); + input.ReadAt(reinterpret_cast(read_buffer.data()), read_buffer.size(), chunk.offset); + + const std::span raw_bytes(read_buffer); if (chunk.compression == Compression::None) { if (chunk.compressed_size != chunk.uncompressed_size) { throw Error::MalformedData("io", "uncompressed chunk size mismatch", path.string()); } - return std::vector(raw_bytes.begin(), raw_bytes.end()); + + return read_buffer; } - return Decompress(raw_bytes, chunk.compression, chunk.uncompressed_size); + DecompressInto(raw_bytes, chunk.compression, chunk.uncompressed_size, decompression_buffer); + + return decompression_buffer; } void ReadBatchColumnChunk(const std::filesystem::path& path, InputFile& input, const ColumnChunkMetadata& chunk, - const uint32_t row_count, const Batch& batch, const size_t column_index) { - const std::vector payload = ReadColumnChunk(path, input, chunk); - const std::string bytes(payload.begin(), payload.end()); - std::istringstream stream(bytes, std::ios::binary); - batch.ReadColumnFrom(column_index, stream, row_count, chunk.uncompressed_size); + const uint32_t row_count, const Batch& batch, const size_t column_index, + std::vector& read_buffer, std::vector& decompression_buffer) { + read_buffer.resize(CheckedChunkSize(path, chunk.compressed_size)); + input.ReadAt(reinterpret_cast(read_buffer.data()), read_buffer.size(), chunk.offset); + + std::span payload(read_buffer); + + if (chunk.compression == Compression::None) { + if (chunk.compressed_size != chunk.uncompressed_size) { + throw Error::MalformedData("io", "uncompressed chunk size mismatch", path.string()); + } + } else { + DecompressInto(payload, chunk.compression, chunk.uncompressed_size, decompression_buffer); + payload = std::span(decompression_buffer); + } + + batch.ReadColumnFrom(column_index, {reinterpret_cast(payload.data()), (payload.size())}, row_count, + chunk.uncompressed_size); } diff --git a/src/io/compression.cpp b/src/io/compression.cpp index ec39b62..213af82 100644 --- a/src/io/compression.cpp +++ b/src/io/compression.cpp @@ -43,6 +43,7 @@ std::vector Compress(const std::span input, const Compre } const int upper_bound = LZ4_compressBound(static_cast(input.size())); + if (upper_bound <= 0) { throw Error::InvalidState("compression", "lz4 upper bound failed"); } @@ -63,12 +64,22 @@ std::vector Compress(const std::span input, const Compre std::vector Decompress(const std::span input, const Compression compression, const uint64_t uncompressed_size) { + std::vector output; + + DecompressInto(input, compression, uncompressed_size, output); + + return output; +} + +void DecompressInto(const std::span input, const Compression compression, + const uint64_t uncompressed_size, std::vector& output) { switch (compression) { case Compression::None: if (input.size() != uncompressed_size) { throw Error::MalformedData("compression", "uncompressed chunk size mismatch"); } - return std::vector(input.begin(), input.end()); + output.assign(input.begin(), input.end()); + return; case Compression::Lz4: break; } @@ -80,7 +91,8 @@ std::vector Decompress(const std::span input, const Comp throw Error::Overflow("compression", "uncompressed output too large for lz4"); } - std::vector output(uncompressed_size); + output.resize(uncompressed_size); + const int decompressed_size = LZ4_decompress_safe(reinterpret_cast(input.data()), reinterpret_cast(output.data()), static_cast(input.size()), static_cast(uncompressed_size)); @@ -92,6 +104,4 @@ std::vector Decompress(const std::span input, const Comp if (static_cast(decompressed_size) != uncompressed_size) { throw Error::MalformedData("compression", "lz4 decompressed size mismatch"); } - - return output; } diff --git a/src/io/csv.cpp b/src/io/csv.cpp index 29281b5..75df7b0 100644 --- a/src/io/csv.cpp +++ b/src/io/csv.cpp @@ -55,6 +55,7 @@ CsvReader& CsvReader::operator=(CsvReader&& other) noexcept { RebindAfterMove(other.in_ == &other.owned_in_, other.in_); other.in_ = nullptr; } + return *this; } @@ -85,6 +86,7 @@ bool CsvReader::ReadRow(std::vector& row) const { if (in_quotes) { throw Error::MalformedData("io", "unexpected EOF inside quoted field"); } + return true; } @@ -94,6 +96,7 @@ bool CsvReader::ReadRow(std::vector& row) const { if (in_quotes) { if (ch == CsvQuote) { const auto peek = input.Peek(); + if (!input.IsEof(peek) && input.ToChar(peek) == CsvQuote) { input.Discard(); row.back().push_back(CsvQuote); @@ -103,6 +106,7 @@ bool CsvReader::ReadRow(std::vector& row) const { } else { row.back().push_back(ch); } + continue; } @@ -126,9 +130,11 @@ bool CsvReader::ReadRow(std::vector& row) const { if (ch == CsvCr) { const auto peek = input.Peek(); + if (!input.IsEof(peek) && input.ToChar(peek) == CsvLf) { input.Discard(); } + return true; } @@ -151,6 +157,7 @@ CsvWriter& CsvWriter::operator=(CsvWriter&& other) noexcept { RebindAfterMove(other.out_ == &other.owned_out_, other.out_); other.out_ = nullptr; } + return *this; } @@ -169,6 +176,7 @@ void CsvWriter::WriteRow(const std::vector& row) const { } const std::string& value = row[i]; + if (!NeedsCsvQuotes(value)) { *out_ << value; continue; @@ -182,7 +190,6 @@ void CsvWriter::WriteRow(const std::vector& row) const { *out_ << ch; } } - *out_ << CsvQuote; } @@ -195,6 +202,7 @@ void CsvWriter::WriteRow(const std::vector& row) const { void CsvWriter::Flush() const { out_->flush(); + if (!*out_) { throw Error::Io("io", "failed to write csv"); } diff --git a/src/io/csv_batch.cpp b/src/io/csv_batch.cpp index 6e8d526..2250d00 100644 --- a/src/io/csv_batch.cpp +++ b/src/io/csv_batch.cpp @@ -12,6 +12,7 @@ static uint64_t AddChecked(const uint64_t current, const uint64_t add) { if (add > std::numeric_limits::max() - current) { throw Error::Overflow("io", "batch byte size overflow"); } + return current + add; } @@ -34,14 +35,17 @@ static uint64_t EstimateValueBytes(const ColumnType type, const std::string_view case ColumnType::String: return value.size(); } + throw Error::Unsupported("io", "unsupported column type"); } static uint64_t EstimateRowBytes(const Schema& schema, const std::vector& row) { uint64_t bytes = 0; + for (size_t i = 0; i < schema.columns.size(); ++i) { bytes = AddChecked(bytes, EstimateValueBytes(schema.columns[i].type, row[i])); } + return bytes; } @@ -145,6 +149,7 @@ CsvBatchWriter::CsvBatchWriter(const std::filesystem::path& path, Schema schema) void CsvBatchWriter::Write(const Batch& batch) { batch.Validate(); + if (batch.GetSchema() != schema_) { throw Error::InconsistentData("io", "batch schema mismatch"); } diff --git a/src/io/file.cpp b/src/io/file.cpp index 7965ba0..33af743 100644 --- a/src/io/file.cpp +++ b/src/io/file.cpp @@ -8,6 +8,7 @@ std::optional GetFileMetadata(const std::filesystem::path& path) { std::error_code ec; + if (!std::filesystem::exists(path, ec)) { if (ec) { throw Error::PathIo("io", path, "check existence"); @@ -23,6 +24,7 @@ std::optional GetFileMetadata(const std::filesystem::path& path) { metadata.is_regular = std::filesystem::is_regular_file(status); metadata.is_directory = std::filesystem::is_directory(status); + if (metadata.is_regular) { metadata.size = std::filesystem::file_size(path, ec); if (ec) { @@ -31,6 +33,7 @@ std::optional GetFileMetadata(const std::filesystem::path& path) { } metadata.last_write_time = std::filesystem::last_write_time(path, ec); + if (ec) { throw Error::PathIo("io", path, "read last write time"); } @@ -41,6 +44,7 @@ std::optional GetFileMetadata(const std::filesystem::path& path) { bool FileExists(const std::filesystem::path& path) { std::error_code ec; const bool exists = std::filesystem::exists(path, ec); + if (ec) { throw Error::PathIo("io", path, "check existence"); } @@ -50,12 +54,14 @@ bool FileExists(const std::filesystem::path& path) { void EnsureParentDirectory(const std::filesystem::path& path) { const auto parent = path.parent_path(); + if (parent.empty()) { return; } std::error_code ec; std::filesystem::create_directories(parent, ec); + if (ec) { throw Error::PathIo("io", path, "create parent directories"); } @@ -63,9 +69,11 @@ void EnsureParentDirectory(const std::filesystem::path& path) { std::ifstream OpenInputFile(const std::filesystem::path& path) { std::ifstream file(path, std::ios::binary); + if (!file.is_open()) { throw Error::PathIo("io", path, "open for read"); } + return file; } @@ -89,25 +97,31 @@ void InputFile::ReadAt(char* dst, const size_t size, const uint64_t offset) { std::string InputFile::ReadStringAt(const uint64_t offset, const size_t size) { std::string result(size, '\0'); + ReadAt(result.data(), result.size(), offset); + return result; } std::istream& InputFile::StreamAt(const uint64_t offset) { SeekInputFile(in_, path_, offset); + return in_; } std::ofstream OpenOutputFile(const std::filesystem::path& path, const FileOpenMode mode) { std::ofstream file(path, std::ios::binary | (mode == FileOpenMode::Append ? std::ios::app : std::ios::trunc)); + if (!file.is_open()) { throw Error::PathIo("io", path, mode == FileOpenMode::Append ? "open for append" : "open for write"); } + return file; } std::string ReadTextFile(const std::filesystem::path& path) { std::ifstream file = OpenInputFile(path); + return std::string(std::istreambuf_iterator(file), std::istreambuf_iterator()); } @@ -120,11 +134,13 @@ std::vector ReadFileBytes(const std::filesystem::path& path) { } const auto end_pos = file.tellg(); + if (end_pos < 0) { throw Error::PathIo("io", path, "read file size"); } std::vector data(end_pos); + if (end_pos == 0) { return data; } @@ -141,6 +157,7 @@ std::vector ReadFileBytes(const std::filesystem::path& path) { void WriteFileBytes(const std::filesystem::path& path, const std::span bytes) { std::ofstream file = OpenOutputFile(path); + if (!bytes.empty()) { file.write(reinterpret_cast(bytes.data()), bytes.size()); if (!file) { @@ -149,6 +166,7 @@ void WriteFileBytes(const std::filesystem::path& path, const std::span(size) || !in) { throw Error::Io("io", "failed to read bytes"); } @@ -16,7 +18,9 @@ void WriteBytes(std::ostream& out, const std::string_view bytes) { if (bytes.empty()) { return; } + out.write(bytes.data(), bytes.size()); + if (!out) { throw Error::Io("io", "failed to write bytes"); } diff --git a/src/model/batch.cpp b/src/model/batch.cpp index 1ed69b3..1e00be7 100644 --- a/src/model/batch.cpp +++ b/src/model/batch.cpp @@ -45,6 +45,7 @@ size_t Batch::RowsCount() const { if (columns_.empty()) { return 0; } + return columns_.front()->Size(); } @@ -102,18 +103,19 @@ void Batch::AppendRowsSelectedFromBatch(const Batch& source, const std::span data, const uint32_t row_count, const uint64_t size) const { if (column_index >= columns_.size()) { throw Error::OutOfRange("model", "column index out of range"); } - columns_[column_index]->ReadFrom(in, row_count, size); + columns_[column_index]->ReadFrom(data, row_count, size); } const Column& Batch::ColumnAt(const size_t i) const { if (i >= columns_.size()) { throw Error::OutOfRange("model", "column index out of range"); } + return *columns_[i]; } diff --git a/src/model/column.cpp b/src/model/column.cpp index c0058f4..cf6290e 100644 --- a/src/model/column.cpp +++ b/src/model/column.cpp @@ -2,6 +2,8 @@ #include +#include "common/error.h" +#include "common/parsing.h" #include "model/column_boolean.h" #include "model/column_character.h" #include "model/column_date.h" @@ -11,8 +13,6 @@ #include "model/column_int64.h" #include "model/column_string.h" #include "model/column_timestamp.h" -#include "common/error.h" -#include "common/parsing.h" std::unique_ptr CreateColumn(const ColumnType type) { switch (type) { @@ -43,6 +43,12 @@ Column::Column(const ColumnType type) : type_(type) {} Int128 Column::ValueAsInt128(const size_t row) const { return ParseInt128(ValueAsString(row)); } +std::string_view Column::ValueAsStringView(const size_t row, std::string& scratch) const { + scratch = ValueAsString(row); + + return scratch; +} + static bool MatchesValueComparison(const Int128 lhs, const Int128 rhs, const ValueComparison comparison) { switch (comparison) { case ValueComparison::Equal: diff --git a/src/model/column_string.cpp b/src/model/column_string.cpp index 82ce4b1..8a34198 100644 --- a/src/model/column_string.cpp +++ b/src/model/column_string.cpp @@ -1,7 +1,9 @@ #include "model/column_string.h" +#include #include #include +#include #include #include "common/error.h" @@ -10,58 +12,129 @@ constexpr char EncodedValueSeparator = ':'; -StringColumn::StringColumn() : MutableColumn(ColumnType::String) {} +StringColumn::StringColumn() : MutableColumn(ColumnType::String), offsets_{0} {} -size_t StringColumn::Size() const { return values_.size(); } +size_t StringColumn::Size() const { return offsets_.empty() ? 0 : offsets_.size() - 1; } -void StringColumn::Reserve(const size_t n) { values_.reserve(n); } +void StringColumn::Reserve(const size_t n) { offsets_.reserve(n + 1); } -void StringColumn::Clear() { values_.clear(); } +void StringColumn::Clear() { + blob_.clear(); + offsets_.clear(); + offsets_.push_back(0); +} -void StringColumn::AppendFromString(const std::string_view value) { values_.emplace_back(value); } +void StringColumn::AppendFromString(const std::string_view value) { AppendValue(value); } void StringColumn::AppendFromColumn(const Column& source, const size_t row) { if (source.Type() != ColumnType::String) { throw Error::InconsistentData(ModuleName(), "column type mismatch"); } - const auto& typed_source = static_cast(source); - CheckRowIndex(ModuleName(), row, typed_source.values_.size()); - values_.push_back(typed_source.values_[row]); + + if (&source == this) { + const std::string copy{StringAt(row)}; + AppendValue(copy); + + return; + } + + std::string scratch; + AppendValue(source.ValueAsStringView(row, scratch)); } void StringColumn::AppendRangeFromColumn(const Column& source, const size_t begin, const size_t count) { if (source.Type() != ColumnType::String) { throw Error::InconsistentData(ModuleName(), "column type mismatch"); } + const auto& typed_source = static_cast(source); - if (begin > typed_source.values_.size() || count > typed_source.values_.size() - begin) { + + if (begin > typed_source.Size() || count > typed_source.Size() - begin) { throw Error::OutOfRange(ModuleName(), "row range out of range"); } - values_.insert(values_.end(), typed_source.values_.begin() + static_cast(begin), - typed_source.values_.begin() + static_cast(begin + count)); + if (count == 0) { + return; + } + + const uint32_t source_begin = typed_source.offsets_[begin]; + const uint32_t source_end = typed_source.offsets_[begin + count]; + const size_t bytes_to_copy = source_end - source_begin; + CheckAppendSize(bytes_to_copy); + + if (&typed_source == this) { + const std::vector blob_slice(blob_.begin() + source_begin, blob_.begin() + source_end); + const std::vector offset_slice(offsets_.begin() + static_cast(begin), + offsets_.begin() + static_cast(begin + count + 1)); + + const uint32_t base = static_cast(blob_.size()); + blob_.insert(blob_.end(), blob_slice.begin(), blob_slice.end()); + offsets_.reserve(offsets_.size() + count); + + for (size_t i = 1; i <= count; ++i) { + offsets_.push_back(base + offset_slice[i] - source_begin); + } + + return; + } + + const uint32_t base = static_cast(blob_.size()); + blob_.insert(blob_.end(), typed_source.blob_.begin() + source_begin, typed_source.blob_.begin() + source_end); + offsets_.reserve(offsets_.size() + count); + + for (size_t i = 1; i <= count; ++i) { + offsets_.push_back(base + typed_source.offsets_[begin + i] - source_begin); + } } void StringColumn::AppendSelectedFromColumn(const Column& source, const std::span rows) { if (source.Type() != ColumnType::String) { throw Error::InconsistentData(ModuleName(), "column type mismatch"); } + const auto& typed_source = static_cast(source); - values_.reserve(values_.size() + rows.size()); + offsets_.reserve(offsets_.size() + rows.size()); + + if (&typed_source == this) { + std::vector snapshot; + snapshot.reserve(rows.size()); + + for (const size_t row : rows) { + snapshot.emplace_back(StringAt(row)); + } + + for (const std::string& value : snapshot) { + AppendValue(value); + } + + return; + } + for (const size_t row : rows) { - CheckRowIndex(ModuleName(), row, typed_source.values_.size()); - values_.push_back(typed_source.values_[row]); + AppendValue(typed_source.StringAt(row)); } } std::string StringColumn::ValueAsString(const size_t row) const { - CheckRowIndex(ModuleName(), row, values_.size()); - return values_[row]; + const std::string_view value = StringAt(row); + + return {value.data(), value.size()}; +} + +std::string_view StringColumn::ValueAsStringView(const size_t row, std::string& /*scratch*/) const { + return StringAt(row); } void StringColumn::SelectRowsByStringSet(const std::unordered_set& values, std::vector& rows) const { - for (size_t row = 0; row < values_.size(); ++row) { - if (values.contains(values_[row])) { + std::unordered_set value_views; + value_views.reserve(values.size()); + + for (const std::string& value : values) { + value_views.insert(value); + } + + for (size_t row = 0; row < Size(); ++row) { + if (value_views.contains(StringAt(row))) { rows.push_back(row); } } @@ -69,8 +142,8 @@ void StringColumn::SelectRowsByStringSet(const std::unordered_set& void StringColumn::SelectRowsByLikePattern(const std::string_view pattern, const bool negated, std::vector& rows) const { - for (size_t row = 0; row < values_.size(); ++row) { - const bool matched = LikeMatches(values_[row], pattern); + for (size_t row = 0; row < Size(); ++row) { + const bool matched = LikeMatches(StringAt(row), pattern); if (negated ? !matched : matched) { rows.push_back(row); } @@ -78,11 +151,10 @@ void StringColumn::SelectRowsByLikePattern(const std::string_view pattern, const } void StringColumn::AppendEncodedValue(const size_t row, std::string& out) const { - CheckRowIndex(ModuleName(), row, values_.size()); - const std::string& value = values_[row]; + const std::string_view value = StringAt(row); out += std::to_string(value.size()); out.push_back(EncodedValueSeparator); - out += value; + out.append(value.data(), value.size()); } std::unique_ptr StringColumn::Clone() const { return std::make_unique(*this); } @@ -90,33 +162,73 @@ std::unique_ptr StringColumn::Clone() const { return std::make_unique StringColumn::CloneMutable() const { return std::make_unique(*this); } void StringColumn::WriteTo(std::ostream& out) const { - for (const auto& value : values_) { + for (size_t row = 0; row < Size(); ++row) { + const std::string_view value = StringAt(row); + if (value.size() > std::numeric_limits::max()) { throw Error::Overflow(ModuleName(), "value exceeds supported size"); } + WriteStream(out, static_cast(value.size())); WriteBytes(out, value); } } -void StringColumn::ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) { - values_.clear(); - values_.reserve(row_count); +void StringColumn::ReadFrom(const std::span data, const uint32_t row_count, const uint64_t size) { + if (static_cast(data.size()) != size) { + throw Error::InconsistentData(ModuleName(), "column chunk size mismatch"); + } + + blob_.clear(); + blob_.reserve(data.size()); - uint64_t consumed = 0; + offsets_.clear(); + offsets_.reserve(static_cast(row_count) + 1); + offsets_.push_back(0); + + const char* ptr = data.empty() ? "" : data.data(); + const char* const end = ptr + data.size(); for (uint32_t row_index = 0; row_index < row_count; ++row_index) { - const uint32_t length = ReadStream(in); - consumed += sizeof(length); + if (static_cast(end - ptr) < sizeof(uint32_t)) { + throw Error::InconsistentData(ModuleName(), "column chunk size mismatch"); + } - std::string value(length, '\0'); - ReadBytes(in, value.data(), value.size()); + uint32_t length = 0; + std::memcpy(&length, ptr, sizeof(length)); + ptr += sizeof(length); + + if (static_cast(end - ptr) < length) { + throw Error::InconsistentData(ModuleName(), "column chunk size mismatch"); + } - consumed += length; - values_.push_back(std::move(value)); + AppendValue({ptr, length}); + ptr += length; } - if (consumed != size) { + if (ptr != end) { throw Error::InconsistentData(ModuleName(), "column chunk size mismatch"); } } + +std::string_view StringColumn::StringAt(const size_t row) const { + CheckRowIndex(ModuleName(), row, Size()); + const uint32_t begin = offsets_[row]; + const uint32_t end = offsets_[row + 1]; + + return {begin == end ? "" : blob_.data() + begin, end - begin}; +} + +void StringColumn::AppendValue(const std::string_view value) { + CheckAppendSize(value.size()); + blob_.insert(blob_.end(), value.begin(), value.end()); + offsets_.push_back(static_cast(blob_.size())); +} + +void StringColumn::CheckAppendSize(const size_t value_size) const { + constexpr size_t MaxBlobSize = std::numeric_limits::max(); + + if (value_size > MaxBlobSize || blob_.size() > MaxBlobSize - value_size) { + throw Error::Overflow(ModuleName(), "string column blob exceeds supported size"); + } +} diff --git a/src/model/metadata.cpp b/src/model/metadata.cpp index ff9d561..1f22941 100644 --- a/src/model/metadata.cpp +++ b/src/model/metadata.cpp @@ -63,6 +63,7 @@ static ColumnarMetadata ReadLegacyMetadata(std::istream& in) { group.row_count = ReadStream(in); const uint32_t columns_in_group = ReadStream(in); + if (columns_in_group != column_count) { throw Error::InconsistentData("model", "row group column count mismatch"); } @@ -89,6 +90,7 @@ static ColumnarMetadata ReadLegacyMetadata(std::istream& in) { for (auto& row_group : metadata.row_groups) { for (auto& column : row_group.columns) { column.has_min_max = ReadStream(in) != 0; + if (!column.has_min_max) { continue; } @@ -105,6 +107,7 @@ static ColumnarMetadata ReadV2Metadata(std::istream& in) { ColumnarMetadata metadata; const uint32_t version = ReadStream(in); + if (version != 2) { throw Error::MalformedData("model", "unsupported metadata version"); } @@ -132,6 +135,7 @@ static ColumnarMetadata ReadV2Metadata(std::istream& in) { group.row_count = ReadStream(in); const uint32_t columns_in_group = ReadStream(in); + if (columns_in_group != column_count) { throw Error::InconsistentData("model", "row group column count mismatch"); } @@ -154,6 +158,7 @@ static ColumnarMetadata ReadV2Metadata(std::istream& in) { for (auto& row_group : metadata.row_groups) { for (auto& column : row_group.columns) { column.has_min_max = ReadStream(in) != 0; + if (!column.has_min_max) { continue; } @@ -172,11 +177,14 @@ static ColumnarMetadata ReadV2Metadata(std::istream& in) { ColumnarMetadata ReadMetadata(std::istream& in) { const std::streampos start = in.tellg(); + if (start != std::streampos(-1)) { char magic[MetadataMagic.size()]; in.read(magic, sizeof(magic)); + if (in.gcount() == static_cast(MetadataMagic.size()) && std::equal(std::begin(magic), std::end(magic), MetadataMagic.begin())) { + return ReadV2Metadata(in); } @@ -227,6 +235,7 @@ void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { for (const auto& row_group : metadata.row_groups) { for (const auto& column : row_group.columns) { WriteStream(out, column.has_min_max ? 1 : 0); + if (!column.has_min_max) { continue; } diff --git a/src/sql_parser/tokenizer.cpp b/src/sql_parser/tokenizer.cpp index 6e2cb46..9fb44a7 100644 --- a/src/sql_parser/tokenizer.cpp +++ b/src/sql_parser/tokenizer.cpp @@ -8,11 +8,13 @@ static bool IsIdentifierStart(const char ch) { const auto uch = static_cast(ch); + return std::isalpha(uch) != 0 || ch == '_'; } static bool IsIdentifierPart(const char ch) { const auto uch = static_cast(ch); + return std::isalnum(uch) != 0 || ch == '_'; } @@ -28,6 +30,7 @@ Expected Tokenizer::GetNext() { } const char ch = query_[pos_]; + if (IsIdentifierStart(ch)) { const size_t start = pos_; ++pos_; @@ -125,6 +128,7 @@ Expected Tokenizer::GetNext() { return MakeToken(Tokens::Equal, "=", start); case '<': ++pos_; + if (pos_ < query_.size() && query_[pos_] == '=') { ++pos_; return MakeToken(Tokens::LessOrEqual, "<=", start); @@ -165,6 +169,7 @@ Expected> TokenizeSql(const std::string_view input) { while (true) { auto token = tokenizer.GetNext(); + if (!token.has_value()) { return tl::unexpected(token.error()); } diff --git a/tests/test_batch.cpp b/tests/test_batch.cpp index eb5f93d..f52fb26 100644 --- a/tests/test_batch.cpp +++ b/tests/test_batch.cpp @@ -3,11 +3,11 @@ #include #include +#include "common/error.h" #include "gtest/gtest.h" #include "io/columnar_batch.h" #include "io/csv.h" #include "io/csv_batch.h" -#include "common/error.h" #include "testing/temp_file.h" static_assert(std::is_copy_constructible_v); @@ -24,6 +24,7 @@ TEST(batch, usage_example_csv_to_columnar_roundtrip) { const TempFile csv_file("batch_usage_csv"); const TempFile columnar_file("batch_usage_columnar"); + WriteRows(csv_file.Path(), { {"1", "alpha"}, {"2", "beta"}, @@ -39,6 +40,7 @@ TEST(batch, usage_example_csv_to_columnar_roundtrip) { ColumnarBatchReader columnar_reader(columnar_file.Path()); auto batch = columnar_reader.ReadNext(); + ASSERT_TRUE(batch.has_value()); EXPECT_EQ(batch->RowsCount(), 2u); EXPECT_EQ(batch->ColumnAt(0).ValueAsString(0), "1"); @@ -67,12 +69,14 @@ TEST(batch, csv_reader_respects_max_rows) { CsvBatchReader reader(data_in.Path(), schema, sizing); auto first = reader.ReadNext(); + ASSERT_TRUE(first.has_value()); EXPECT_EQ(first->RowsCount(), 2u); EXPECT_EQ(first->ColumnAt(0).ValueAsString(0), "1"); EXPECT_EQ(first->ColumnAt(1).ValueAsString(1), "be,ta"); auto second = reader.ReadNext(); + ASSERT_TRUE(second.has_value()); EXPECT_EQ(second->RowsCount(), 1u); EXPECT_EQ(second->ColumnAt(0).ValueAsString(0), "3"); @@ -88,7 +92,7 @@ TEST(batch, write_batch_csv_writes_single_batch) { {"name", ColumnType::String}, }; - Batch batch(schema); + const Batch batch(schema); batch.AppendValueFromString(0, "1"); batch.AppendValueFromString(1, "alpha"); batch.AppendValueFromString(0, "2"); @@ -132,6 +136,7 @@ TEST(batch, columnar_roundtrip) { std::move(columnar_writer).Finalize(); ColumnarBatchReader columnar_reader(columnar_file.Path()); + EXPECT_EQ(columnar_reader.GetMetadata().row_groups.size(), 2u); std::vector> roundtrip_rows; @@ -164,11 +169,11 @@ TEST(batch, copy_is_deep) { {"name", ColumnType::String}, }; - Batch original(schema); + const Batch original(schema); original.AppendValueFromString(0, "1"); original.AppendValueFromString(1, "alpha"); - Batch copied = original; + const Batch copied = original; copied.AppendValueFromString(0, "2"); copied.AppendValueFromString(1, "beta"); @@ -203,6 +208,7 @@ TEST(batch, csv_reader_respects_max_values) { for (size_t i = 0; i < data_rows.size(); ++i) { auto batch = reader.ReadNext(); + ASSERT_TRUE(batch.has_value()); EXPECT_EQ(batch->RowsCount(), 1u); EXPECT_EQ(batch->ColumnAt(0).ValueAsString(0), data_rows[i][0]); @@ -234,6 +240,7 @@ TEST(batch, csv_reader_respects_max_bytes) { for (size_t i = 0; i < data_rows.size(); ++i) { auto batch = reader.ReadNext(); + ASSERT_TRUE(batch.has_value()); EXPECT_EQ(batch->RowsCount(), 1u); EXPECT_EQ(batch->ColumnAt(0).ValueAsString(0), data_rows[i][0]); @@ -272,6 +279,7 @@ TEST(batch, columnar_writer_rejects_after_finalize) { std::move(writer).Finalize(); Batch batch(schema); + EXPECT_THROW(writer.Write(batch), Error); EXPECT_THROW(std::move(writer).Finalize(), Error); } diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp index 9902a56..7f07a83 100644 --- a/tests/test_columnar.cpp +++ b/tests/test_columnar.cpp @@ -45,13 +45,16 @@ TEST(columnar, csv_to_columnar_and_back) { ConvertColumnarToCsv(columnar_file.Path(), schema_out.Path(), data_out.Path()); const auto [columns] = ReadSchemaCsv(schema_out.Path()); + ASSERT_EQ(columns.size(), schema_rows.size()); + for (size_t i = 0; i < schema_rows.size(); ++i) { EXPECT_EQ(columns[i].name, schema_rows[i][0]); EXPECT_EQ(ColumnTypeToString(columns[i].type), schema_rows[i][1]); } const auto data_roundtrip = ReadRows(data_out.Path()); + EXPECT_EQ(data_roundtrip, data_rows); } @@ -78,6 +81,7 @@ TEST(columnar, metadata_offsets_and_sizes) { ColumnarBatchReader reader(columnar_file.Path()); const auto& [schema, row_groups] = reader.GetMetadata(); + ASSERT_EQ(schema.columns.size(), 2u); EXPECT_EQ(schema.columns[0].name, "id"); EXPECT_EQ(schema.columns[0].type, ColumnType::Int64); @@ -93,10 +97,12 @@ TEST(columnar, metadata_offsets_and_sizes) { auto string_chunk_size = [&data_rows](const size_t start, const size_t count) -> uint64_t { uint64_t total = 0; + for (size_t i = 0; i < count; ++i) { total += sizeof(uint32_t); total += data_rows[start + i][1].size(); } + return total; }; @@ -122,6 +128,7 @@ TEST(columnar, metadata_offsets_and_sizes) { EXPECT_FALSE(row_groups[1].columns[1].has_min_max); uint64_t expected_offset = 0; + for (const auto& row_group : row_groups) { for (const auto& column : row_group.columns) { EXPECT_EQ(column.offset, expected_offset); @@ -130,6 +137,7 @@ TEST(columnar, metadata_offsets_and_sizes) { } const auto file_info = GetFileMetadata(columnar_file.Path()); + ASSERT_TRUE(file_info.has_value()); EXPECT_TRUE(file_info->is_regular); EXPECT_LT(expected_offset, file_info->size); @@ -232,6 +240,7 @@ TEST(columnar, metadata_records_minmax_stats_for_date_columns) { ColumnarBatchReader reader(columnar_file.Path()); const auto& row_groups = reader.GetMetadata().row_groups; + ASSERT_EQ(row_groups.size(), 2u); ASSERT_EQ(row_groups[0].columns.size(), 1u); ASSERT_EQ(row_groups[1].columns.size(), 1u); @@ -255,9 +264,11 @@ TEST(columnar, lz4_compressed_roundtrip) { WriteRows(schema_in.Path(), {{"id", "int64"}, {"payload", "string"}}); std::vector> data_rows; + for (int i = 0; i < 256; ++i) { data_rows.push_back({std::to_string(i), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); } + WriteRows(data_in.Path(), data_rows); ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 256, Compression::Lz4); @@ -265,6 +276,7 @@ TEST(columnar, lz4_compressed_roundtrip) { ColumnarBatchReader reader(columnar_file.Path()); const auto& row_group = reader.GetMetadata().row_groups.at(0); + ASSERT_EQ(row_group.columns.size(), 2u); EXPECT_EQ(ReadRows(data_out.Path()), data_rows); EXPECT_EQ(ReadRows(schema_out.Path()), @@ -285,6 +297,7 @@ TEST(columnar, lz4_falls_back_to_none_when_chunk_does_not_shrink) { const ColumnarBatchReader reader(columnar_file.Path()); const auto& chunk = reader.GetMetadata().row_groups.at(0).columns.at(0); + EXPECT_EQ(chunk.compression, Compression::None); EXPECT_EQ(chunk.compressed_size, sizeof(int64_t)); EXPECT_EQ(chunk.uncompressed_size, sizeof(int64_t)); @@ -311,7 +324,9 @@ TEST(columnar, read_legacy_metadata_without_compression_fields) { ASSERT_EQ(metadata.schema.columns.size(), 1u); ASSERT_EQ(metadata.row_groups.size(), 1u); + const auto& chunk = metadata.row_groups[0].columns[0]; + EXPECT_EQ(chunk.offset, 10u); EXPECT_EQ(chunk.compressed_size, 24u); EXPECT_EQ(chunk.uncompressed_size, 24u); diff --git a/tests/test_columns.cpp b/tests/test_columns.cpp index 56a895f..af019c0 100644 --- a/tests/test_columns.cpp +++ b/tests/test_columns.cpp @@ -2,16 +2,17 @@ #include #include +#include "common/error.h" +#include "common/parsing.h" +#include "gtest/gtest.h" #include "model/column.h" #include "model/column_int64.h" #include "model/column_string.h" -#include "common/error.h" -#include "gtest/gtest.h" -#include "common/parsing.h" static void ExpectColumnRoundtrip(const ColumnType type, const std::vector& values, const std::vector& expected_values = {}) { auto column = CreateColumn(type); + for (const auto& value : values) { column->AppendFromString(value); } @@ -21,12 +22,14 @@ static void ExpectColumnRoundtrip(const ColumnType type, const std::vectorReadFrom(in, values.size(), bytes.size()); + read_back->ReadFrom({bytes.data(), bytes.size()}, values.size(), bytes.size()); ASSERT_EQ(read_back->Size(), values.size()); + const auto& expected = expected_values.empty() ? values : expected_values; + ASSERT_EQ(expected.size(), values.size()); + for (size_t i = 0; i < expected.size(); ++i) { EXPECT_EQ(read_back->ValueAsString(i), expected[i]); } @@ -42,8 +45,7 @@ TEST(columns, int64_roundtrip) { const std::string bytes = buffer.str(); Int64Column read_back; - std::stringstream in(bytes); - read_back.ReadFrom(in, 2, bytes.size()); + read_back.ReadFrom({bytes.data(), bytes.size()}, 2, bytes.size()); EXPECT_EQ(read_back.Size(), 2u); EXPECT_EQ(read_back.ValueAsString(0), "10"); @@ -61,8 +63,7 @@ TEST(columns, string_roundtrip) { const std::string bytes = buffer.str(); StringColumn read_back; - std::stringstream in(bytes); - read_back.ReadFrom(in, 3, bytes.size()); + read_back.ReadFrom({bytes.data(), bytes.size()}, 3, bytes.size()); EXPECT_EQ(read_back.Size(), 3u); EXPECT_EQ(read_back.ValueAsString(0), "alpha"); @@ -72,12 +73,14 @@ TEST(columns, string_roundtrip) { TEST(columns, int64_invalid_value_throws) { Int64Column values; + EXPECT_THROW(values.AppendFromString("not_a_number"), Error); } TEST(columns, string_out_of_range_throws) { StringColumn values; values.AppendFromString("alpha"); + EXPECT_THROW(values.ValueAsString(1), Error); } @@ -85,14 +88,12 @@ TEST(columns, supported_scalar_types_roundtrip) { ExpectColumnRoundtrip(ColumnType::Boolean, {"true", "false", "1"}, {"true", "false", "true"}); ExpectColumnRoundtrip(ColumnType::Int16, {"-32768", "0", "32767"}); ExpectColumnRoundtrip(ColumnType::Int32, {"-2147483648", "7", "2147483647"}); - ExpectColumnRoundtrip( - ColumnType::Int128, - {"-170141183460469231731687303715884105728", "0", "170141183460469231731687303715884105727"}); + ExpectColumnRoundtrip(ColumnType::Int128, + {"-170141183460469231731687303715884105728", "0", "170141183460469231731687303715884105727"}); ExpectColumnRoundtrip(ColumnType::Date, {"1970-01-01", "2024-02-29", "2030-12-31"}); - ExpectColumnRoundtrip( - ColumnType::Timestamp, - {"1970-01-01 00:00:00", "2024-02-29 12:34:56.123456", "2030-12-31T23:59:59.5"}, - {"1970-01-01 00:00:00", "2024-02-29 12:34:56.123456", "2030-12-31 23:59:59.5"}); + ExpectColumnRoundtrip(ColumnType::Timestamp, + {"1970-01-01 00:00:00", "2024-02-29 12:34:56.123456", "2030-12-31T23:59:59.5"}, + {"1970-01-01 00:00:00", "2024-02-29 12:34:56.123456", "2030-12-31 23:59:59.5"}); ExpectColumnRoundtrip(ColumnType::Character, {"A", ",", "\n"}); } @@ -121,18 +122,48 @@ TEST(columns, scalar_invalid_values_throw) { TEST(columns, type_names_roundtrip) { const std::vector types = { - ColumnType::Boolean, - ColumnType::Int16, - ColumnType::Int32, - ColumnType::Int64, - ColumnType::Int128, - ColumnType::String, - ColumnType::Date, - ColumnType::Timestamp, - ColumnType::Character, + ColumnType::Boolean, ColumnType::Int16, ColumnType::Int32, ColumnType::Int64, ColumnType::Int128, + ColumnType::String, ColumnType::Date, ColumnType::Timestamp, ColumnType::Character, }; for (const auto type : types) { EXPECT_EQ(ParseColumnType(ColumnTypeToString(type)), type); } } + +TEST(columns, string_self_append_does_not_dangle) { + StringColumn column; + column.AppendFromString("alpha"); + column.AppendFromString("beta"); + column.AppendFromString("gamma"); + + column.AppendFromColumn(column, 0); + column.AppendRangeFromColumn(column, 1, 2); + const std::vector rows = {2, 0}; + column.AppendSelectedFromColumn(column, rows); + + const std::vector expected = {"alpha", "beta", "gamma", "alpha", "beta", "gamma", "gamma", "alpha"}; + + ASSERT_EQ(column.Size(), expected.size()); + + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(column.ValueAsString(i), expected[i]); + } +} + +TEST(columns, fixed_self_append_range_does_not_dangle) { + Int64Column column; + for (int i = 0; i < 4; ++i) { + column.AppendFromString(std::to_string(i)); + } + + column.AppendRangeFromColumn(column, 1, 3); + + const std::vector expected = {0, 1, 2, 3, 1, 2, 3}; + + ASSERT_EQ(column.Size(), expected.size()); + + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(column.ValueAsInt128(i), static_cast(expected[i])); + } +} diff --git a/tests/test_csv.cpp b/tests/test_csv.cpp index c892da5..4fd184c 100644 --- a/tests/test_csv.cpp +++ b/tests/test_csv.cpp @@ -3,8 +3,8 @@ #include #include -#include "io/csv.h" #include "gtest/gtest.h" +#include "io/csv.h" static_assert(!std::is_copy_constructible_v); static_assert(!std::is_copy_assignable_v); @@ -24,6 +24,7 @@ TEST(csv, read_write_roundtrip_with_quotes) { writer.WriteRow({"", "plain"}); std::vector row; + ASSERT_TRUE(reader.ReadRow(row)); EXPECT_EQ(row, (std::vector{"1", "2,3", "he\"llo", "line1\nline2"})); @@ -51,6 +52,7 @@ TEST(csv, moved_reader_and_writer_remain_usable) { std::stringstream buffer; CsvWriter writer(buffer); CsvWriter moved_writer(std::move(writer)); + moved_writer.WriteRow({"1", "value"}); moved_writer.Flush(); @@ -59,6 +61,7 @@ TEST(csv, moved_reader_and_writer_remain_usable) { CsvReader moved_reader(std::move(reader)); std::vector row; + ASSERT_TRUE(moved_reader.ReadRow(row)); EXPECT_EQ(row, (std::vector{"1", "value"})); EXPECT_FALSE(moved_reader.ReadRow(row)); diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index 03dd56f..a21821a 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -45,6 +45,7 @@ static Batch BuildHitsTable(const std::string_view query) { executor.RegisterTable("hits", columnar_file.Path()); auto result = executor.Execute(query); + if (!result.has_value()) { throw result.error(); } @@ -54,6 +55,7 @@ static Batch BuildHitsTable(const std::string_view query) { TEST(executor, usage_example_register_table_and_execute_query) { const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); EXPECT_EQ(SingleRowValues(batch), std::vector{"5"}); } @@ -63,14 +65,17 @@ TEST(executor, groups_rows_and_orders_by_aggregate_descending) { "SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;"); EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + auto actual_rows = BatchRows(batch); auto expected_rows = std::vector>{ {"7", "2"}, {"1", "1"}, {"3", "1"}, }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); } @@ -97,17 +102,21 @@ TEST(executor, encodes_multicolumn_group_keys_with_length_prefix) { executor.RegisterTable("events", columnar_file.Path()); auto result = executor.Execute("SELECT KeyLeft, KeyRight, COUNT(*) FROM events GROUP BY KeyLeft, KeyRight;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); EXPECT_EQ(BatchColumnNames(result.value()), (std::vector{"KeyLeft", "KeyRight", "COUNT(*)"})); + auto actual_rows = BatchRows(result.value()); auto expected_rows = std::vector>{ {"a", "bc", "1"}, {"a|", "bc", "2"}, {"ab", "c", "2"}, }; + std::ranges::sort(actual_rows); std::ranges::sort(expected_rows); + EXPECT_EQ(actual_rows, expected_rows); } @@ -117,14 +126,17 @@ TEST(executor, supports_from_with_as_alias_and_qualified_columns) { "COUNT(*) DESC;"); EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + auto actual_rows = BatchRows(batch); auto expected_rows = std::vector>{ {"7", "2"}, {"1", "1"}, {"3", "1"}, }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); } @@ -134,14 +146,17 @@ TEST(executor, supports_from_with_bare_alias) { "COUNT(*) DESC;"); EXPECT_EQ(BatchColumnNames(batch), (std::vector{"AdvEngineID", "COUNT(*)"})); + auto actual_rows = BatchRows(batch); auto expected_rows = std::vector>{ {"7", "2"}, {"1", "1"}, {"3", "1"}, }; + std::ranges::sort(actual_rows.begin() + 1, actual_rows.end()); std::ranges::sort(expected_rows.begin() + 1, expected_rows.end()); + EXPECT_EQ(actual_rows, expected_rows); } @@ -157,13 +172,16 @@ TEST(executor, supports_select_alias_order_by_alias_and_limit) { "SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 2;"); EXPECT_EQ(BatchColumnNames(batch), (std::vector{"RegionID", "u"})); + auto actual_rows = BatchRows(batch); auto expected_rows = std::vector>{ {"10", "2"}, {"20", "2"}, }; + std::ranges::sort(actual_rows); std::ranges::sort(expected_rows); + EXPECT_EQ(actual_rows, expected_rows); } @@ -227,10 +245,10 @@ TEST(executor, supports_clickbench_star_order_by_projected_column) { executor.RegisterTable("hits", columnar_file.Path()); auto result = executor.Execute("SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime ASC LIMIT 2;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); - EXPECT_EQ(BatchColumnNames(result.value()), - (std::vector{"WatchID", "EventTime", "URL", "Title"})); + EXPECT_EQ(BatchColumnNames(result.value()), (std::vector{"WatchID", "EventTime", "URL", "Title"})); EXPECT_EQ(BatchRows(result.value()), (std::vector>{ {"3", "2024-01-01 00:00:00", "https://google.com/b", "second"}, {"2", "2024-01-03 00:00:00", "http://google.com/a", "first"}, @@ -249,8 +267,10 @@ TEST(executor, supports_multiple_aggregates_with_alias_and_limit) { {"10", "3", "2", "200", "2"}, {"20", "14", "2", "350", "2"}, }; + std::ranges::sort(actual_rows); std::ranges::sort(expected_rows); + EXPECT_EQ(actual_rows, expected_rows); } @@ -281,6 +301,7 @@ TEST(executor, supports_top_k_group_by_multiple_numeric_keys_with_compact_aggreg auto result = executor.Execute( "SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, " "ClientIP ORDER BY c DESC LIMIT 2;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); EXPECT_EQ(BatchColumnNames(result.value()), @@ -294,12 +315,14 @@ TEST(executor, supports_top_k_group_by_multiple_numeric_keys_with_compact_aggreg TEST(executor, executes_basic_aggregate_queries) { { const Batch batch = BuildHitsTable("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(*)"}); EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { const Batch batch = BuildHitsTable("SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"SUM(AdvEngineID)", "COUNT(*)", "AVG(ResolutionWidth)"})); EXPECT_EQ(SingleRowValues(batch), (std::vector{"18", "5", "300"})); @@ -307,30 +330,35 @@ TEST(executor, executes_basic_aggregate_queries) { { const Batch batch = BuildHitsTable("SELECT AVG(UserID) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"AVG(UserID)"}); EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { const Batch batch = BuildHitsTable("SELECT COUNT(DISTINCT UserID) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(DISTINCT UserID)"}); EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { const Batch batch = BuildHitsTable("SELECT COUNT(DISTINCT SearchPhrase) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), std::vector{"COUNT(DISTINCT SearchPhrase)"}); EXPECT_EQ(SingleRowValues(batch), std::vector{"4"}); } { const Batch batch = BuildHitsTable("SELECT MIN(EventDate), MAX(EventDate) FROM hits;"); + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"MIN(EventDate)", "MAX(EventDate)"})); EXPECT_EQ(SingleRowValues(batch), (std::vector{"2024-01-01", "2024-01-09"})); } { const Batch batch = BuildHitsTable("select sum(AdvEngineID), count(*) from hits where SearchPhrase = 'alpha';"); + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"SUM(AdvEngineID)", "COUNT(*)"})); EXPECT_EQ(SingleRowValues(batch), (std::vector{"3", "2"})); } @@ -353,6 +381,7 @@ TEST(executor, rejects_unknown_aggregate_functions) { executor.RegisterTable("hits", columnar_file.Path()); const ExecuteExpected result = executor.Execute("SELECT median(Value) FROM hits;"); + ASSERT_FALSE(result.has_value()); EXPECT_EQ(result.error().GetCode(), Error::Code::Unsupported); } @@ -380,6 +409,7 @@ TEST(executor, supports_group_by_computed_numeric_and_timestamp_keys) { { auto result = executor.Execute( "SELECT ClientIP - 1, COUNT(*) AS c FROM hits GROUP BY ClientIP - 1 ORDER BY c DESC, ClientIP - 1 ASC;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); EXPECT_EQ(BatchRows(result.value()), (std::vector>{ {"9", "2"}, @@ -390,6 +420,7 @@ TEST(executor, supports_group_by_computed_numeric_and_timestamp_keys) { { auto result = executor.Execute( "SELECT extract(minute FROM EventTime) AS m, COUNT(*) AS c FROM hits GROUP BY m ORDER BY m ASC;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); EXPECT_EQ(BatchRows(result.value()), (std::vector>{ {"3", "2"}, @@ -400,6 +431,7 @@ TEST(executor, supports_group_by_computed_numeric_and_timestamp_keys) { { auto result = executor.Execute( "SELECT DATE_TRUNC('minute', EventTime) AS m, COUNT(*) AS c FROM hits GROUP BY m ORDER BY m ASC;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); EXPECT_EQ(BatchRows(result.value()), (std::vector>{ {"2024-01-01 12:03:00", "2"}, @@ -428,10 +460,12 @@ TEST(executor, prunes_row_groups_for_typed_where_filters_before_reading_chunks) ColumnarBatchReader reader(columnar_file.Path()); ColumnarMetadata metadata = reader.GetMetadata(); + ASSERT_EQ(metadata.row_groups.size(), 2u); ASSERT_EQ(metadata.row_groups[1].columns.size(), 2u); const auto file_info = GetFileMetadata(columnar_file.Path()); + ASSERT_TRUE(file_info.has_value()); metadata.row_groups[1].columns[0].offset = file_info->size + 1024; @@ -441,6 +475,7 @@ TEST(executor, prunes_row_groups_for_typed_where_filters_before_reading_chunks) const std::string metadata_blob = metadata_stream.str(); std::vector bytes = ReadFileBytes(columnar_file.Path()); + ASSERT_GE(bytes.size(), sizeof(uint64_t) + 4u); const size_t footer_offset = bytes.size() - sizeof(uint64_t) - 4u; @@ -463,6 +498,7 @@ TEST(executor, prunes_row_groups_for_typed_where_filters_before_reading_chunks) executor.RegisterTable("hits", columnar_file.Path()); auto result = executor.Execute("SELECT COUNT(*) FROM hits WHERE EventDate = '2024-01-01';"); + ASSERT_TRUE(result.has_value()) << result.error().what(); EXPECT_EQ(SingleRowValues(result.value()), std::vector{"2"}); } diff --git a/tests/test_fileio.cpp b/tests/test_fileio.cpp index d45683e..0c808c5 100644 --- a/tests/test_fileio.cpp +++ b/tests/test_fileio.cpp @@ -3,9 +3,9 @@ #include #include +#include "common/error.h" #include "gtest/gtest.h" #include "io/file.h" -#include "common/error.h" #include "testing/temp_file.h" TEST(fileio, write_and_read_roundtrip) { @@ -27,11 +27,14 @@ TEST(fileio, append_bytes) { std::ofstream out = OpenOutputFile(temp.Path(), FileOpenMode::Append); out.write(reinterpret_cast(extra.data()), extra.size()); ASSERT_TRUE(out.good()); + out.flush(); + ASSERT_TRUE(out.good()); const auto read_back = ReadFileBytes(temp.Path()); const std::vector expected = {10, 11, 12, 13}; + EXPECT_EQ(read_back, expected); } @@ -42,6 +45,7 @@ TEST(fileio, metadata_for_file) { WriteFileBytes(temp.Path(), payload); const auto metadata = GetFileMetadata(temp.Path()); + ASSERT_TRUE(metadata.has_value()); EXPECT_TRUE(metadata->is_regular); EXPECT_FALSE(metadata->is_directory); @@ -67,8 +71,11 @@ TEST(fileio, read_text_roundtrip) { std::ofstream out = OpenOutputFile(temp.Path()); out << payload; + ASSERT_TRUE(out.good()); + out.flush(); + ASSERT_TRUE(out.good()); EXPECT_EQ(ReadTextFile(temp.Path()), payload); @@ -76,6 +83,7 @@ TEST(fileio, read_text_roundtrip) { TEST(fileio, file_exists_checks) { const TempFile temp("fileio_exists"); + EXPECT_FALSE(FileExists(temp.Path())); WriteFileBytes(temp.Path(), std::vector{1}); @@ -88,6 +96,7 @@ TEST(fileio, ensure_parent_directory_creates_missing_directories) { const auto parent = file_path.parent_path(); ASSERT_FALSE(std::filesystem::exists(parent)); + EnsureParentDirectory(file_path); EXPECT_TRUE(std::filesystem::exists(parent)); EXPECT_TRUE(std::filesystem::is_directory(parent)); @@ -99,13 +108,17 @@ TEST(fileio, stream_roundtrip) { std::ofstream out = OpenOutputFile(temp.Path()); out.write(reinterpret_cast(payload.data()), payload.size()); + ASSERT_TRUE(out.good()); + out.flush(); + ASSERT_TRUE(out.good()); std::ifstream in = OpenInputFile(temp.Path()); std::vector read_back(payload.size()); in.read(reinterpret_cast(read_back.data()), read_back.size()); + ASSERT_EQ(static_cast(in.gcount()), read_back.size()); EXPECT_EQ(read_back, payload); } @@ -119,12 +132,16 @@ TEST(fileio, stream_append) { std::ofstream out = OpenOutputFile(temp.Path(), FileOpenMode::Append); out.write(reinterpret_cast(extra.data()), extra.size()); + ASSERT_TRUE(out.good()); + out.flush(); + ASSERT_TRUE(out.good()); const auto read_back = ReadFileBytes(temp.Path()); const std::vector expected = {1, 2, 3, 4, 5}; + EXPECT_EQ(read_back, expected); } @@ -135,13 +152,17 @@ TEST(fileio, stream_seek_and_tell) { WriteFileBytes(temp.Path(), payload); std::ifstream in = OpenInputFile(temp.Path()); + EXPECT_EQ(in.tellg(), std::streampos(0)); + in.seekg(std::streampos(2)); + ASSERT_TRUE(in.good()); EXPECT_EQ(in.tellg(), std::streampos(2)); uint8_t value = 0; in.read(reinterpret_cast(&value), 1); + ASSERT_EQ(in.gcount(), 1); EXPECT_EQ(value, payload[2]); } @@ -153,6 +174,7 @@ TEST(fileio, input_file_reads_at_offsets) { WriteFileBytes(temp.Path(), payload); InputFile file(temp.Path()); + EXPECT_EQ(file.ReadAt(3), payload[3]); EXPECT_EQ(file.ReadStringAt(1, 3), std::string("\x0B\x0C\x0D", 3)); } diff --git a/tests/test_tokenizer.cpp b/tests/test_tokenizer.cpp index 0e9d919..d8e3a8f 100644 --- a/tests/test_tokenizer.cpp +++ b/tests/test_tokenizer.cpp @@ -21,17 +21,17 @@ TEST(sql_tokenizer, tokenizes_select_query) { TokenizeSql("SELECT DISTINCT name, COUNT(*), LENGTH(city) FROM sales GROUP BY name ORDER BY name LIMIT 10;"); ASSERT_TRUE(result.has_value()); + const auto& tokens = result.value(); - EXPECT_EQ( - CollectTypes(tokens), - (std::vector{ - Tokens::Select, Tokens::Distinct, Tokens::NameToken, Tokens::Comma, Tokens::Count, - Tokens::OpenBracket, Tokens::Asterisk, Tokens::CloseBracket, Tokens::Comma, Tokens::Length, - Tokens::OpenBracket, Tokens::NameToken, Tokens::CloseBracket, Tokens::From, Tokens::NameToken, - Tokens::Group, Tokens::By, Tokens::NameToken, Tokens::Order, Tokens::By, - Tokens::NameToken, Tokens::Limit, Tokens::NumericLiteral, Tokens::Semicolon, Tokens::EndOfInput, - })); + EXPECT_EQ(CollectTypes(tokens), + (std::vector{ + Tokens::Select, Tokens::Distinct, Tokens::NameToken, Tokens::Comma, Tokens::Count, + Tokens::OpenBracket, Tokens::Asterisk, Tokens::CloseBracket, Tokens::Comma, Tokens::Length, + Tokens::OpenBracket, Tokens::NameToken, Tokens::CloseBracket, Tokens::From, Tokens::NameToken, + Tokens::Group, Tokens::By, Tokens::NameToken, Tokens::Order, Tokens::By, + Tokens::NameToken, Tokens::Limit, Tokens::NumericLiteral, Tokens::Semicolon, Tokens::EndOfInput, + })); ASSERT_EQ(tokens.size(), 25U); EXPECT_NE(std::dynamic_pointer_cast(tokens[0]), nullptr); @@ -49,6 +49,7 @@ TEST(sql_tokenizer, tokenizes_create_with_literals_and_case_insensitive_keywords const auto result = TokenizeSql("create report as SeLeCt 'abc''def', total_sum - 10.5 FROM source_table"); ASSERT_TRUE(result.has_value()); + const auto& tokens = result.value(); EXPECT_EQ(CollectTypes(tokens), (std::vector{ @@ -81,7 +82,9 @@ TEST(sql_tokenizer, tokenizer_returns_tokens_one_by_one) { std::vector types; while (true) { auto token = tokenizer.GetNext(); + ASSERT_TRUE(token.has_value()); + types.push_back(token.value()->GetType()); if (token.value()->GetType() == Tokens::EndOfInput) { @@ -108,10 +111,12 @@ TEST(sql_tokenizer, tokenizer_returns_tokens_one_by_one) { TEST(sql_tokenizer, rejects_invalid_input) { const auto unterminated = TokenizeSql("SELECT 'oops"); + ASSERT_FALSE(unterminated.has_value()); EXPECT_EQ(unterminated.error().GetCode(), Error::Code::MalformedData); const auto invalid_character = TokenizeSql("SELECT @oops"); + ASSERT_FALSE(invalid_character.has_value()); EXPECT_EQ(invalid_character.error().GetCode(), Error::Code::MalformedData); }