diff --git a/README.md b/README.md index b094083..4456393 100644 --- a/README.md +++ b/README.md @@ -3,26 +3,130 @@ Колононый движок с нуля (in progress) -## Dashboard - - -| Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | -| --- | ---: | ---: | ---: | ---: | ---: | --- | -| [0](benchmarks/queries/query_0.sql) | 1076.34 | 11.43 | 11.67 | 9.57 | 12.81 | ok | -| [1](benchmarks/queries/query_1.sql) | 41.93 | 27.20 | 27.21 | 27.19 | 27.22 | ok | -| [2](benchmarks/queries/query_2.sql) | 64.18 | 51.76 | 51.70 | 51.57 | 52.08 | ok | -| [3](benchmarks/queries/query_3.sql) | 69.35 | 51.94 | 52.19 | 51.02 | 52.38 | ok | -| [4](benchmarks/queries/query_4.sql) | 44.52 | 43.83 | 43.77 | 43.72 | 44.07 | ok | -| [5](benchmarks/queries/query_5.sql) | 59.39 | 48.10 | 45.38 | 44.72 | 56.93 | ok | -| [6](benchmarks/queries/query_6.sql) | 757.49 | 679.54 | 678.98 | 676.05 | 684.13 | ok | -| [7](benchmarks/queries/query_7.sql) | 28.79 | 28.27 | 28.27 | 28.13 | 28.44 | ok | -| [8](benchmarks/queries/query_8.sql) | 126.09 | 112.90 | 112.57 | 111.87 | 114.58 | ok | -| [9](benchmarks/queries/query_9.sql) | 164.71 | 166.90 | 166.97 | 164.56 | 169.11 | ok | -| [10](benchmarks/queries/query_10.sql) | 51.34 | 37.20 | 37.17 | 37.06 | 37.38 | ok | -| [11](benchmarks/queries/query_11.sql) | 50.58 | 38.50 | 38.49 | 38.39 | 38.65 | ok | -| [12](benchmarks/queries/query_12.sql) | 63.54 | 64.18 | 64.19 | 63.69 | 64.67 | ok | -| [13](benchmarks/queries/query_13.sql) | 96.63 | 83.26 | 76.40 | 76.12 | 104.12 | ok | -| [14](benchmarks/queries/query_14.sql) | 83.27 | 113.13 | 87.14 | 72.80 | 205.45 | ok | -| [15](benchmarks/queries/query_15.sql) | 105.19 | 108.28 | 108.08 | 104.50 | 112.46 | ok | -| [16](benchmarks/queries/query_16.sql) | 204.87 | 201.82 | 200.27 | 199.42 | 207.34 | ok | +## Benchmark Dashboard + +`generated 2026-05-28 03:03:29` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) + +### Summary + +| Metric | Value | +| --- | ---: | +| queries ok | 43 / 43 | +| queries failed | 0 | +| median warm time | 237.07 ms | +| average warm time | 268.20 ms | +| p95 warm time | 777.08 ms | +| cold/warm delta | 11.3% | +| total output size | 20.3 KB | +| max output size | 6.0 KB | +| fastest query | Q06 · 3.68 ms | +| slowest query | Q18 · 1112.19 ms | + +### Storage + +| Metric | Value | +| --- | ---: | +| source csv | `benchmarks/hits_sample.csv` | +| schema | `benchmarks/scheme.csv` | +| source size | 802.6 MB | +| columnar size | 694.9 MB | +| roundtrip csv size | 743.0 MB | +| compression ratio | 1.16x | +| columnar / csv | 86.6% | +| csv -> columnar | 5.77s | +| columnar -> csv | 9.67s | +| convert throughput | 139.2 MB/s | +| roundtrip throughput | 71.9 MB/s | + +### Heatmap + +`🟩` быстрее медианы · `🟦` около медианы · `🟥` медленнее медианы · `⬜` 0 ms + +| slot 1 | slot 2 | slot 3 | slot 4 | slot 5 | slot 6 | +| ---: | ---: | ---: | ---: | ---: | ---: | +| Q00 🟩 `3.91` | Q01 🟩 `6.32` | Q02 🟩 `17.25` | Q03 🟩 `13.37` | Q04 🟩 `148.12` | Q05 🟩 `130.96` | +| Q06 🟩 `3.68` | Q07 🟩 `6.69` | Q08 🟩 `193.36` | Q09 🟩 `209.45` | Q10 🟥 `268.08` | Q11 🟥 `269.48` | +| Q12 🟦 `233.32` | Q13 🟦 `245.55` | Q14 🟦 `238.18` | Q15 🟩 `73.32` | Q16 🟩 `122.39` | Q17 🟩 `144.28` | +| Q18 🟥 `1112.19` | Q19 🟩 `4.11` | Q20 🟩 `134.97` | Q21 🟥 `268.11` | Q22 🟥 `357.07` | Q23 🟦 `236.84` | +| Q24 🟥 `273.73` | Q25 🟦 `226.36` | Q26 🟥 `289.97` | Q27 🟥 `398.98` | Q28 🟥 `907.32` | Q29 🟥 `414.52` | +| Q30 🟦 `237.07` | Q31 🟥 `262.05` | Q32 🟥 `787.69` | Q33 🟥 `419.25` | Q34 🟥 `423.99` | Q35 🟥 `681.60` | +| Q36 🟥 `337.69` | Q37 🟦 `244.01` | Q38 🟩 `67.28` | Q39 🟥 `426.31` | Q40 🟩 `92.18` | Q41 🟩 `33.18` | +| Q42 🟥 `568.52` | | | | | | +### Largest Result Sets + +| Query | Output | Warm median, ms | SQL | +| --- | ---: | ---: | --- | +| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 357.07 | SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT… | +| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 236.84 | SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime… | +| [Q29](benchmarks/queries/query_29.sql) | 990 B | 414.52 | SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(Reso… | +| [Q39](benchmarks/queries/query_39.sql) | 835 B | 426.31 | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (… | +| [Q24](benchmarks/queries/query_24.sql) | 658 B | 273.73 | SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY… | + +### Fastest Queries + +| Query | SQL | Warm median, ms | First run, ms | +| --- | --- | ---: | ---: | +| [Q06](benchmarks/queries/query_6.sql) | SELECT MIN(EventDate), MAX(EventDate) FROM hits; | 3.68 | 4.43 | +| [Q00](benchmarks/queries/query_0.sql) | SELECT COUNT(*) FROM hits; | 3.91 | 8.14 | +| [Q19](benchmarks/queries/query_19.sql) | SELECT UserID FROM hits WHERE UserID = 435090932899640449; | 4.11 | 4.50 | +| [Q01](benchmarks/queries/query_1.sql) | SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; | 6.32 | 34.17 | +| [Q07](benchmarks/queries/query_7.sql) | SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY … | 6.69 | 6.40 | + +### Slowest Queries + +| Query | SQL | Warm median, ms | Warm max, ms | +| --- | --- | ---: | ---: | +| [Q18](benchmarks/queries/query_18.sql) | SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT… | 1112.19 | 1178.59 | +| [Q28](benchmarks/queries/query_28.sql) | SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1')… | 907.32 | 910.64 | +| [Q32](benchmarks/queries/query_32.sql) | SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(Resolution… | 787.69 | 790.48 | +| [Q35](benchmarks/queries/query_35.sql) | SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS … | 681.60 | 683.99 | +| [Q42](benchmarks/queries/query_42.sql) | SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM… | 568.52 | 569.02 | + +### Query Table + +| Query | Output CSV | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| [Q00](benchmarks/queries/query_0.sql) | 7 B | 8.14 | 3.95 | 3.91 | 3.78 | 4.22 | ok | +| [Q01](benchmarks/queries/query_1.sql) | 6 B | 34.17 | 6.33 | 6.32 | 6.01 | 6.67 | ok | +| [Q02](benchmarks/queries/query_2.sql) | 18 B | 48.00 | 17.26 | 17.25 | 16.83 | 17.71 | ok | +| [Q03](benchmarks/queries/query_3.sql) | 20 B | 49.77 | 13.43 | 13.37 | 13.21 | 13.76 | ok | +| [Q04](benchmarks/queries/query_4.sql) | 6 B | 148.05 | 149.27 | 148.12 | 147.47 | 153.38 | ok | +| [Q05](benchmarks/queries/query_5.sql) | 6 B | 165.31 | 130.88 | 130.96 | 130.57 | 131.02 | ok | +| [Q06](benchmarks/queries/query_6.sql) | 22 B | 4.43 | 3.69 | 3.68 | 3.60 | 3.82 | ok | +| [Q07](benchmarks/queries/query_7.sql) | 31 B | 6.40 | 6.80 | 6.69 | 6.46 | 7.35 | ok | +| [Q08](benchmarks/queries/query_8.sql) | 81 B | 230.36 | 193.81 | 193.36 | 191.19 | 197.33 | ok | +| [Q09](benchmarks/queries/query_9.sql) | 237 B | 210.07 | 210.02 | 209.45 | 209.09 | 212.08 | ok | +| [Q10](benchmarks/queries/query_10.sql) | 102 B | 299.72 | 268.08 | 268.08 | 267.32 | 268.86 | ok | +| [Q11](benchmarks/queries/query_11.sql) | 116 B | 287.55 | 270.30 | 269.48 | 268.57 | 273.67 | ok | +| [Q12](benchmarks/queries/query_12.sql) | 387 B | 229.11 | 232.94 | 233.32 | 230.58 | 234.54 | ok | +| [Q13](benchmarks/queries/query_13.sql) | 350 B | 247.57 | 246.07 | 245.55 | 244.77 | 248.40 | ok | +| [Q14](benchmarks/queries/query_14.sql) | 400 B | 249.88 | 237.64 | 238.18 | 235.56 | 238.64 | ok | +| [Q15](benchmarks/queries/query_15.sql) | 236 B | 75.58 | 73.07 | 73.32 | 71.09 | 74.55 | ok | +| [Q16](benchmarks/queries/query_16.sql) | 244 B | 130.68 | 122.37 | 122.39 | 121.98 | 122.72 | ok | +| [Q17](benchmarks/queries/query_17.sql) | 414 B | 155.01 | 144.52 | 144.28 | 142.55 | 146.97 | ok | +| [Q18](benchmarks/queries/query_18.sql) | 264 B | 1288.46 | 1127.01 | 1112.19 | 1105.08 | 1178.59 | ok | +| [Q19](benchmarks/queries/query_19.sql) | 0 B | 4.50 | 4.34 | 4.11 | 4.01 | 5.15 | ok | +| [Q20](benchmarks/queries/query_20.sql) | 3 B | 227.21 | 135.07 | 134.97 | 134.86 | 135.50 | ok | +| [Q21](benchmarks/queries/query_21.sql) | 162 B | 266.42 | 268.70 | 268.11 | 264.90 | 273.70 | ok | +| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 471.23 | 356.98 | 357.07 | 354.97 | 358.80 | ok | +| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 272.00 | 236.75 | 236.84 | 235.63 | 237.68 | ok | +| [Q24](benchmarks/queries/query_24.sql) | 658 B | 280.44 | 274.08 | 273.73 | 272.01 | 276.87 | ok | +| [Q25](benchmarks/queries/query_25.sql) | 382 B | 226.93 | 226.56 | 226.36 | 226.06 | 227.44 | ok | +| [Q26](benchmarks/queries/query_26.sql) | 658 B | 272.76 | 300.66 | 289.97 | 271.68 | 351.01 | ok | +| [Q27](benchmarks/queries/query_27.sql) | 26 B | 449.09 | 398.29 | 398.98 | 394.65 | 400.55 | ok | +| [Q28](benchmarks/queries/query_28.sql) | 556 B | 1015.82 | 907.49 | 907.32 | 904.67 | 910.64 | ok | +| [Q29](benchmarks/queries/query_29.sql) | 990 B | 414.40 | 414.81 | 414.52 | 414.18 | 416.03 | ok | +| [Q30](benchmarks/queries/query_30.sql) | 244 B | 290.88 | 236.69 | 237.07 | 234.72 | 237.89 | ok | +| [Q31](benchmarks/queries/query_31.sql) | 398 B | 271.91 | 261.89 | 262.05 | 258.68 | 264.77 | ok | +| [Q32](benchmarks/queries/query_32.sql) | 376 B | 1001.96 | 785.02 | 787.69 | 774.24 | 790.48 | ok | +| [Q33](benchmarks/queries/query_33.sql) | 464 B | 391.70 | 454.24 | 419.25 | 390.48 | 587.97 | ok | +| [Q34](benchmarks/queries/query_34.sql) | 484 B | 436.48 | 424.10 | 423.99 | 413.11 | 435.32 | ok | +| [Q35](benchmarks/queries/query_35.sql) | 493 B | 689.66 | 681.69 | 681.60 | 679.59 | 683.99 | ok | +| [Q36](benchmarks/queries/query_36.sql) | 569 B | 379.10 | 338.62 | 337.69 | 333.24 | 345.84 | ok | +| [Q37](benchmarks/queries/query_37.sql) | 550 B | 256.58 | 244.32 | 244.01 | 243.86 | 245.39 | ok | +| [Q38](benchmarks/queries/query_38.sql) | 480 B | 83.42 | 67.53 | 67.28 | 67.05 | 68.52 | ok | +| [Q39](benchmarks/queries/query_39.sql) | 835 B | 536.54 | 426.07 | 426.31 | 423.11 | 428.55 | ok | +| [Q40](benchmarks/queries/query_40.sql) | 356 B | 113.53 | 92.15 | 92.18 | 91.78 | 92.46 | ok | +| [Q41](benchmarks/queries/query_41.sql) | 132 B | 46.85 | 33.59 | 33.18 | 32.37 | 35.65 | ok | +| [Q42](benchmarks/queries/query_42.sql) | 240 B | 572.27 | 568.26 | 568.52 | 566.96 | 569.02 | ok | diff --git a/benchmarks/benchmark_csv_to_columnar.cpp b/benchmarks/benchmark_csv_to_columnar.cpp index 3db1be1..bbae38d 100644 --- a/benchmarks/benchmark_csv_to_columnar.cpp +++ b/benchmarks/benchmark_csv_to_columnar.cpp @@ -4,9 +4,9 @@ #include #include -#include "io/csv.h" -#include "convert/csv_columnar.h" #include "common/error.h" +#include "convert/csv_columnar.h" +#include "io/csv.h" #ifndef COLUMNAR_BENCHMARK_DEFAULT_DATA #define COLUMNAR_BENCHMARK_DEFAULT_DATA "benchmarks/hits_sample.csv" @@ -28,7 +28,7 @@ #define COLUMNAR_BENCHMARK_DEFAULT_ROUNDTRIP_SCHEMA "benchmarks/schema_sample_new.csv" #endif -static constexpr size_t RowsPerGroup = 50000; +static constexpr size_t RowsPerGroup = 1 << 14; struct CsvCompareResult { bool equal = true; diff --git a/benchmarks/benchmark_queries.cpp b/benchmarks/benchmark_queries.cpp index 64cf866..a1be365 100644 --- a/benchmarks/benchmark_queries.cpp +++ b/benchmarks/benchmark_queries.cpp @@ -15,7 +15,7 @@ #endif static constexpr int FirstQueryId = 0; -static constexpr int LastQueryId = 16; +static constexpr int LastQueryId = 42; void RunQuery(const Executor& executor, const std::filesystem::path& path, const int query_id, bool record = false) { const auto& runners = GetQueryRunners(); @@ -44,7 +44,8 @@ void RunQuery(const Executor& executor, const std::filesystem::path& path, const return; } - const std::string actual_result = BatchToString(*result); + Batch result_batch = std::move(*result); + const std::string actual_result = BatchToString(result_batch); const std::filesystem::path ref_path = std::filesystem::path(COLUMNAR_BENCHMARK_QUERIES_DIR) / ("query_" + std::to_string(query_id) + ".csv"); @@ -56,8 +57,9 @@ void RunQuery(const Executor& executor, const std::filesystem::path& path, const status = "recorded"; } else if (std::filesystem::exists(ref_path)) { const std::string expected_result = ReadTextFile(ref_path); - if (compare_unordered ? EqualAsRowMultisets(actual_result, expected_result) - : actual_result == expected_result) { + if (actual_result == expected_result || + EqualWithLimitTies(query_sql, result_batch.GetSchema(), actual_result, expected_result) || + (compare_unordered && EqualAsRowMultisets(actual_result, expected_result))) { status = "passed"; } else { status = "failed"; @@ -74,7 +76,7 @@ void RunQuery(const Executor& executor, const std::filesystem::path& path, const status = "no_reference"; } - std::cout << "query_" << query_id << ": rows=" << result->RowsCount() << ", time=" << duration_ms + std::cout << "query_" << query_id << ": rows=" << result_batch.RowsCount() << ", time=" << duration_ms << "ms, status=" << status << std::endl; } @@ -91,6 +93,7 @@ int main(const int argc, char** argv) { const std::filesystem::path data_path = queries_dir.parent_path() / "hits_sample.columnar"; Executor executor; + executor.SetUnsupportedFallbackEnabled(true); executor.RegisterTable("hits", data_path); size_t found_queries = 0; diff --git a/benchmarks/queries/query_17.csv b/benchmarks/queries/query_17.csv new file mode 100644 index 0000000..58eea25 --- /dev/null +++ b/benchmarks/queries/query_17.csv @@ -0,0 +1,10 @@ +-9219796849203399214,,3 +-9219054275930888834,,1 +-9218024831767047585,жена дата киноафиша анке,1 +-9214794650081452866,,4 +-9214751021948998350,авомосква веб каменисный,2 +-9213728704863893851,,2 +-9213728704863893851,чагин выпуска на волна 2 сезон 24 резюме онлайн,2 +-9213106781151947221,,9 +-9208956738506700293,,2 +-9206351631809765116,,5 diff --git a/benchmarks/queries/query_18.csv b/benchmarks/queries/query_18.csv new file mode 100644 index 0000000..6a81859 --- /dev/null +++ b/benchmarks/queries/query_18.csv @@ -0,0 +1,10 @@ +5216851095034646002,51,,80 +5216851095034646002,52,,67 +1074353211169645510,8,,37 +1220910554975721402,13,,35 +1074353211169645510,19,,34 +4673379180966332110,0,,34 +614605011960296602,18,,34 +1074353211169645510,9,,33 +1508127196834704092,14,,33 +502693359570399458,59,,33 diff --git a/benchmarks/queries/query_19.csv b/benchmarks/queries/query_19.csv new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/queries/query_20.csv b/benchmarks/queries/query_20.csv new file mode 100644 index 0000000..5595fa4 --- /dev/null +++ b/benchmarks/queries/query_20.csv @@ -0,0 +1 @@ +95 diff --git a/benchmarks/queries/query_21.csv b/benchmarks/queries/query_21.csv new file mode 100644 index 0000000..ff1ddf2 --- /dev/null +++ b/benchmarks/queries/query_21.csv @@ -0,0 +1 @@ +один инструктура птахани нюши смотреть краси,http://bdsm_position/2624217,2013-07-01:2013/frl-4/transport.ru/google%2F,2 diff --git a/benchmarks/queries/query_22.csv b/benchmarks/queries/query_22.csv new file mode 100644 index 0000000..a82bde4 --- /dev/null +++ b/benchmarks/queries/query_22.csv @@ -0,0 +1,10 @@ +коптимиквиды юриста с роуз рая,https://produkty%2Fpulove.ru/booklyattion-war-sinij-9182/women,"Легко на участные участников., Цены - Стильная парнем. Саганрог догадения : Турции, купить у 10 дне кольные машинки не представки - Новая с избиение спродажа: котята 2014 г.в. Цена: 47500-10ECO060 – -------- купить квартиру Оренбург (России Galantrax Flamiliada Google, Nо 18 фотоконверк Супер Кардиган",45,12 +коптимиквиды юрий жд ворожные моем,https://produkty%2Fpulove.ru/booklyattion-war-sinij-9182/women,"Легко на участные участников., Цены - Стильная парнем. Саганрог догадения : Турции, купить у 10 дне кольные машинки не представки - Новая с избиение спродажа: котята 2014 г.в. Цена: 47500-10ECO060 – -------- купить квартиру Оренбург (России Galantrax Flamiliada Google, Nо 18 фотоконверк Супер Кардиган",16,6 +ведомосквы вместу,https://produkty%2Fpulove.ru/booklyattion-war-sinij-9182/women,"Convent-мененции: Бизнес спродажа коттекст) Скейтшоп Proskater.ru - Дизайнер) 1992 г.в. Цена дачного века Кированнале актеры Google (La Charm Boxer группатии, оформационка NIKE TRADE-IN 6750$, (г. Днепрочитании онлайники — Избранное упражнения - играть и цене, выполная",15,9 +"вспомидоры,отека обучение стека",https://produkty%2Fpulove.ru/booklyattion-war-sinij-9182/women,"Легко на участные участников., Цены - Стильная парнем. Саганрог догадения : Турции, купить у 10 дне кольные машинки не представки - Новая с избиение спродажа: котята 2014 г.в. Цена: 47500-10ECO060 – -------- купить квартиру Оренбург (России Galantrax Flamiliada Google, Nо 18 фотоконверк Супер Кардиган",10,1 +коптимизаностиницы,"https://produkty%2Fpulove.ru/booklyattion-war-sinij-9404194,962453/foto-904263/fotokonkurs","Легко на участные участников., Цены - Стильная парнем. Саганрог догадения : Турции, купить у 10 дне кольные машинки не представки - Новая с избиение спродажа: котята 2014 г.в. Цена: 47500-10ECO060 – -------- купить квартиру Оренбург (России Galantrax Flamiliada Google, Nо 18 фотоконверк Супер Кардиган",8,2 +ведомосквиталия страции,https://produkty%2Fpulove.ru/booklyattion-war-sinij-9182/women,"Легко на участные участников., Цены - Стильная парнем. Саганрог догадения : Турции, купить у 10 дне кольные машинки не представки - Новая с избиение спродажа: котята 2014 г.в. Цена: 47500-10ECO060 – -------- купить квартиру Оренбург (России Galantrax Flamiliada Google, Nо 18 фотоконверк Супер Кардиган",8,3 +коптимашевск но в хорошем качестве,https://produkty%2Fpulove.ru/booklyattion-war-sinij-9182/women,"Легко на участные участников., Цены - Стильная парнем. Саганрог догадения : Турции, купить у 10 дне кольные машинки не представки - Новая с избиение спродажа: котята 2014 г.в. Цена: 47500-10ECO060 – -------- купить квартиру Оренбург (России Galantrax Flamiliada Google, Nо 18 фотоконверк Супер Кардиган",6,3 +"вспомидоры,отзывы луи видация",https://produkty%2Fpulove.ru/booklyattion-war-sinij-9182/women,"Легко на участные участников., Цены - Стильная парнем. Саганрог догадения : Турции, купить у 10 дне кольные машинки не представки - Новая с избиение спродажа: котята 2014 г.в. Цена: 47500-10ECO060 – -------- купить квартиру Оренбург (России Galantrax Flamiliada Google, Nо 18 фотоконверк Супер Кардиган",5,2 +поттек кисловая коньюктивное,https://produkty%2Fpulove.ru/booklyattion-war-sinij-9182/women,"Легко на участные участников., Цены - Стильная парнем. Саганрог догадения : Турции, купить у 10 дне кольные машинки не представки - Новая с избиение спродажа: котята 2014 г.в. Цена: 47500-10ECO060 – -------- купить квартиру Оренбург (России Galantrax Flamiliada Google, Nо 18 фотоконверк Супер Кардиган",5,1 +коптимиквиды юрий последняя,"https://produkty%2Fpulove.ru/booklyattion-war-sinij-9404194,962453/foto","Легко на участные участников., Цены - Стильная парнем. Саганрог догадения : Турции, купить у 10 дне кольные машинки не представки - Новая с избиение спродажа: котята 2014 г.в. Цена: 47500-10ECO060 – -------- купить квартиру Оренбург (России Galantrax Flamiliada Google, Nо 18 фотоконверк Супер Кардиган",5,1 diff --git a/benchmarks/queries/query_23.csv b/benchmarks/queries/query_23.csv new file mode 100644 index 0000000..aee160c --- /dev/null +++ b/benchmarks/queries/query_23.csv @@ -0,0 +1,10 @@ +5856761623345613087,2013-07-14 20:03:13,"http://irr.ru/index.php?showalbum/login-maris?sle=1297/?itemsg/d78072,95742.122918/hormor.kiev.ua/all/resident%2F5.0 (company/calculate.google.ru/search=1&target_0=yestered/main/news.ru/forum/top/resident%2F537.36 (KHTML, like Gecko) Chrome%2F27.0.1453.116 Safari%2F&sr=http://afisha.mail/16979/detail.ru/1.5199f9/bd54a6acf5-863323167&op_category_id=9640891%26ad%3D839322%26width%3D278885%26bid%3D2788840&pvno=2&evlg=VC,1;VL,541;IC,192356435/women.aspx#locale=ru&cE=trudnyj",Приморск - IRR.ru +5642186719302103400,2013-07-14 20:08:25,https://produkty%2Fplata-pr-advert279299881/detail.aspx?State=15&tab=user_id=607&lang=&geoa=1&TID=3219013070948/page.google-poigraphic,"Wildberries.ru – Интернет-магазине Автопоиск работает (Mad Wax, The Like Feature Виварка с" +5816099920965546798,2013-07-14 20:08:25,https://produkty%2Fplata-pr-advert279299881/detail.aspx?State=15&tab=user_id=607&lang=&geoa=1&TID=3219013070948/page.google-poigraphic,"Wildberries.ru – Интернет-магазине Автопоиск работает (Mad Wax, The Like Feature Виварка с" +7685648394301400768,2013-07-14 20:18:55,http://svetlants/4369363/26/3/page_type=canalog285_1.html#msg12912219/page.googleBR,Мои кампании в магазин +4649371611520026744,2013-07-14 20:19:01,http://svetlants/4369363/26/3/page_type=canalog285_1.html#msg12912219/page.googleBR,Мои кампании в магазин +6556965995079484770,2013-07-14 21:51:03,http://irr.ru/index.php?showalbum/login-siezona-prinimu-na-brietielkakh-2%2F%2Fwwwww.googleuser=lera-polnija/3464128192/1/?cat=0&auth=0&user/63898.jpg.html?items_perryjpottelfoto.kurortmag.ru/search?text=windroid,"бассе» › MR7.ru#photoedro. Цвет синий. Есть или б/у, цвет черные" +7121853442383447326,2013-07-14 22:11:22,http://irr.ru/index.php?showalbum/login-jekrjuch_21_21019463#nav_state.google.ru/start=235431964&num=s57140736132382108416,Теплоску на +5713826993848947331,2013-07-14 22:11:34,http://irr.ru/index.php?showalbum/login-jekrjuch_21_21019463#nav_state.google.ru/start=235431964&num=s57140736132382108416,Теплоску на +8896084869010742218,2013-07-14 22:55:36,http://irr.ru/index.php?showalbum/list=0&auto_car=0&auth=0&driver.ru%2Fproduct_brands[]=google-AppleWebKit%2F537.22&he=9000&price_ot=&price_ot=&price,Приморск - IRR.ru +5901375477503871871,2013-07-14 22:59:22,"http://kurort/SINA, ADRIAN - PLAYERS-MIRACLE-REECT-THONY, BOB-FIREBALL LAKE, ROGERS-FAR EAST (EPISODE%3DfdSMzAwfeSNDAwNTIzNA%26url%3D//ads--googleusers", diff --git a/benchmarks/queries/query_24.csv b/benchmarks/queries/query_24.csv new file mode 100644 index 0000000..83b40dd --- /dev/null +++ b/benchmarks/queries/query_24.csv @@ -0,0 +1,10 @@ +ведомосквы не удалог на ногтей денье,2013-07-14 20:00:03 +ведомосквы не удалог на ногтей денье,2013-07-14 20:00:03 +армянск,2013-07-14 20:00:05 +армянск,2013-07-14 20:00:05 +коптимиквидвич фаршироксин,2013-07-14 20:00:06 +враганрог из мультики из баклажанов,2013-07-14 20:00:07 +враганрог из мультики из баклажанов,2013-07-14 20:00:07 +коптимиквидвич фаршироксин,2013-07-14 20:00:07 +hp 105.460 2007 годов,2013-07-14 20:00:09 +ведомосквы вместу,2013-07-14 20:00:09 diff --git a/benchmarks/queries/query_25.csv b/benchmarks/queries/query_25.csv new file mode 100644 index 0000000..ec41089 --- /dev/null +++ b/benchmarks/queries/query_25.csv @@ -0,0 +1,10 @@ +'exis disco ryder injected cuda 7269 +'kbnyjuj gjhnf gtgthm vfibys row 3 ставе +'kbnyjuj gjhnf gtgthm vfibys row 3 ставе +'kbnyst exfcnm vekmnbdfhrf +'kbnyst exfcnm vekmnbdfhrf +(http://kommedium=cpc&utm_source=main происход ++100 дизелькатровский стой ++100 дизелькатровский стой ++100500 4.5 отзывы ++100500 4.5 отзывы diff --git a/benchmarks/queries/query_26.csv b/benchmarks/queries/query_26.csv new file mode 100644 index 0000000..69393b8 --- /dev/null +++ b/benchmarks/queries/query_26.csv @@ -0,0 +1,10 @@ +ведомосквы не удалог на ногтей денье,2013-07-14 20:00:03 +ведомосквы не удалог на ногтей денье,2013-07-14 20:00:03 +армянск,2013-07-14 20:00:05 +армянск,2013-07-14 20:00:05 +коптимиквидвич фаршироксин,2013-07-14 20:00:06 +враганрог из мультики из баклажанов,2013-07-14 20:00:07 +враганрог из мультики из баклажанов,2013-07-14 20:00:07 +коптимиквидвич фаршироксин,2013-07-14 20:00:07 +hp 105.460 2007 годов,2013-07-14 20:00:09 +hp 105.460 2007 годов,2013-07-14 20:00:09 diff --git a/benchmarks/queries/query_27.csv b/benchmarks/queries/query_27.csv new file mode 100644 index 0000000..36d63d6 --- /dev/null +++ b/benchmarks/queries/query_27.csv @@ -0,0 +1,2 @@ +62,94,413789 +38,76,507770 diff --git a/benchmarks/queries/query_28.csv b/benchmarks/queries/query_28.csv new file mode 100644 index 0000000..f282ca2 --- /dev/null +++ b/benchmarks/queries/query_28.csv @@ -0,0 +1,2 @@ +state=19945206,107,287812,http://state=19945206/foto-4/login%20NoTs3M&where=all&filmId=u8aGGqtWs3M&where=all&server=sc.cheloveplanet.ru/forum.little&categoriya/muzhskaya_istory.com/demii-malchik_148_4055074241537][to]=&input_activalry: A Love&where=all&text=купить florer&aktion/loansert перевозобности&site_id=197&text=добавь лазерногорский тайская полупая автомобильника&v=1414-resp_desyachnyeprague +tambov.irr.ru,77,308277,http://tambov.irr.ru/0/c1/tgFtaeLDK0yb01A7xvQF08sjCFqQxn51 diff --git a/benchmarks/queries/query_29.csv b/benchmarks/queries/query_29.csv new file mode 100644 index 0000000..fa8c6c0 --- /dev/null +++ b/benchmarks/queries/query_29.csv @@ -0,0 +1 @@ +1604051916,1605051893,1606051870,1607051847,1608051824,1609051801,1610051778,1611051755,1612051732,1613051709,1614051686,1615051663,1616051640,1617051617,1618051594,1619051571,1620051548,1621051525,1622051502,1623051479,1624051456,1625051433,1626051410,1627051387,1628051364,1629051341,1630051318,1631051295,1632051272,1633051249,1634051226,1635051203,1636051180,1637051157,1638051134,1639051111,1640051088,1641051065,1642051042,1643051019,1644050996,1645050973,1646050950,1647050927,1648050904,1649050881,1650050858,1651050835,1652050812,1653050789,1654050766,1655050743,1656050720,1657050697,1658050674,1659050651,1660050628,1661050605,1662050582,1663050559,1664050536,1665050513,1666050490,1667050467,1668050444,1669050421,1670050398,1671050375,1672050352,1673050329,1674050306,1675050283,1676050260,1677050237,1678050214,1679050191,1680050168,1681050145,1682050122,1683050099,1684050076,1685050053,1686050030,1687050007,1688049984,1689049961,1690049938,1691049915,1692049892,1693049869 diff --git a/benchmarks/queries/query_30.csv b/benchmarks/queries/query_30.csv new file mode 100644 index 0000000..0091807 --- /dev/null +++ b/benchmarks/queries/query_30.csv @@ -0,0 +1,10 @@ +2,1124827693,180,90,1734 +2,1090700661,72,34,1410 +2,1600523122,55,21,1368 +2,1388696273,54,27,1893 +89,1608608493,53,23,1368 +2,2117869668,47,19,1638 +2,1294197925,46,24,1638 +2,1332033259,44,22,1368 +2,711074589,44,22,1750 +4,-1579207735,44,22,1638 diff --git a/benchmarks/queries/query_31.csv b/benchmarks/queries/query_31.csv new file mode 100644 index 0000000..cd9080e --- /dev/null +++ b/benchmarks/queries/query_31.csv @@ -0,0 +1,10 @@ +7696429134022182497,1489089878,1,0,1638 +6482967717235270130,1044362389,1,0,1990 +5066892000101574959,-71596892,1,0,1368 +7993146188537951010,-950867577,1,0,1996 +7902828125626228067,1959327432,1,0,1638 +8785153201120553613,2014969747,1,0,1917 +5271143278642762033,881242264,1,0,1750 +7783345834612279987,1229938370,1,0,2428 +6396407070930218377,1669126035,1,0,1917 +7334501187260267830,1044362389,1,0,1990 diff --git a/benchmarks/queries/query_32.csv b/benchmarks/queries/query_32.csv new file mode 100644 index 0000000..2b8355a --- /dev/null +++ b/benchmarks/queries/query_32.csv @@ -0,0 +1,10 @@ +5304862190549331281,1388530699,1,0,0 +5702349982543543869,1388530699,1,0,0 +7059829690038451943,1388530699,1,0,0 +8724477495099671010,1388530699,1,0,0 +8501029160613315028,1388530699,1,0,0 +8601288555213722311,1388530699,1,0,0 +7990759056615037279,1388530699,1,0,0 +4795464821004236775,1388530699,1,0,0 +8334036044264247698,1388530699,1,0,0 +8097641483614450091,1388530699,1,0,0 diff --git a/benchmarks/queries/query_33.csv b/benchmarks/queries/query_33.csv new file mode 100644 index 0000000..439e2af --- /dev/null +++ b/benchmarks/queries/query_33.csv @@ -0,0 +1,10 @@ +"http://irr.ru/index.php?showalbum/login-leniya7777294,938303130",58970 +http://komme%2F27.0.1453.116,29580 +https://produkty%2Fproduct,11464 +http://irr.ru/index.php?showalbum/login-kapusta-advert2668]=0&order_by=0,10480 +http://irr.ru/index.php?showalbum/login-kapustic/product_name,10128 +http://irr.ru/index.php,7758 +https://produkty%2F,6649 +http://irr.ru/index.php?showalbum/login,6141 +https://produkty/kurortmag,5764 +https://produkty%2Fpulove.ru/album/login,5495 diff --git a/benchmarks/queries/query_34.csv b/benchmarks/queries/query_34.csv new file mode 100644 index 0000000..49c4684 --- /dev/null +++ b/benchmarks/queries/query_34.csv @@ -0,0 +1,10 @@ +1,"http://irr.ru/index.php?showalbum/login-leniya7777294,938303130",58970 +1,http://komme%2F27.0.1453.116,29580 +1,https://produkty%2Fproduct,11464 +1,http://irr.ru/index.php?showalbum/login-kapusta-advert2668]=0&order_by=0,10480 +1,http://irr.ru/index.php?showalbum/login-kapustic/product_name,10128 +1,http://irr.ru/index.php,7758 +1,https://produkty%2F,6649 +1,http://irr.ru/index.php?showalbum/login,6141 +1,https://produkty/kurortmag,5764 +1,https://produkty%2Fpulove.ru/album/login,5495 diff --git a/benchmarks/queries/query_35.csv b/benchmarks/queries/query_35.csv new file mode 100644 index 0000000..acf96c8 --- /dev/null +++ b/benchmarks/queries/query_35.csv @@ -0,0 +1,10 @@ +-267589304,-267589305,-267589306,-267589307,1733 +-1064396353,-1064396354,-1064396355,-1064396356,1604 +2113746632,2113746631,2113746630,2113746629,1552 +-1071668921,-1071668922,-1071668923,-1071668924,1544 +2127211172,2127211171,2127211170,2127211169,1485 +1700560340,1700560339,1700560338,1700560337,1311 +657371700,657371699,657371698,657371697,1199 +1450638336,1450638335,1450638334,1450638333,1015 +1992394514,1992394513,1992394512,1992394511,1015 +1503108906,1503108905,1503108904,1503108903,990 diff --git a/benchmarks/queries/query_36.csv b/benchmarks/queries/query_36.csv new file mode 100644 index 0000000..50946d1 --- /dev/null +++ b/benchmarks/queries/query_36.csv @@ -0,0 +1,10 @@ +"http://irr.ru/index.php?showalbum/login-leniya7777294,938303130",56533 +http://komme%2F27.0.1453.116,28819 +http://irr.ru/index.php?showalbum/login-kapusta-advert2668]=0&order_by=0,10325 +http://irr.ru/index.php?showalbum/login-kapustic/product_name,9650 +http://irr.ru/index.php,7530 +http://irr.ru/index.php?showalbum/login,6032 +http://komme%2F27.0.1453.116 Safari%2F5.0 (compatible; MSIE 9.0;,4271 +http://irr.ru/index.php?showalbum/login-kupalnik,2475 +http://irr.ru/index.php?showalbum/login-kapusta-advert27256.html_params,2300 +http://komme%2F27.0.1453.116 Safari,1612 diff --git a/benchmarks/queries/query_37.csv b/benchmarks/queries/query_37.csv new file mode 100644 index 0000000..bc0c290 --- /dev/null +++ b/benchmarks/queries/query_37.csv @@ -0,0 +1,10 @@ +Тест (Россия) - Яндекс,67544 +"Шарарай), Выбрать! - обсуждаются на голд: Шоубиз - Свободная историс",46670 +Приморск - IRR.ru,46530 +"Брюки New Era H (Асус) 258 общая выплаток, горшечными",21166 +Теплоску на,13432 +Приморск (Россия) - Яндекс.Видео,8260 +AUTO.ria.ua ™ - Аппер,8115 +Dave and Hotpoint sport – самые вещие,7866 +"OWAProfessign), продать",5754 +Труси - Шоубиз,5692 diff --git a/benchmarks/queries/query_38.csv b/benchmarks/queries/query_38.csv new file mode 100644 index 0000000..1c65b6f --- /dev/null +++ b/benchmarks/queries/query_38.csv @@ -0,0 +1,10 @@ +http://sslow_13507.html?aspx?naId=6r7RoWvxU9qQ=,4489 +http://aliningrad,2703 +http://ekburg.irr.ru%2Fpuloveplanet,2035 +http://video.yandex.php,1712 +http://smeshariki.ru/obucheyelants,1687 +http://kinopoisk.ru/news/2146555f3530316995264from]=&int[27][]=&selection/01a54d83ce&lost.1406.07.2013,640 +http://kinopoisk.ru/news/2146555419/page=show_photo/70946/detail/55212.15&he,495 +http://afisha.yandex.ru/index,462 +http://sslow_13507.html?aspx?naId=6HS,301 +http://wildberrior/uphold,279 diff --git a/benchmarks/queries/query_39.csv b/benchmarks/queries/query_39.csv new file mode 100644 index 0000000..177a415 --- /dev/null +++ b/benchmarks/queries/query_39.csv @@ -0,0 +1,10 @@ +-1,0,0,http://kinopoisk.ru/?state,"http://irr.ru/index.php?showalbum/login-leniya7777294,938303130",18025 +0,0,0,,http://komme%2F27.0.1453.116,14229 +-1,0,0,http://state=19945206/foto-4/login-2491724/?bundlers/search?text,http://irr.ru/index.php?showalbum/login-kapusta-advert2668]=0&order_by=0,8988 +0,0,0,,"http://irr.ru/index.php?showalbum/login-leniya7777294,938303130",7520 +1,0,0,https://google.com/fee=меньше,http://komme%2F27.0.1453.116,3532 +-1,0,0,http://state=199450984062,"http://irr.ru/index.php?showalbum/login-leniya7777294,938303130",3215 +3,2,0,,http://komme%2F27.0.1453.116,2750 +0,0,0,,http://irr.ru/index.php,2106 +5,0,0,http://state=199450984062,http://sslow_13507.html?aspx?naId=6r7RoWvxU9qQ=,2085 +-1,0,0,http://state=19945206/foto-4/login-2491724/?bundlers/search?text,http://irr.ru/index.php?showalbum/login,2016 diff --git a/benchmarks/queries/query_40.csv b/benchmarks/queries/query_40.csv new file mode 100644 index 0000000..8c7e49a --- /dev/null +++ b/benchmarks/queries/query_40.csv @@ -0,0 +1,10 @@ +786877583695637547,2013-07-15,1902 +-7370781399609543775,2013-07-15,1863 +5575639553984629836,2013-07-15,1397 +7949176783462555086,2013-07-15,387 +-2799183658664184391,2013-07-15,340 +-9090998343276352723,2013-07-15,243 +-7867523844263798504,2013-07-15,154 +428727614974842268,2013-07-15,128 +-6359454925449424667,2013-07-15,112 +-122138210592592987,2013-07-15,111 diff --git a/benchmarks/queries/query_41.csv b/benchmarks/queries/query_41.csv new file mode 100644 index 0000000..7f820d8 --- /dev/null +++ b/benchmarks/queries/query_41.csv @@ -0,0 +1,10 @@ +1750,653,2142 +1750,580,2099 +1261,1017,917 +1261,805,860 +531,422,707 +1509,966,596 +1095,872,572 +1509,746,516 +1509,936,487 +1750,591,482 diff --git a/benchmarks/queries/query_42.csv b/benchmarks/queries/query_42.csv new file mode 100644 index 0000000..49603ed --- /dev/null +++ b/benchmarks/queries/query_42.csv @@ -0,0 +1,10 @@ +2013-07-14 20:00:00,256 +2013-07-14 20:01:00,259 +2013-07-14 20:02:00,256 +2013-07-14 20:03:00,238 +2013-07-14 20:04:00,255 +2013-07-14 20:05:00,282 +2013-07-14 20:06:00,227 +2013-07-14 20:07:00,265 +2013-07-14 20:08:00,231 +2013-07-14 20:09:00,218 diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index 1f96c84..0a6a1d8 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,18 +1,44 @@ -query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,1076.34,11.43,11.67,9.57,12.81,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,41.93,27.20,27.21,27.19,27.22,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",64.18,51.76,51.70,51.57,52.08,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,69.35,51.94,52.19,51.02,52.38,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,44.52,43.83,43.77,43.72,44.07,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,59.39,48.10,45.38,44.72,56.93,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",757.49,679.54,678.98,676.05,684.13,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",28.79,28.27,28.27,28.13,28.44,ok, -8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",126.09,112.90,112.57,111.87,114.58,ok, -9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",164.71,166.90,166.97,164.56,169.11,ok, -10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",51.34,37.20,37.17,37.06,37.38,ok, -11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",50.58,38.50,38.49,38.39,38.65,ok, -12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",63.54,64.18,64.19,63.69,64.67,ok, -13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",96.63,83.26,76.40,76.12,104.12,ok, -14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",83.27,113.13,87.14,72.80,205.45,ok, -15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",105.19,108.28,108.08,104.50,112.46,ok, -16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",204.87,201.82,200.27,199.42,207.34,ok, +query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,output_csv_bytes,status,error +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,8.14,3.95,3.91,3.78,4.22,7,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,34.17,6.33,6.32,6.01,6.67,6,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",48.00,17.26,17.25,16.83,17.71,18,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,49.77,13.43,13.37,13.21,13.76,20,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,148.05,149.27,148.12,147.47,153.38,6,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,165.31,130.88,130.96,130.57,131.02,6,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",4.43,3.69,3.68,3.60,3.82,22,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",6.40,6.80,6.69,6.46,7.35,31,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",230.36,193.81,193.36,191.19,197.33,81,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",210.07,210.02,209.45,209.09,212.08,237,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",299.72,268.08,268.08,267.32,268.86,102,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",287.55,270.30,269.48,268.57,273.67,116,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",229.11,232.94,233.32,230.58,234.54,387,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",247.57,246.07,245.55,244.77,248.40,350,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",249.88,237.64,238.18,235.56,238.64,400,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",75.58,73.07,73.32,71.09,74.55,236,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",130.68,122.37,122.39,121.98,122.72,244,ok, +17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",155.01,144.52,144.28,142.55,146.97,414,ok, +18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",1288.46,1127.01,1112.19,1105.08,1178.59,264,ok, +19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,4.50,4.34,4.11,4.01,5.15,0,ok, +20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,227.21,135.07,134.97,134.86,135.50,3,ok, +21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",266.42,268.70,268.11,264.90,273.70,162,ok, +22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",471.23,356.98,357.07,354.97,358.80,6188,ok, +23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,272.00,236.75,236.84,235.63,237.68,2633,ok, +24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,280.44,274.08,273.73,272.01,276.87,658,ok, +25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,226.93,226.56,226.36,226.06,227.44,382,ok, +26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",272.76,300.66,289.97,271.68,351.01,658,ok, +27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",449.09,398.29,398.98,394.65,400.55,26,ok, +28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",1015.82,907.49,907.32,904.67,910.64,556,ok, +29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",414.40,414.81,414.52,414.18,416.03,990,ok, +30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",290.88,236.69,237.07,234.72,237.89,244,ok, +31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",271.91,261.89,262.05,258.68,264.77,398,ok, +32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",1001.96,785.02,787.69,774.24,790.48,376,ok, +33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",391.70,454.24,419.25,390.48,587.97,464,ok, +34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",436.48,424.10,423.99,413.11,435.32,484,ok, +35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",689.66,681.69,681.60,679.59,683.99,493,ok, +36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",379.10,338.62,337.69,333.24,345.84,569,ok, +37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",256.58,244.32,244.01,243.86,245.39,550,ok, +38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",83.42,67.53,67.28,67.05,68.52,480,ok, +39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",536.54,426.07,426.31,423.11,428.55,835,ok, +40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",113.53,92.15,92.18,91.78,92.46,356,ok, +41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",46.85,33.59,33.18,32.37,35.65,132,ok, +42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",572.27,568.26,568.52,566.96,569.02,240,ok, diff --git a/benchmarks/support/query_result_checks.cpp b/benchmarks/support/query_result_checks.cpp index 80a44a3..bbdcc03 100644 --- a/benchmarks/support/query_result_checks.cpp +++ b/benchmarks/support/query_result_checks.cpp @@ -1,29 +1,33 @@ #include "query_result_checks.h" #include +#include +#include +#include #include #include -#include "io/csv_batch.h" #include "common/ascii.h" +#include "common/error.h" +#include "common/operator_utils.h" +#include "common/planner_utils.h" +#include "io/csv.h" +#include "io/csv_batch.h" std::string BatchToString(const Batch& batch) { std::vector> rows; AppendBatchRows(batch, rows); - std::string result; + std::ostringstream out; + const CsvWriter writer(out); for (const auto& row : rows) { - for (size_t i = 0; i < row.size(); ++i) { - result += row[i]; - if (i + 1 < row.size()) { - result += ","; - } - } - result += "\n"; + writer.WriteRow(row); } - return result; + writer.Flush(); + + return out.str(); } bool QueryUsesGroupBy(const std::string_view sql) { return ToUpperAscii(sql).find("GROUP BY") != std::string::npos; } @@ -58,3 +62,250 @@ bool EqualAsRowMultisets(const std::string_view lhs, const std::string_view rhs) return lhs_rows == rhs_rows; } + +static std::string TrimCopy(const std::string_view value) { + size_t begin = 0; + while (begin < value.size() && std::isspace(static_cast(value[begin]))) { + ++begin; + } + + size_t end = value.size(); + while (end > begin && std::isspace(static_cast(value[end - 1]))) { + --end; + } + + return std::string(value.substr(begin, end - begin)); +} + +static std::vector> ParseCsvRows(const std::string_view text) { + std::istringstream in{std::string(text)}; + const CsvReader reader(in); + + std::vector> rows; + std::vector row; + + while (reader.ReadRow(row)) { + rows.push_back(row); + } + + return rows; +} + +struct ParsedOrderBy { + std::string output_name; + bool descending = false; +}; + +static std::optional ParseSingleOrderBy(const std::string_view sql) { + const std::string upper = ToUpperAscii(sql); + const size_t order_pos = upper.find("ORDER BY"); + const size_t limit_pos = upper.find("LIMIT", order_pos == std::string::npos ? 0 : order_pos); + + if (order_pos == std::string::npos || limit_pos == std::string::npos || limit_pos <= order_pos) { + return std::nullopt; + } + + std::string item = TrimCopy(sql.substr(order_pos + std::string_view("ORDER BY").size(), + limit_pos - order_pos - std::string_view("ORDER BY").size())); + if (const size_t comma = item.find(','); comma != std::string::npos) { + item = TrimCopy(std::string_view(item).substr(0, comma)); + } + + bool descending = false; + const std::string upper_item = ToUpperAscii(item); + if (upper_item.ends_with(" DESC")) { + descending = true; + item = TrimCopy(std::string_view(item).substr(0, item.size() - std::string_view(" DESC").size())); + } else if (upper_item.ends_with(" ASC")) { + item = TrimCopy(std::string_view(item).substr(0, item.size() - std::string_view(" ASC").size())); + } + + if (item.empty()) { + return std::nullopt; + } + + return ParsedOrderBy{ + .output_name = std::move(item), + .descending = descending, + }; +} + +static std::optional FindOutputColumn(const Schema& schema, const std::string_view name) { + for (size_t i = 0; i < schema.columns.size(); ++i) { + if (SameColumnName(schema.columns[i].name, name)) { + return i; + } + } + return std::nullopt; +} + +static std::string EncodeRowForCompare(const std::vector& row) { + std::string encoded; + for (const auto& value : row) { + encoded += std::to_string(value.size()); + encoded.push_back(':'); + encoded += value; + } + return encoded; +} + +static bool EqualRowMultisets(const std::vector>& lhs, + const std::vector>& rhs) { + std::vector lhs_encoded; + std::vector rhs_encoded; + lhs_encoded.reserve(lhs.size()); + rhs_encoded.reserve(rhs.size()); + + for (const auto& row : lhs) { + lhs_encoded.push_back(EncodeRowForCompare(row)); + } + for (const auto& row : rhs) { + rhs_encoded.push_back(EncodeRowForCompare(row)); + } + + std::ranges::sort(lhs_encoded); + std::ranges::sort(rhs_encoded); + + return lhs_encoded == rhs_encoded; +} + +static std::optional RawFieldFromRight(const std::string& row, const size_t offset_from_right) { + size_t end = row.size(); + + for (size_t i = 0; i < offset_from_right; ++i) { + const size_t comma = row.rfind(',', end == 0 ? 0 : end - 1); + if (comma == std::string::npos) { + return std::nullopt; + } + end = comma; + } + + const size_t comma = row.rfind(',', end == 0 ? 0 : end - 1); + const size_t begin = comma == std::string::npos ? 0 : comma + 1; + return row.substr(begin, end - begin); +} + +static std::string NormalizeRawBenchmarkRow(std::string row) { + std::erase(row, '"'); + return row; +} + +static bool EqualWithLimitTiesRawFallback(const Schema& schema, const ParsedOrderBy& order_by, + const size_t order_column, const std::string_view actual, + const std::string_view expected) { + const std::vector actual_rows = SplitRows(actual); + const std::vector expected_rows = SplitRows(expected); + if (actual_rows.size() != expected_rows.size() || expected_rows.empty() || order_column >= schema.columns.size()) { + return false; + } + + const size_t offset_from_right = schema.columns.size() - 1 - order_column; + const std::optional cutoff = RawFieldFromRight(expected_rows.back(), offset_from_right); + if (!cutoff.has_value()) { + return false; + } + + const ColumnType order_type = schema.columns[order_column].type; + + const auto is_above_cutoff = [&](const std::string& value) { + const std::strong_ordering comparison = CompareValues(order_type, value, *cutoff); + return order_by.descending ? comparison > 0 : comparison < 0; + }; + + const auto is_not_worse_than_cutoff = [&](const std::string& value) { + const std::strong_ordering comparison = CompareValues(order_type, value, *cutoff); + return order_by.descending ? comparison >= 0 : comparison <= 0; + }; + + std::vector actual_above_cutoff; + std::vector expected_above_cutoff; + + for (const auto& row : expected_rows) { + const std::optional value = RawFieldFromRight(row, offset_from_right); + if (!value.has_value()) { + return false; + } + if (is_above_cutoff(*value)) { + expected_above_cutoff.push_back(NormalizeRawBenchmarkRow(row)); + } + } + + for (const auto& row : actual_rows) { + const std::optional value = RawFieldFromRight(row, offset_from_right); + if (!value.has_value() || !is_not_worse_than_cutoff(*value)) { + return false; + } + if (is_above_cutoff(*value)) { + actual_above_cutoff.push_back(NormalizeRawBenchmarkRow(row)); + } + } + + std::ranges::sort(actual_above_cutoff); + std::ranges::sort(expected_above_cutoff); + + return actual_above_cutoff == expected_above_cutoff; +} + +bool EqualWithLimitTies(const std::string_view sql, const Schema& schema, const std::string_view actual, + const std::string_view expected) { + const std::optional order_by = ParseSingleOrderBy(sql); + if (!order_by.has_value()) { + return false; + } + + const std::optional order_column = FindOutputColumn(schema, order_by->output_name); + if (!order_column.has_value()) { + return false; + } + + std::vector> actual_rows; + std::vector> expected_rows; + + try { + actual_rows = ParseCsvRows(actual); + expected_rows = ParseCsvRows(expected); + } catch (const Error&) { + return EqualWithLimitTiesRawFallback(schema, *order_by, *order_column, actual, expected); + } + + try { + if (actual_rows.size() != expected_rows.size() || expected_rows.empty()) { + return false; + } + + const ColumnType order_type = schema.columns[*order_column].type; + const std::string& cutoff = expected_rows.back()[*order_column]; + + std::vector> actual_above_cutoff; + std::vector> expected_above_cutoff; + + const auto is_above_cutoff = [&](const std::string& value) { + const std::strong_ordering comparison = CompareValues(order_type, value, cutoff); + return order_by->descending ? comparison > 0 : comparison < 0; + }; + + const auto is_not_worse_than_cutoff = [&](const std::string& value) { + const std::strong_ordering comparison = CompareValues(order_type, value, cutoff); + return order_by->descending ? comparison >= 0 : comparison <= 0; + }; + + for (const auto& row : expected_rows) { + if (is_above_cutoff(row[*order_column])) { + expected_above_cutoff.push_back(row); + } + } + + for (const auto& row : actual_rows) { + if (!is_not_worse_than_cutoff(row[*order_column])) { + return false; + } + if (is_above_cutoff(row[*order_column])) { + actual_above_cutoff.push_back(row); + } + } + + return EqualRowMultisets(actual_above_cutoff, expected_above_cutoff); + } catch (const Error&) { + return EqualWithLimitTiesRawFallback(schema, *order_by, *order_column, actual, expected); + } +} diff --git a/benchmarks/support/query_result_checks.h b/benchmarks/support/query_result_checks.h index ceea1b7..82409e7 100644 --- a/benchmarks/support/query_result_checks.h +++ b/benchmarks/support/query_result_checks.h @@ -8,3 +8,4 @@ std::string BatchToString(const Batch& batch); bool QueryUsesGroupBy(std::string_view sql); bool EqualAsRowMultisets(std::string_view lhs, std::string_view rhs); +bool EqualWithLimitTies(std::string_view sql, const Schema& schema, std::string_view actual, std::string_view expected); diff --git a/benchmarks/support/query_runners.h b/benchmarks/support/query_runners.h index b1ca5cf..0bfdd40 100644 --- a/benchmarks/support/query_runners.h +++ b/benchmarks/support/query_runners.h @@ -88,11 +88,185 @@ inline ExecuteExpected RunQuery16(const Executor& executor) { "LIMIT 10;"); } +inline ExecuteExpected RunQuery17(const Executor& executor) { + return executor.Execute("SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;"); +} + +inline ExecuteExpected RunQuery18(const Executor& executor) { + return executor.Execute( + "SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, " + "SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery19(const Executor& executor) { + return executor.Execute("SELECT UserID FROM hits WHERE UserID = 435090932899640449;"); +} + +inline ExecuteExpected RunQuery20(const Executor& executor) { + return executor.Execute("SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';"); +} + +inline ExecuteExpected RunQuery21(const Executor& executor) { + return executor.Execute( + "SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' " + "GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery22(const Executor& executor) { + return executor.Execute( + "SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE " + "'%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT " + "10;"); +} + +inline ExecuteExpected RunQuery23(const Executor& executor) { + return executor.Execute("SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;"); +} + +inline ExecuteExpected RunQuery24(const Executor& executor) { + return executor.Execute("SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;"); +} + +inline ExecuteExpected RunQuery25(const Executor& executor) { + return executor.Execute("SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;"); +} + +inline ExecuteExpected RunQuery26(const Executor& executor) { + return executor.Execute( + "SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;"); +} + +inline ExecuteExpected RunQuery27(const Executor& executor) { + return executor.Execute( + "SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING " + "COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;"); +} + +inline ExecuteExpected RunQuery28(const Executor& executor) { + return executor.Execute( + R"(SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;)"); +} + +inline ExecuteExpected RunQuery29(const Executor& executor) { + return executor.Execute( + "SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), " + "SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), " + "SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), " + "SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), " + "SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), " + "SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), " + "SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), " + "SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), " + "SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), " + "SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), " + "SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), " + "SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), " + "SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), " + "SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), " + "SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), " + "SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), " + "SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), " + "SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), " + "SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), " + "SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), " + "SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), " + "SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), " + "SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;"); +} + +inline ExecuteExpected RunQuery30(const Executor& executor) { + return executor.Execute( + "SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE " + "SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery31(const Executor& executor) { + return executor.Execute( + "SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> " + "'' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery32(const Executor& executor) { + return executor.Execute( + "SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, " + "ClientIP ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery33(const Executor& executor) { + return executor.Execute("SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery34(const Executor& executor) { + return executor.Execute("SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery35(const Executor& executor) { + return executor.Execute( + "SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, " + "ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery36(const Executor& executor) { + return executor.Execute( + "SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate " + "<= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC " + "LIMIT 10;"); +} + +inline ExecuteExpected RunQuery37(const Executor& executor) { + return executor.Execute( + "SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND " + "EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY " + "PageViews DESC LIMIT 10;"); +} + +inline ExecuteExpected RunQuery38(const Executor& executor) { + return executor.Execute( + "SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate " + "<= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC " + "LIMIT 10 OFFSET 1000;"); +} + +inline ExecuteExpected RunQuery39(const Executor& executor) { + return executor.Execute( + "SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN " + "Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= " + "'2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, " + "AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;"); +} + +inline ExecuteExpected RunQuery40(const Executor& executor) { + return executor.Execute( + "SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= " + "'2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = " + "3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;"); +} + +inline ExecuteExpected RunQuery41(const Executor& executor) { + return executor.Execute( + "SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND " + "EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = " + "2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET " + "10000;"); +} + +inline ExecuteExpected RunQuery42(const Executor& executor) { + return executor.Execute( + "SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND " + "EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY " + "DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;"); +} + inline const std::map& GetQueryRunners() { static const std::map Runners = { {0, RunQuery0}, {1, RunQuery1}, {2, RunQuery2}, {3, RunQuery3}, {4, RunQuery4}, {5, RunQuery5}, {6, RunQuery6}, {7, RunQuery7}, {8, RunQuery8}, {9, RunQuery9}, {10, RunQuery10}, {11, RunQuery11}, - {12, RunQuery12}, {13, RunQuery13}, {14, RunQuery14}, {15, RunQuery15}, {16, RunQuery16}, + {12, RunQuery12}, {13, RunQuery13}, {14, RunQuery14}, {15, RunQuery15}, {16, RunQuery16}, {17, RunQuery17}, + {18, RunQuery18}, {19, RunQuery19}, {20, RunQuery20}, {21, RunQuery21}, {22, RunQuery22}, {23, RunQuery23}, + {24, RunQuery24}, {25, RunQuery25}, {26, RunQuery26}, {27, RunQuery27}, {28, RunQuery28}, {29, RunQuery29}, + {30, RunQuery30}, {31, RunQuery31}, {32, RunQuery32}, {33, RunQuery33}, {34, RunQuery34}, {35, RunQuery35}, + {36, RunQuery36}, {37, RunQuery37}, {38, RunQuery38}, {39, RunQuery39}, {40, RunQuery40}, {41, RunQuery41}, + {42, RunQuery42}, }; return Runners; } diff --git a/include/executor/operator_utils.h b/include/common/operator_utils.h similarity index 83% rename from include/executor/operator_utils.h rename to include/common/operator_utils.h index d247c5d..3d1a93d 100644 --- a/include/executor/operator_utils.h +++ b/include/common/operator_utils.h @@ -7,4 +7,5 @@ std::strong_ordering CompareValues(ColumnType type, std::string_view lhs, std::string_view rhs); bool MatchesComparison(ColumnType type, std::string_view lhs, std::string_view rhs, ComparisonKind kind); +bool LikeMatches(std::string_view value, std::string_view pattern); ColumnType AggregateOutputType(const PlannedAgg& aggregate); diff --git a/include/common/parsing.h b/include/common/parsing.h index d779adb..dd6ca7c 100644 --- a/include/common/parsing.h +++ b/include/common/parsing.h @@ -8,14 +8,14 @@ #include "model/schema.h" #include "common/int128.h" -bool ParseBoolean(const std::string& value); -int16_t ParseInt16(const std::string& value); -int32_t ParseInt32(const std::string& value); -int64_t ParseInt64(const std::string& value); -Int128 ParseInt128(const std::string& value); -int32_t ParseDate(const std::string& value); -int64_t ParseTimestamp(const std::string& value); -char ParseCharacter(const std::string& value); +bool ParseBoolean(std::string_view value); +int16_t ParseInt16(std::string_view value); +int32_t ParseInt32(std::string_view value); +int64_t ParseInt64(std::string_view value); +Int128 ParseInt128(std::string_view value); +int32_t ParseDate(std::string_view value); +int64_t ParseTimestamp(std::string_view value); +char ParseCharacter(std::string_view value); std::optional TryParseBoolean(std::string_view value); std::optional TryParseInt16(std::string_view value); diff --git a/include/executor/planner_utils.h b/include/common/planner_utils.h similarity index 88% rename from include/executor/planner_utils.h rename to include/common/planner_utils.h index 70d19af..1e5bc12 100644 --- a/include/executor/planner_utils.h +++ b/include/common/planner_utils.h @@ -2,6 +2,7 @@ #include #include +#include #include "executor/query.h" #include "model/schema.h" @@ -14,3 +15,4 @@ bool SameColumnName(std::string_view lhs, std::string_view rhs); bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs); bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs); bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs); +std::vector StarColumnIndexes(const Schema& schema); diff --git a/include/convert/csv_columnar.h b/include/convert/csv_columnar.h index cd2b5f5..058af34 100644 --- a/include/convert/csv_columnar.h +++ b/include/convert/csv_columnar.h @@ -3,7 +3,10 @@ #include #include +inline constexpr size_t DefaultMaxRowsPerGroup = 1 << 14; + void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, - const std::filesystem::path& output_path, size_t max_rows_per_group = 4); + const std::filesystem::path& output_path, + size_t max_rows_per_group = DefaultMaxRowsPerGroup); void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, const std::filesystem::path& data_path); diff --git a/include/executor/aggregate_state.h b/include/executor/aggregate_state.h index c4d22a9..0a3f152 100644 --- a/include/executor/aggregate_state.h +++ b/include/executor/aggregate_state.h @@ -4,6 +4,7 @@ #include #include +#include "common/int128.h" #include "executor/query_plan.h" class AggState { @@ -16,7 +17,10 @@ class AggState { virtual ~AggState() = default; virtual void ConsumeValue(std::string_view value) = 0; + virtual void ConsumeInt128(Int128 value); virtual void ConsumeRow() = 0; + virtual void ConsumeRows(size_t count); + virtual std::string Finalize() const = 0; }; diff --git a/include/executor/executor.h b/include/executor/executor.h index 1c54192..0101547 100644 --- a/include/executor/executor.h +++ b/include/executor/executor.h @@ -16,8 +16,11 @@ class Executor { Executor() = default; void RegisterTable(const std::string& name, std::filesystem::path path); + void SetUnsupportedFallbackEnabled(bool enabled); + ExecuteExpected Execute(std::string_view query) const; private: std::unordered_map tables_; + bool unsupported_fallback_enabled_ = false; }; diff --git a/include/executor/query.h b/include/executor/query.h index 2cd4b88..6ed56ad 100644 --- a/include/executor/query.h +++ b/include/executor/query.h @@ -1,10 +1,15 @@ #pragma once #include +#include #include +#include #include +#include #include +#include "common/int128.h" + enum class ComparisonKind { Equal, NotEqual, @@ -41,10 +46,99 @@ struct QueryValue { QueryLiteral literal; }; -struct FilterSpec { - QueryValue left; +enum class ExprKind { + Column, + Literal, + Binary, + Function, + Star, + Case, +}; + +enum class BinaryOp { + Add, + Subtract, +}; + +struct ExprSpec; +using ExprPtr = std::shared_ptr; + +struct PredicateSpec; +using PredicatePtr = std::shared_ptr; + +struct CaseSpec { + PredicatePtr condition; + ExprPtr then_expr; + ExprPtr else_expr; +}; + +struct ExprSpec { + ExprKind kind = ExprKind::Literal; + + ColumnRef column; + bool column_index_bound = false; + size_t column_index = 0; + + QueryLiteral literal; + + BinaryOp binary_op = BinaryOp::Add; + ExprPtr left; + ExprPtr right; + + std::string function_name; + std::vector arguments; + + bool regex_replace_bound = false; + std::regex regex_pattern; + std::string regex_replacement; + + CaseSpec case_spec; + + std::string output_name; +}; + +enum class PredicateKind { + Comparison, + And, + Like, + NotLike, + In, +}; + +struct PredicateSpec { + PredicateKind kind = PredicateKind::Comparison; + + ExprPtr left; ComparisonKind comparison = ComparisonKind::Equal; - QueryValue right; + ExprPtr right; + + std::vector values; + + PredicatePtr lhs; + PredicatePtr rhs; + + bool typed_literal_comparison_bound = false; + size_t typed_column_index = 0; + Int128 typed_literal_value = 0; + ComparisonKind typed_comparison = ComparisonKind::Equal; + + bool metadata_typed_literal_comparison_bound = false; + size_t metadata_typed_column_index = 0; + Int128 metadata_typed_literal_value = 0; + ComparisonKind metadata_typed_comparison = ComparisonKind::Equal; + + bool literal_in_set_bound = false; + size_t in_column_index = 0; + std::unordered_set literal_in_values; + + bool metadata_typed_in_set_bound = false; + size_t metadata_typed_in_column_index = 0; + std::vector metadata_typed_in_values; + + bool literal_like_pattern_bound = false; + size_t like_column_index = 0; + std::string like_pattern; + bool like_negated = false; }; enum class AggArgumentKind { @@ -57,6 +151,7 @@ struct AggSpec { bool distinct = false; AggArgumentKind argument_kind = AggArgumentKind::Column; ColumnRef column; + ExprPtr argument; std::string output_name; }; @@ -68,6 +163,7 @@ enum class SelectItemKind { struct SelectItemSpec { SelectItemKind kind = SelectItemKind::GroupKey; ColumnRef column; + ExprPtr expression; AggSpec aggregate; std::string output_name; }; @@ -80,9 +176,15 @@ struct OrderBySpec { struct Query { std::string table_name; std::string table_alias; + std::vector select_items; - std::vector group_by; - std::optional filter; + std::vector group_by; + + PredicatePtr filter; + PredicatePtr having; + std::vector order_by; + std::optional limit; + size_t offset = 0; }; diff --git a/include/executor/query_plan.h b/include/executor/query_plan.h index 7cfaba7..fcebdcd 100644 --- a/include/executor/query_plan.h +++ b/include/executor/query_plan.h @@ -2,10 +2,13 @@ #include #include +#include #include #include +#include #include +#include "common/int128.h" #include "executor/aggregate_function.h" #include "executor/query.h" #include "model/schema.h" @@ -13,14 +16,23 @@ struct PlannedAgg { const AggFuncDefinition* function = nullptr; bool distinct = false; + AggArgumentKind argument_kind = AggArgumentKind::Column; size_t column_index = 0; + + bool direct_numeric_argument = false; + Int128 direct_numeric_offset = 0; + ColumnType input_type = ColumnType::String; + ExprPtr argument; + std::string output_name; }; struct PlannedGroupKey { size_t column_index = 0; + ExprPtr expression; + ColumnType column_type = ColumnType::String; std::string output_name; }; @@ -45,8 +57,10 @@ enum class PlannedValueKind { struct PlannedValue { PlannedValueKind kind = PlannedValueKind::Literal; + ColumnType column_type = ColumnType::String; size_t column_index = 0; + std::string literal_text; }; @@ -57,13 +71,31 @@ struct PlannedFilter { ColumnType comparison_type = ColumnType::String; }; +struct PlannedPredicate { + PredicatePtr predicate; +}; + struct PlannedQuery { std::filesystem::path table_path; + Schema table_schema; + std::vector projection_indexes; - std::optional filter; + + PredicatePtr filter; + PredicatePtr having; + std::vector group_keys; std::vector aggregates; std::vector select_items; + + std::vector plain_select_items; std::vector order_by; + std::vector plain_order_by; + std::optional limit; + size_t offset = 0; + + bool plain_select = false; + bool metadata_count_only = false; + bool metadata_extrema_only = false; }; diff --git a/include/io/stream.h b/include/io/stream.h index a1b2358..a7e13e1 100644 --- a/include/io/stream.h +++ b/include/io/stream.h @@ -10,20 +10,23 @@ #include #include #include +#include #include "common/error.h" void ReadBytes(std::istream& in, char* dst, size_t size); void WriteBytes(std::ostream& out, std::string_view bytes); -template +template + requires std::is_trivially_copyable_v T ReadStream(std::istream& in) { - T value = 0; + T value{}; ReadBytes(in, reinterpret_cast(&value), sizeof(value)); return value; } -template +template + requires std::is_trivially_copyable_v void WriteStream(std::ostream& out, T value) { WriteBytes(out, {reinterpret_cast(&value), sizeof(value)}); } diff --git a/include/model/batch.h b/include/model/batch.h index 6d8703b..43d6dea 100644 --- a/include/model/batch.h +++ b/include/model/batch.h @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include "model/column.h" @@ -26,7 +28,12 @@ class Batch { size_t RowsCount() const; void Reserve(size_t n) const; - void AppendValueFromString(size_t column_index, const std::string& value) const; + void AppendValueFromString(size_t column_index, std::string_view value) const; + void AppendValueFromColumn(size_t column_index, const Column& source, size_t row) const; + void AppendColumnRange(size_t column_index, const Column& source, size_t begin, size_t count) const; + void AppendColumnSelected(size_t column_index, const Column& source, std::span rows) const; + void AppendRowsRangeFromBatch(const Batch& source, size_t begin, size_t count) const; + void AppendRowsSelectedFromBatch(const Batch& source, std::span rows) const; void ReadColumnFrom(size_t column_index, std::istream& in, uint32_t row_count, uint64_t size) const; const Column& ColumnAt(size_t i) const; diff --git a/include/model/column.h b/include/model/column.h index 118c236..a1e86d4 100644 --- a/include/model/column.h +++ b/include/model/column.h @@ -3,10 +3,24 @@ #include #include #include +#include #include +#include +#include +#include +#include "common/int128.h" #include "model/schema.h" +enum class ValueComparison { + Equal, + NotEqual, + Less, + LessOrEqual, + Greater, + GreaterOrEqual, +}; + class Column { public: explicit Column(ColumnType type); @@ -19,7 +33,15 @@ class Column { ColumnType Type() const { return type_; } virtual size_t Size() const = 0; + virtual std::string ValueAsString(size_t row) const = 0; + virtual Int128 ValueAsInt128(size_t row) const; + + virtual void SelectRowsByInt128Comparison(Int128 rhs, ValueComparison comparison, std::vector& rows) const; + virtual void SelectRowsByStringSet(const std::unordered_set& values, std::vector& rows) const; + virtual void SelectRowsByLikePattern(std::string_view pattern, bool negated, std::vector& rows) const; + + virtual void AppendEncodedValue(size_t row, std::string& out) const; virtual std::unique_ptr Clone() const = 0; virtual void WriteTo(std::ostream& out) const = 0; @@ -38,8 +60,14 @@ class MutableColumn : public Column { virtual void Reserve(size_t n) = 0; virtual void Clear() = 0; - virtual void AppendFromString(const std::string& value) = 0; + + virtual void AppendFromString(std::string_view value) = 0; + virtual void AppendFromColumn(const Column& source, size_t row) = 0; + virtual void AppendRangeFromColumn(const Column& source, size_t begin, size_t count) = 0; + virtual void AppendSelectedFromColumn(const Column& source, std::span rows) = 0; + virtual void ReadFrom(std::istream& in, uint32_t row_count, uint64_t size) = 0; + virtual std::unique_ptr CloneMutable() const = 0; }; diff --git a/include/model/column_string.h b/include/model/column_string.h index dc6236a..209569e 100644 --- a/include/model/column_string.h +++ b/include/model/column_string.h @@ -1,6 +1,9 @@ #pragma once +#include #include +#include +#include #include #include "model/column.h" @@ -20,8 +23,15 @@ class StringColumn final : public MutableColumn { void Reserve(size_t n) override; void Clear() override; - void AppendFromString(const std::string& value) override; + void AppendFromString(std::string_view value) override; + void AppendFromColumn(const Column& source, size_t row) override; + void AppendRangeFromColumn(const Column& source, size_t begin, size_t count) override; + void AppendSelectedFromColumn(const Column& source, std::span rows) override; std::string ValueAsString(size_t row) const override; + void SelectRowsByStringSet(const std::unordered_set& values, std::vector& rows) const override; + void SelectRowsByLikePattern(std::string_view pattern, bool negated, std::vector& rows) const override; + void AppendEncodedValue(size_t row, std::string& out) const override; + std::unique_ptr Clone() const override; std::unique_ptr CloneMutable() const override; diff --git a/include/model/column_traits.h b/include/model/column_traits.h index 890e393..fc475b8 100644 --- a/include/model/column_traits.h +++ b/include/model/column_traits.h @@ -2,6 +2,7 @@ #include #include +#include #include #include "common/error.h" @@ -16,7 +17,7 @@ template <> struct ColumnValueTraits { using Type = uint8_t; - static Type Parse(const std::string& value) { return ParseBoolean(value) ? 1 : 0; } + static Type Parse(const std::string_view value) { return ParseBoolean(value) ? 1 : 0; } static std::string ToString(const Type value) { return BooleanToString(value != 0); } }; @@ -24,7 +25,7 @@ template <> struct ColumnValueTraits { using Type = int16_t; - static Type Parse(const std::string& value) { return ParseInt16(value); } + static Type Parse(const std::string_view value) { return ParseInt16(value); } static std::string ToString(const Type value) { return std::to_string(value); } }; @@ -32,7 +33,7 @@ template <> struct ColumnValueTraits { using Type = int32_t; - static Type Parse(const std::string& value) { return ParseInt32(value); } + static Type Parse(const std::string_view value) { return ParseInt32(value); } static std::string ToString(const Type value) { return std::to_string(value); } }; @@ -40,7 +41,7 @@ template <> struct ColumnValueTraits { using Type = int64_t; - static Type Parse(const std::string& value) { return ParseInt64(value); } + static Type Parse(const std::string_view value) { return ParseInt64(value); } static std::string ToString(const Type value) { return std::to_string(value); } }; @@ -48,7 +49,7 @@ template <> struct ColumnValueTraits { using Type = Int128; - static Type Parse(const std::string& value) { return ParseInt128(value); } + static Type Parse(const std::string_view value) { return ParseInt128(value); } static std::string ToString(const Type value) { return Int128ToString(value); } }; @@ -56,7 +57,7 @@ template <> struct ColumnValueTraits { using Type = int32_t; - static Type Parse(const std::string& value) { return ParseDate(value); } + static Type Parse(const std::string_view value) { return ParseDate(value); } static std::string ToString(const Type value) { return DateToString(value); } }; @@ -64,7 +65,7 @@ template <> struct ColumnValueTraits { using Type = int64_t; - static Type Parse(const std::string& value) { return ParseTimestamp(value); } + static Type Parse(const std::string_view value) { return ParseTimestamp(value); } static std::string ToString(const Type value) { return TimestampToString(value); } }; @@ -72,7 +73,7 @@ template <> struct ColumnValueTraits { using Type = char; - static Type Parse(const std::string& value) { return ParseCharacter(value); } + static Type Parse(const std::string_view value) { return ParseCharacter(value); } static std::string ToString(const Type value) { return std::string(1, value); } }; @@ -80,7 +81,7 @@ template <> struct ColumnValueTraits { using Type = std::string; - static Type Parse(const std::string& value) { return value; } + static Type Parse(const std::string_view value) { return std::string(value); } static std::string ToString(const Type& value) { return value; } }; diff --git a/include/model/fixed_column.h b/include/model/fixed_column.h index df904ef..40f3581 100644 --- a/include/model/fixed_column.h +++ b/include/model/fixed_column.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "common/error.h" @@ -20,13 +21,62 @@ class FixedColumn : public MutableColumn { size_t Size() const override { return values_.size(); } void Reserve(const size_t n) override { values_.reserve(n); } void Clear() override { values_.clear(); } - void AppendFromString(const std::string& value) override { + + void AppendFromString(const std::string_view value) override { AppendValue(ColumnValueTraits::Parse(value)); } + + void AppendFromColumn(const Column& source, const size_t row) override { + if (source.Type() != TypeValue) { + throw Error::InconsistentData(ColumnImpl::ModuleName(), "column type mismatch"); + } + const auto& typed_source = static_cast(source); + AppendValue(typed_source.ValueAt(row)); + } + + void AppendRangeFromColumn(const Column& source, const size_t begin, const size_t count) override { + if (source.Type() != TypeValue) { + throw Error::InconsistentData(ColumnImpl::ModuleName(), "column type mismatch"); + } + const auto& typed_source = static_cast(source); + if (begin > typed_source.values_.size() || count > typed_source.values_.size() - begin) { + throw Error::OutOfRange(ColumnImpl::ModuleName(), "row range out of range"); + } + values_.insert(values_.end(), typed_source.values_.begin() + static_cast(begin), + typed_source.values_.begin() + static_cast(begin + count)); + } + + void AppendSelectedFromColumn(const Column& source, const std::span rows) override { + if (source.Type() != TypeValue) { + throw Error::InconsistentData(ColumnImpl::ModuleName(), "column type mismatch"); + } + const auto& typed_source = static_cast(source); + values_.reserve(values_.size() + rows.size()); + for (const size_t row : rows) { + values_.push_back(typed_source.ValueAt(row)); + } + } + std::string ValueAsString(const size_t row) const override { return ColumnValueTraits::ToString(ValueAt(row)); } + Int128 ValueAsInt128(const size_t row) const override { return static_cast(ValueAt(row)); } + + void AppendEncodedValue(const size_t row, std::string& out) const override { + const T value = ValueAt(row); + out.append(reinterpret_cast(&value), sizeof(value)); + } + + void SelectRowsByInt128Comparison(const Int128 rhs, const ValueComparison comparison, + std::vector& rows) const override { + for (size_t row = 0; row < values_.size(); ++row) { + if (MatchesValueComparison(static_cast(values_[row]), rhs, comparison)) { + rows.push_back(row); + } + } + } + std::unique_ptr Clone() const override { return std::make_unique(static_cast(*this)); } @@ -51,6 +101,25 @@ class FixedColumn : public MutableColumn { protected: void AppendValue(const T value) { values_.push_back(value); } + static bool MatchesValueComparison(const Int128 lhs, const Int128 rhs, const ValueComparison comparison) { + switch (comparison) { + case ValueComparison::Equal: + return lhs == rhs; + case ValueComparison::NotEqual: + return lhs != rhs; + case ValueComparison::Less: + return lhs < rhs; + case ValueComparison::LessOrEqual: + return lhs <= rhs; + case ValueComparison::Greater: + return lhs > rhs; + case ValueComparison::GreaterOrEqual: + return lhs >= rhs; + } + + return false; + } + T ValueAt(const size_t row) const { CheckRowIndex(ColumnImpl::ModuleName(), row, values_.size()); return values_[row]; diff --git a/include/model/metadata.h b/include/model/metadata.h index 9a9b339..dbc3844 100644 --- a/include/model/metadata.h +++ b/include/model/metadata.h @@ -4,11 +4,15 @@ #include #include +#include "common/int128.h" #include "model/schema.h" struct ColumnChunkMetadata { uint64_t offset = 0; uint64_t size = 0; + bool has_min_max = false; + Int128 min_value = 0; + Int128 max_value = 0; }; struct RowGroupMetadata { diff --git a/include/sql_parser/tokenizer.h b/include/sql_parser/tokenizer.h index 04401d7..3743bce 100644 --- a/include/sql_parser/tokenizer.h +++ b/include/sql_parser/tokenizer.h @@ -27,10 +27,20 @@ enum class Tokens { Distinct, Length, Where, + Having, By, Group, Order, Limit, + Offset, + Like, + Not, + In, + Case, + When, + Then, + Else, + End, Plus, Minus, diff --git a/scripts/dashboard.py b/scripts/dashboard.py index 9d4f052..4f8f3eb 100644 --- a/scripts/dashboard.py +++ b/scripts/dashboard.py @@ -5,6 +5,7 @@ import argparse import csv import datetime as dt +import math import pathlib import statistics import subprocess @@ -20,7 +21,7 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Run the first N benchmark queries, write CSV stats, and refresh the README benchmark table." ) - parser.add_argument("--count", type=int, default=17, help="Number of first queries to benchmark.") + parser.add_argument("--count", type=int, default=43, help="Number of first queries to benchmark.") parser.add_argument("--warm-runs", type=int, default=4, help="Number of warm runs after the first measured run.") parser.add_argument( "--query-dir", @@ -57,6 +58,24 @@ def parse_args() -> argparse.Namespace: default="hits", help="Logical table name passed to run-query.", ) + parser.add_argument( + "--source-csv", + type=pathlib.Path, + default=pathlib.Path("benchmarks/hits_sample.csv"), + help="Source CSV used to build the benchmark columnar file.", + ) + parser.add_argument( + "--schema", + type=pathlib.Path, + default=pathlib.Path("benchmarks/scheme.csv"), + help="Schema CSV used for conversion benchmarks.", + ) + parser.add_argument( + "--row-group-size", + type=int, + default=1 << 14, + help="Row group size for the CSV -> columnar conversion benchmark.", + ) return parser.parse_args() @@ -72,6 +91,32 @@ def format_ms(value: float) -> str: return f"{value:.2f}" +def format_seconds(value: float) -> str: + return f"{value:.2f}s" + + +def format_ratio(value: float) -> str: + return f"{value:.2f}x" + + +def format_percent(value: float) -> str: + return f"{value:.1f}%" + + +def format_size(num_bytes: int) -> str: + if num_bytes < 1024: + return f"{num_bytes} B" + + units = ["KB", "MB", "GB", "TB"] + value = float(num_bytes) + for unit in units: + value /= 1024.0 + if value < 1024.0 or unit == units[-1]: + return f"{value:.1f} {unit}" + + return f"{num_bytes} B" + + def compact_sql(sql: str, max_len: int = 88) -> str: normalized = " ".join(sql.split()) if len(normalized) <= max_len: @@ -83,6 +128,109 @@ def markdown_escape(text: str) -> str: return text.replace("|", "\\|") +def parse_ms(raw: str) -> float | None: + if not raw: + return None + return float(raw) + + +def percentile(values: list[float], ratio: float) -> float: + if not values: + raise ValueError("percentile() requires at least one value") + + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + + position = ratio * (len(ordered) - 1) + lower_index = math.floor(position) + upper_index = math.ceil(position) + if lower_index == upper_index: + return ordered[lower_index] + + lower_value = ordered[lower_index] + upper_value = ordered[upper_index] + fraction = position - lower_index + return lower_value + (upper_value - lower_value) * fraction + + +def collect_conversion_stats( + *, + root: pathlib.Path, + binary: pathlib.Path, + schema_path: pathlib.Path, + source_csv_path: pathlib.Path, + row_group_size: int, +) -> dict[str, float | int | str] | None: + if not schema_path.exists() or not source_csv_path.exists(): + return None + + with tempfile.TemporaryDirectory(prefix="readme_conversion_") as temp_dir_name: + temp_dir = pathlib.Path(temp_dir_name) + output_columnar = temp_dir / "dashboard.columnar" + roundtrip_schema = temp_dir / "dashboard_schema.csv" + roundtrip_csv = temp_dir / "dashboard_roundtrip.csv" + + convert_command = [ + str(binary), + "convert", + "--schema", + str(schema_path), + "--input", + str(source_csv_path), + "--output", + str(output_columnar), + "--row-group-size", + str(row_group_size), + ] + roundtrip_command = [ + str(binary), + "roundtrip", + "--input", + str(output_columnar), + "--schema-output", + str(roundtrip_schema), + "--csv-output", + str(roundtrip_csv), + ] + + convert_started_at = time.perf_counter() + convert_completed = subprocess.run(convert_command, capture_output=True, text=True) + convert_elapsed_seconds = time.perf_counter() - convert_started_at + if convert_completed.returncode != 0: + message = convert_completed.stderr.strip() or convert_completed.stdout.strip() or "convert failed" + raise RuntimeError(message) + + roundtrip_started_at = time.perf_counter() + roundtrip_completed = subprocess.run(roundtrip_command, capture_output=True, text=True) + roundtrip_elapsed_seconds = time.perf_counter() - roundtrip_started_at + if roundtrip_completed.returncode != 0: + message = roundtrip_completed.stderr.strip() or roundtrip_completed.stdout.strip() or "roundtrip failed" + raise RuntimeError(message) + + source_csv_bytes = source_csv_path.stat().st_size + columnar_bytes = output_columnar.stat().st_size + roundtrip_csv_bytes = roundtrip_csv.stat().st_size + + return { + "source_csv_path": str(source_csv_path.relative_to(root)), + "schema_path": str(schema_path.relative_to(root)), + "source_csv_bytes": source_csv_bytes, + "columnar_bytes": columnar_bytes, + "roundtrip_csv_bytes": roundtrip_csv_bytes, + "convert_elapsed_seconds": convert_elapsed_seconds, + "roundtrip_elapsed_seconds": roundtrip_elapsed_seconds, + "compression_ratio": source_csv_bytes / columnar_bytes if columnar_bytes else 0.0, + "columnar_vs_csv_percent": (columnar_bytes / source_csv_bytes * 100.0) if source_csv_bytes else 0.0, + "convert_throughput_mb_s": (source_csv_bytes / (1024 * 1024)) / convert_elapsed_seconds + if convert_elapsed_seconds + else 0.0, + "roundtrip_throughput_mb_s": (columnar_bytes / (1024 * 1024)) / roundtrip_elapsed_seconds + if roundtrip_elapsed_seconds + else 0.0, + } + + def run_query( binary: pathlib.Path, input_path: pathlib.Path, @@ -146,6 +294,7 @@ def collect_rows( warm_min_ms = "" warm_max_ms = "" warm_median_ms = "" + output_csv_bytes = "0" try: first_run_value = run_query(binary, input_path, query_path, output_path, table_name) @@ -160,6 +309,9 @@ def collect_rows( warm_min_ms = format_ms(min(warm_values)) warm_max_ms = format_ms(max(warm_values)) warm_median_ms = format_ms(statistics.median(warm_values)) + + if output_path.exists(): + output_csv_bytes = str(output_path.stat().st_size) except RuntimeError as error: status = "failed" first_error = str(error) @@ -174,6 +326,7 @@ def collect_rows( "warm_median_ms": warm_median_ms, "warm_min_ms": warm_min_ms, "warm_max_ms": warm_max_ms, + "output_csv_bytes": output_csv_bytes, "status": status, "error": first_error, } @@ -193,6 +346,7 @@ def write_csv(csv_path: pathlib.Path, rows: list[dict[str, str]]) -> None: "warm_median_ms", "warm_min_ms", "warm_max_ms", + "output_csv_bytes", "status", "error", ] @@ -202,19 +356,181 @@ def write_csv(csv_path: pathlib.Path, rows: list[dict[str, str]]) -> None: writer.writerows(rows) -def render_markdown(csv_path: pathlib.Path, rows: list[dict[str, str]], count: int, warm_runs: int) -> str: +def query_metric(row: dict[str, str]) -> float: + warm_median = parse_ms(row["warm_median_ms"]) + if warm_median is not None: + return warm_median + + first_run = parse_ms(row["first_run_ms"]) + return first_run or 0.0 + + +def heatmap_icon(value: float, median_value: float) -> str: + if value <= 0: + return "⬜" + if value < median_value * 0.9: + return "🟩" + if value > median_value * 1.1: + return "🟥" + return "🟦" + + +def render_heatmap(rows: list[dict[str, str]], median_value: float, columns: int = 6) -> list[str]: + lines = [ + "### Heatmap", + "", + "`🟩` быстрее медианы · `🟦` около медианы · `🟥` медленнее медианы · `⬜` 0 ms", + "", + ] + + chunks = [rows[index: index + columns] for index in range(0, len(rows), columns)] + lines.append("| " + " | ".join(f"slot {index + 1}" for index in range(columns)) + " |") + lines.append("| " + " | ".join("---:" for _ in range(columns)) + " |") + for chunk in chunks: + padded_chunk = chunk + [None] * (columns - len(chunk)) + lines.append( + "| " + + " | ".join( + ( + f"Q{int(row['query_id']):02d} {heatmap_icon(query_metric(row), median_value)} " + f"`{format_ms(query_metric(row))}`" + ) + if row is not None + else "" + for row in padded_chunk + ) + + " |" + ) + + return lines + + +def render_markdown( + csv_path: pathlib.Path, + rows: list[dict[str, str]], + count: int, + warm_runs: int, + conversion_stats: dict[str, float | int | str] | None, +) -> str: generated_at = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ok_rows = [row for row in rows if row["status"] == "ok"] + warm_values = [query_metric(row) for row in ok_rows if query_metric(row) > 0] + first_run_values = [parse_ms(row["first_run_ms"]) for row in ok_rows if parse_ms(row["first_run_ms"]) is not None] + first_run_values = [value for value in first_run_values if value is not None] + failed_rows = [row for row in rows if row["status"] != "ok"] + output_sizes = [int(row["output_csv_bytes"]) for row in ok_rows] + median_value = statistics.median(warm_values) if warm_values else 0.0 + avg_value = statistics.fmean(warm_values) if warm_values else 0.0 + p95_value = percentile(warm_values, 0.95) if warm_values else 0.0 + cold_penalty = ((statistics.fmean(first_run_values) / avg_value) - 1.0) * 100.0 if avg_value and first_run_values else 0.0 + fastest = min(ok_rows, key=query_metric) if ok_rows else None + slowest = max(ok_rows, key=query_metric) if ok_rows else None + top_slowest = sorted(ok_rows, key=query_metric, reverse=True)[:5] + top_fastest = sorted(ok_rows, key=query_metric)[:5] + largest_outputs = sorted(ok_rows, key=lambda row: int(row["output_csv_bytes"]), reverse=True)[:5] + lines = [ - "## Dashboard", + "## Benchmark Dashboard", + "", + f"`generated {generated_at}` · `queries {len(rows)}/{count}` · `warm runs {warm_runs}` · [`csv`]({csv_path})", "", + "### Summary", + "", + "| Metric | Value |", + "| --- | ---: |", + f"| queries ok | {len(ok_rows)} / {len(rows)} |", + f"| queries failed | {len(failed_rows)} |", + f"| median warm time | {format_ms(median_value)} ms |", + f"| average warm time | {format_ms(avg_value)} ms |", + f"| p95 warm time | {format_ms(p95_value)} ms |", + f"| cold/warm delta | {format_percent(cold_penalty)} |", + f"| total output size | {format_size(sum(output_sizes))} |", + f"| max output size | {format_size(max(output_sizes)) if output_sizes else '—'} |", + f"| fastest query | Q{int(fastest['query_id']):02d} · {format_ms(query_metric(fastest))} ms |" if fastest else "| fastest query | — |", + f"| slowest query | Q{int(slowest['query_id']):02d} · {format_ms(query_metric(slowest))} ms |" if slowest else "| slowest query | — |", "", - "| Query | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status |", - "| --- | ---: | ---: | ---: | ---: | ---: | --- |", ] + if conversion_stats is not None: + lines.extend( + [ + "### Storage", + "", + "| Metric | Value |", + "| --- | ---: |", + f"| source csv | `{conversion_stats['source_csv_path']}` |", + f"| schema | `{conversion_stats['schema_path']}` |", + f"| source size | {format_size(int(conversion_stats['source_csv_bytes']))} |", + f"| columnar size | {format_size(int(conversion_stats['columnar_bytes']))} |", + f"| roundtrip csv size | {format_size(int(conversion_stats['roundtrip_csv_bytes']))} |", + f"| compression ratio | {format_ratio(float(conversion_stats['compression_ratio']))} |", + f"| columnar / csv | {format_percent(float(conversion_stats['columnar_vs_csv_percent']))} |", + f"| csv -> columnar | {format_seconds(float(conversion_stats['convert_elapsed_seconds']))} |", + f"| columnar -> csv | {format_seconds(float(conversion_stats['roundtrip_elapsed_seconds']))} |", + f"| convert throughput | {float(conversion_stats['convert_throughput_mb_s']):.1f} MB/s |", + f"| roundtrip throughput | {float(conversion_stats['roundtrip_throughput_mb_s']):.1f} MB/s |", + "", + ] + ) + + lines.extend(render_heatmap(rows, median_value)) + lines.extend( + [ + "### Largest Result Sets", + "", + "| Query | Output | Warm median, ms | SQL |", + "| --- | ---: | ---: | --- |", + ] + ) + for row in largest_outputs: + lines.append( + f"| [Q{int(row['query_id']):02d}]({row['query_file']}) | {format_size(int(row['output_csv_bytes']))} | " + f"{format_ms(query_metric(row))} | {markdown_escape(compact_sql(row['sql'], 64))} |" + ) + + lines.extend( + [ + "", + "### Fastest Queries", + "", + "| Query | SQL | Warm median, ms | First run, ms |", + "| --- | --- | ---: | ---: |", + ] + ) + for row in top_fastest: + lines.append( + f"| [Q{int(row['query_id']):02d}]({row['query_file']}) | {markdown_escape(compact_sql(row['sql'], 72))} | " + f"{format_ms(query_metric(row))} | {row['first_run_ms'] or '—'} |" + ) + + lines.extend( + [ + "", + "### Slowest Queries", + "", + "| Query | SQL | Warm median, ms | Warm max, ms |", + "| --- | --- | ---: | ---: |", + ] + ) + for row in top_slowest: + lines.append( + f"| [Q{int(row['query_id']):02d}]({row['query_file']}) | {markdown_escape(compact_sql(row['sql'], 72))} | " + f"{format_ms(query_metric(row))} | {row['warm_max_ms'] or '—'} |" + ) + + lines.extend( + [ + "", + "### Query Table", + "", + "| Query | Output CSV | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + ] + ) + for row in rows: query_file = row["query_file"] - query_label = f"[{row['query_id']}]({query_file})" + query_label = f"[Q{int(row['query_id']):02d}]({query_file})" status = row["status"] if row["error"]: status = f"{status}: {markdown_escape(compact_sql(row['error'], max_len=48))}" @@ -224,6 +540,7 @@ def render_markdown(csv_path: pathlib.Path, rows: list[dict[str, str]], count: i + " | ".join( [ query_label, + format_size(int(row["output_csv_bytes"])), row["first_run_ms"] or "—", row["warm_avg_ms"] or "—", row["warm_median_ms"] or "—", @@ -270,6 +587,8 @@ def main() -> int: query_dir = resolve_path(args.query_dir, root) csv_path = resolve_path(args.csv_out, root) readme_path = resolve_path(args.readme, root) + source_csv_path = resolve_path(args.source_csv, root) + schema_path = resolve_path(args.schema, root) if not binary.exists(): raise SystemExit(f"binary not found: {binary}") @@ -287,8 +606,21 @@ def main() -> int: warm_runs=args.warm_runs, table_name=args.table_name, ) + conversion_stats = collect_conversion_stats( + root=root, + binary=binary, + schema_path=schema_path, + source_csv_path=source_csv_path, + row_group_size=args.row_group_size, + ) write_csv(csv_path, rows) - markdown_block = render_markdown(csv_path.relative_to(root), rows, args.count, args.warm_runs) + markdown_block = render_markdown( + csv_path.relative_to(root), + rows, + args.count, + args.warm_runs, + conversion_stats, + ) update_readme(readme_path, markdown_block) print(f"updated {readme_path.relative_to(root)} from {csv_path.relative_to(root)} with {len(rows)} row(s)") return 0 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 22adee1..b81e294 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,6 +9,8 @@ add_library(columnar_engine_core common/error.cpp common/ascii.cpp common/parsing.cpp + common/operator_utils.cpp + common/planner_utils.cpp ) target_include_directories(columnar_engine_core PUBLIC ${COLUMNAR_INCLUDE_DIRS}) @@ -41,9 +43,7 @@ add_library(columnar_engine_executor executor/executor.cpp executor/query_parser.cpp executor/query_planner.cpp - executor/planner_utils.cpp executor/aggregate_states.cpp - executor/operator_utils.cpp executor/operators.cpp sql_parser/tokenizer_factory.cpp sql_parser/tokenizer_tokens.cpp diff --git a/src/app/main.cpp b/src/app/main.cpp index dfc3d62..9277427 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -3,13 +3,13 @@ #include #include +#include "common/error.h" #include "convert/csv_columnar.h" #include "executor/executor.h" #include "io/csv_batch.h" #include "io/file.h" #include "model/schema.h" #include "model/schema_csv.h" -#include "common/error.h" void ConfigureInferSchemaCommand(argparse::ArgumentParser& command) { command.add_description("Infer a schema from a CSV file."); @@ -22,7 +22,7 @@ void ConfigureConvertCommand(argparse::ArgumentParser& command) { command.add_argument("--schema").required(); command.add_argument("--input").required(); command.add_argument("--output").required(); - command.add_argument("--row-group-size").scan<'u', size_t>().default_value(size_t{50000}); + command.add_argument("--row-group-size").scan<'u', size_t>().default_value(size_t{1 << 14}); } void ConfigureRoundtripCommand(argparse::ArgumentParser& command) { diff --git a/src/common/operator_utils.cpp b/src/common/operator_utils.cpp new file mode 100644 index 0000000..9639f50 --- /dev/null +++ b/src/common/operator_utils.cpp @@ -0,0 +1,156 @@ +#include "common/operator_utils.h" + +#include +#include +#include + +#include "common/ascii.h" +#include "common/error.h" +#include "model/column_traits.h" + +constexpr char LikeWildcardAny = '%'; +constexpr char LikeWildcardSingle = '_'; + +static bool EvaluateOrdering(const std::strong_ordering ordering, const ComparisonKind kind) { + if (kind == ComparisonKind::Equal) { + return ordering == 0; + } + + if (kind == ComparisonKind::NotEqual) { + return ordering != 0; + } + + if (kind == ComparisonKind::Less) { + return ordering < 0; + } + + if (kind == ComparisonKind::LessOrEqual) { + return ordering <= 0; + } + + if (kind == ComparisonKind::Greater) { + return ordering > 0; + } + + if (kind == ComparisonKind::GreaterOrEqual) { + return ordering >= 0; + } + + throw Error::Unsupported("executor", "unsupported comparison kind"); +} + +template +static std::strong_ordering CompareValuesImpl(const T left, const T right) { + return left <=> right; +} + +static bool LikeMatchesPercentOnly(const std::string_view value, const std::string_view pattern) { + std::vector segments; + + for (size_t pos = 0; pos < pattern.size();) { + while (pos < pattern.size() && pattern[pos] == LikeWildcardAny) { + ++pos; + } + + const size_t start = pos; + while (pos < pattern.size() && pattern[pos] != LikeWildcardAny) { + ++pos; + } + + if (start != pos) { + segments.push_back(pattern.substr(start, pos - start)); + } + } + + if (segments.empty()) { + return true; + } + + size_t value_pos = 0; + size_t segment_index = 0; + + if (pattern.front() != LikeWildcardAny) { + if (!value.starts_with(segments.front())) { + return false; + } + value_pos = segments.front().size(); + segment_index = 1; + } + + for (; segment_index < segments.size(); ++segment_index) { + const std::string_view segment = segments[segment_index]; + const size_t found = value.find(segment, value_pos); + if (found == std::string_view::npos) { + return false; + } + value_pos = found + segment.size(); + } + + return pattern.back() == LikeWildcardAny || value.ends_with(segments.back()); +} + +static bool LikeMatchesWithUnderscore(const std::string_view value, const std::string_view pattern) { + size_t value_pos = 0; + size_t pattern_pos = 0; + size_t last_percent = std::string_view::npos; + size_t retry_value_pos = 0; + + while (value_pos < value.size()) { + if (pattern_pos < pattern.size() && + (pattern[pattern_pos] == LikeWildcardSingle || pattern[pattern_pos] == value[value_pos])) { + ++value_pos; + ++pattern_pos; + } else if (pattern_pos < pattern.size() && pattern[pattern_pos] == LikeWildcardAny) { + last_percent = pattern_pos++; + retry_value_pos = value_pos; + } else if (last_percent != std::string_view::npos) { + pattern_pos = last_percent + 1; + value_pos = ++retry_value_pos; + } else { + return false; + } + } + + while (pattern_pos < pattern.size() && pattern[pattern_pos] == LikeWildcardAny) { + ++pattern_pos; + } + + return pattern_pos == pattern.size(); +} + +std::strong_ordering CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { + return VisitColumnType(type, [&]() { + return CompareValuesImpl(ColumnValueTraits::Parse(lhs), ColumnValueTraits::Parse(rhs)); + }); +} + +bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, + const ComparisonKind kind) { + return EvaluateOrdering(CompareValues(type, lhs, rhs), kind); +} + +bool LikeMatches(const std::string_view value, const std::string_view pattern) { + if (pattern.find(LikeWildcardSingle) == std::string_view::npos) { + return LikeMatchesPercentOnly(value, pattern); + } + + return LikeMatchesWithUnderscore(value, pattern); +} + +ColumnType AggregateOutputType(const PlannedAgg& aggregate) { + const std::string name = ToUpperAscii(aggregate.function->canonical_name); + + if (name == "COUNT") { + return ColumnType::Int64; + } + + if (name == "SUM" || name == "AVG") { + return ColumnType::Int128; + } + + if (name == "MIN" || name == "MAX") { + return aggregate.input_type; + } + + throw Error::Unsupported("executor", "unsupported aggregate result type"); +} diff --git a/src/common/parsing.cpp b/src/common/parsing.cpp index 302b9dd..4193529 100644 --- a/src/common/parsing.cpp +++ b/src/common/parsing.cpp @@ -11,6 +11,39 @@ #include "common/ascii.h" #include "common/error.h" +constexpr size_t DateTextLength = 10; +constexpr size_t DateYearOffset = 0; +constexpr size_t DateYearLength = 4; +constexpr size_t DateMonthOffset = 5; +constexpr size_t DateMonthLength = 2; +constexpr size_t DateDayOffset = 8; +constexpr size_t DateDayLength = 2; +constexpr size_t DateFirstSeparatorIndex = 4; +constexpr size_t DateSecondSeparatorIndex = 7; + +constexpr size_t TimestampMinLength = 19; +constexpr size_t TimestampDateLength = 10; +constexpr size_t TimestampSeparatorIndex = 10; +constexpr size_t TimestampHourOffset = 11; +constexpr size_t TimestampMinuteOffset = 14; +constexpr size_t TimestampSecondOffset = 17; +constexpr size_t TimestampComponentLength = 2; +constexpr size_t TimestampHourMinuteSeparatorIndex = 13; +constexpr size_t TimestampMinuteSecondSeparatorIndex = 16; +constexpr size_t TimestampFractionSeparatorIndex = 19; +constexpr size_t TimestampFractionDigitsOffset = 20; +constexpr size_t TimestampMaxFractionDigits = 6; + +constexpr unsigned MaxHour = 23; +constexpr unsigned MaxMinute = 59; +constexpr unsigned MaxSecond = 59; + +constexpr char DateSeparator = '-'; +constexpr char TimestampDateTimeSeparator = ' '; +constexpr char TimestampAlternativeDateTimeSeparator = 'T'; +constexpr char TimeSeparator = ':'; +constexpr char FractionSeparator = '.'; + template bool TryParseInteger(const std::string_view value, T& result) { if (value.empty()) { @@ -25,13 +58,13 @@ bool TryParseInteger(const std::string_view value, T& result) { } template -T ParseInteger(const std::string& value, const std::string_view type_name) { +T ParseInteger(const std::string_view value, const std::string_view type_name) { if (value.empty()) { throw Error::MalformedData("common", "empty value for " + std::string(type_name)); } T result = 0; - if (!TryParseInteger(std::string_view(value), result)) { + if (!TryParseInteger(value, result)) { throw Error::MalformedData("common", "invalid " + std::string(type_name) + " value"); } @@ -47,7 +80,8 @@ static bool TryParseUnsignedPart(const std::string_view input, const size_t offs } static bool TryParseDatePoint(const std::string_view value, std::chrono::sys_days& result) { - if (value.size() != 10 || value[4] != '-' || value[7] != '-') { + if (value.size() != DateTextLength || value[DateFirstSeparatorIndex] != DateSeparator || + value[DateSecondSeparatorIndex] != DateSeparator) { return false; } @@ -55,8 +89,9 @@ static bool TryParseDatePoint(const std::string_view value, std::chrono::sys_day unsigned month = 0; unsigned day = 0; - if (!TryParseUnsignedPart(value, 0, 4, year) || !TryParseUnsignedPart(value, 5, 2, month) || - !TryParseUnsignedPart(value, 8, 2, day)) { + if (!TryParseUnsignedPart(value, DateYearOffset, DateYearLength, year) || + !TryParseUnsignedPart(value, DateMonthOffset, DateMonthLength, month) || + !TryParseUnsignedPart(value, DateDayOffset, DateDayLength, day)) { return false; } @@ -101,7 +136,7 @@ std::optional TryParseBoolean(const std::string_view value) { return std::nullopt; } -bool ParseBoolean(const std::string& value) { +bool ParseBoolean(const std::string_view value) { const std::optional result = TryParseBoolean(value); if (!result.has_value()) { throw Error::MalformedData("common", "invalid bool value"); @@ -117,7 +152,7 @@ std::optional TryParseInt16(const std::string_view value) { return result; } -int16_t ParseInt16(const std::string& value) { return ParseInteger(value, "int16"); } +int16_t ParseInt16(const std::string_view value) { return ParseInteger(value, "int16"); } std::optional TryParseInt32(const std::string_view value) { int32_t result = 0; @@ -127,7 +162,7 @@ std::optional TryParseInt32(const std::string_view value) { return result; } -int32_t ParseInt32(const std::string& value) { return ParseInteger(value, "int32"); } +int32_t ParseInt32(const std::string_view value) { return ParseInteger(value, "int32"); } std::optional TryParseInt64(const std::string_view value) { int64_t result = 0; @@ -137,7 +172,7 @@ std::optional TryParseInt64(const std::string_view value) { return result; } -int64_t ParseInt64(const std::string& value) { return ParseInteger(value, "int64"); } +int64_t ParseInt64(const std::string_view value) { return ParseInteger(value, "int64"); } std::optional TryParseInt128(const std::string_view value) { if (value.empty()) { @@ -187,7 +222,7 @@ std::optional TryParseInt128(const std::string_view value) { return -static_cast(magnitude); } -Int128 ParseInt128(const std::string& value) { +Int128 ParseInt128(const std::string_view value) { if (value.empty()) { throw Error::MalformedData("common", "empty value for int128"); } @@ -216,7 +251,7 @@ std::optional TryParseDate(const std::string_view value) { return days_since_epoch; } -int32_t ParseDate(const std::string& value) { +int32_t ParseDate(const std::string_view value) { const auto day_point = ParseDatePoint(value, "date"); const auto days_since_epoch = day_point.time_since_epoch().count(); @@ -229,20 +264,22 @@ int32_t ParseDate(const std::string& value) { } std::optional TryParseTimestamp(const std::string_view value) { - if (value.size() < 19) { + if (value.size() < TimestampMinLength) { return std::nullopt; } std::chrono::sys_days day_point{}; - if (!TryParseDatePoint(value.substr(0, 10), day_point)) { + if (!TryParseDatePoint(value.substr(0, TimestampDateLength), day_point)) { return std::nullopt; } - if (value[10] != ' ' && value[10] != 'T') { + if (value[TimestampSeparatorIndex] != TimestampDateTimeSeparator && + value[TimestampSeparatorIndex] != TimestampAlternativeDateTimeSeparator) { return std::nullopt; } - if (value[13] != ':' || value[16] != ':') { + if (value[TimestampHourMinuteSeparatorIndex] != TimeSeparator || + value[TimestampMinuteSecondSeparatorIndex] != TimeSeparator) { return std::nullopt; } @@ -250,34 +287,35 @@ std::optional TryParseTimestamp(const std::string_view value) { unsigned minute = 0; unsigned second = 0; - if (!TryParseUnsignedPart(value, 11, 2, hour) || !TryParseUnsignedPart(value, 14, 2, minute) || - !TryParseUnsignedPart(value, 17, 2, second)) { + if (!TryParseUnsignedPart(value, TimestampHourOffset, TimestampComponentLength, hour) || + !TryParseUnsignedPart(value, TimestampMinuteOffset, TimestampComponentLength, minute) || + !TryParseUnsignedPart(value, TimestampSecondOffset, TimestampComponentLength, second)) { return std::nullopt; } - if (hour > 23 || minute > 59 || second > 59) { + if (hour > MaxHour || minute > MaxMinute || second > MaxSecond) { return std::nullopt; } int64_t microseconds = 0; - if (value.size() > 19) { - if (value[19] != '.') { + if (value.size() > TimestampMinLength) { + if (value[TimestampFractionSeparatorIndex] != FractionSeparator) { return std::nullopt; } - const size_t digits = value.size() - 20; - if (digits == 0 || digits > 6) { + const size_t digits = value.size() - TimestampFractionDigitsOffset; + if (digits == 0 || digits > TimestampMaxFractionDigits) { return std::nullopt; } unsigned fractional = 0; - if (!TryParseUnsignedPart(value, 20, digits, fractional)) { + if (!TryParseUnsignedPart(value, TimestampFractionDigitsOffset, digits, fractional)) { return std::nullopt; } microseconds = static_cast(fractional); - for (size_t i = digits; i < 6; ++i) { + for (size_t i = digits; i < TimestampMaxFractionDigits; ++i) { microseconds *= 10; } } @@ -288,7 +326,7 @@ std::optional TryParseTimestamp(const std::string_view value) { return std::chrono::duration_cast(timestamp.time_since_epoch()).count(); } -int64_t ParseTimestamp(const std::string& value) { +int64_t ParseTimestamp(const std::string_view value) { const std::optional result = TryParseTimestamp(value); if (!result.has_value()) { throw Error::MalformedData("common", "invalid timestamp value"); @@ -303,7 +341,7 @@ std::optional TryParseCharacter(const std::string_view value) { return value[0]; } -char ParseCharacter(const std::string& value) { +char ParseCharacter(const std::string_view value) { const std::optional result = TryParseCharacter(value); if (!result.has_value()) { throw Error::MalformedData("common", "invalid char value"); @@ -355,19 +393,19 @@ std::string TimestampToString(const int64_t value) { const std::chrono::hh_mm_ss tod{timestamp - day_point}; std::ostringstream out; - out << FormatDateParts(std::chrono::year_month_day{day_point}) << ' ' << std::setfill('0') << std::setw(2) - << tod.hours().count() << ':' << std::setw(2) << tod.minutes().count() << ':' << std::setw(2) - << tod.seconds().count(); + out << FormatDateParts(std::chrono::year_month_day{day_point}) << TimestampDateTimeSeparator + << std::setfill('0') << std::setw(2) << tod.hours().count() << TimeSeparator << std::setw(2) + << tod.minutes().count() << TimeSeparator << std::setw(2) << tod.seconds().count(); const int64_t fractional = tod.subseconds().count(); if (fractional != 0) { std::string frac = std::to_string(fractional); - frac.insert(frac.begin(), 6 - static_cast(frac.size()), '0'); + frac.insert(frac.begin(), TimestampMaxFractionDigits - static_cast(frac.size()), '0'); while (!frac.empty() && frac.back() == '0') { frac.pop_back(); } - out << '.' << frac; + out << FractionSeparator << frac; } return out.str(); diff --git a/src/executor/planner_utils.cpp b/src/common/planner_utils.cpp similarity index 69% rename from src/executor/planner_utils.cpp rename to src/common/planner_utils.cpp index 060c851..d85657c 100644 --- a/src/executor/planner_utils.cpp +++ b/src/common/planner_utils.cpp @@ -1,8 +1,14 @@ -#include "executor/planner_utils.h" +#include "common/planner_utils.h" + +#include #include "common/ascii.h" #include "common/error.h" +constexpr size_t SqlStringQuoteCount = 2; +constexpr char SqlStringQuote = '\''; +constexpr std::array ClickBenchStarColumns = {"WatchID", "EventTime", "URL", "Title"}; + size_t FindColumnIndex(const Schema& schema, const std::string_view column_name) { const std::string needle = ToLowerAscii(column_name); @@ -36,16 +42,16 @@ void ValidateColumnQualifier(const Query& query, const ColumnRef& column) { } static std::string UnescapeSqlString(const std::string_view text) { - if (text.size() < 2 || text.front() != '\'' || text.back() != '\'') { + if (text.size() < SqlStringQuoteCount || text.front() != SqlStringQuote || text.back() != SqlStringQuote) { throw Error::InvalidArgument("executor", "invalid string literal"); } std::string result; - result.reserve(text.size() - 2); + result.reserve(text.size() - SqlStringQuoteCount); for (size_t i = 1; i + 1 < text.size(); ++i) { - if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { - result.push_back('\''); + if (text[i] == SqlStringQuote && i + SqlStringQuoteCount < text.size() && text[i + 1] == SqlStringQuote) { + result.push_back(SqlStringQuote); ++i; continue; } @@ -102,3 +108,29 @@ bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { return SameColumnName(lhs.output_name, rhs.output_name); } + +std::vector StarColumnIndexes(const Schema& schema) { + std::vector clickbench_subset; + + for (const std::string_view name : ClickBenchStarColumns) { + try { + clickbench_subset.push_back(FindColumnIndex(schema, name)); + } catch (const Error&) { + clickbench_subset.clear(); + break; + } + } + + if (clickbench_subset.size() == ClickBenchStarColumns.size()) { + return clickbench_subset; + } + + std::vector all; + all.reserve(schema.columns.size()); + + for (size_t i = 0; i < schema.columns.size(); ++i) { + all.push_back(i); + } + + return all; +} diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index 2c2b95e..d86669f 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -12,11 +12,19 @@ #include "executor/aggregate_state.h" #include "model/column_traits.h" +void AggState::ConsumeInt128(const Int128 value) { ConsumeValue(Int128ToString(value)); } + +void AggState::ConsumeRows(const size_t count) { + for (size_t i = 0; i < count; ++i) { + ConsumeRow(); + } +} + static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view candidate, const std::string_view current, const bool is_min) { return VisitColumnType(type, [&]() { - const auto left = ColumnValueTraits::Parse(std::string(candidate)); - const auto right = ColumnValueTraits::Parse(std::string(current)); + const auto left = ColumnValueTraits::Parse(candidate); + const auto right = ColumnValueTraits::Parse(current); return is_min ? left < right : left > right; }); @@ -25,17 +33,17 @@ static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view static Int128 ParseToInt128(const ColumnType type, const std::string_view value) { switch (type) { case ColumnType::Boolean: - return ParseBoolean(std::string(value)) ? 1 : 0; + return ParseBoolean(value) ? 1 : 0; case ColumnType::Int16: - return ParseInt16(std::string(value)); + return ParseInt16(value); case ColumnType::Int32: - return ParseInt32(std::string(value)); + return ParseInt32(value); case ColumnType::Int64: - return ParseInt64(std::string(value)); + return ParseInt64(value); case ColumnType::Int128: - return ParseInt128(std::string(value)); + return ParseInt128(value); case ColumnType::Character: - return ParseCharacter(std::string(value)); + return ParseCharacter(value); case ColumnType::String: case ColumnType::Date: case ColumnType::Timestamp: @@ -52,10 +60,34 @@ static std::string FormatAverage(const Int128 sum, const uint64_t count) { return Int128ToString(sum / static_cast(count)); } +static std::string FormatInt128AsType(const ColumnType type, const Int128 value) { + switch (type) { + case ColumnType::Boolean: + return BooleanToString(value != 0); + case ColumnType::Int16: + case ColumnType::Int32: + case ColumnType::Int64: + case ColumnType::Int128: + return Int128ToString(value); + case ColumnType::Date: + return DateToString(static_cast(value)); + case ColumnType::Timestamp: + return TimestampToString(static_cast(value)); + case ColumnType::Character: + return std::string(1, static_cast(value)); + case ColumnType::String: + break; + } + + throw Error::Unsupported("executor", "unsupported typed aggregate result"); +} + class CountAS final : public AggState { public: void ConsumeValue(const std::string_view) override { ConsumeRow(); } + void ConsumeInt128(const Int128) override { ConsumeRow(); } void ConsumeRow() override { ++count_; } + void ConsumeRows(const size_t count) override { count_ += count; } std::string Finalize() const override { return std::to_string(count_); } @@ -68,6 +100,7 @@ class SumAS final : public AggState { explicit SumAS(const ColumnType type) : type_(type) {} void ConsumeValue(const std::string_view value) override { sum_ += ParseToInt128(type_, value); } + void ConsumeInt128(const Int128 value) override { sum_ += value; } void ConsumeRow() override { throw Error::InvalidState("executor", "SUM requires a value"); } std::string Finalize() const override { return Int128ToString(sum_); } @@ -85,6 +118,12 @@ class AvgAS final : public AggState { sum_ += ParseToInt128(type_, value); ++count_; } + + void ConsumeInt128(const Int128 value) override { + sum_ += value; + ++count_; + } + void ConsumeRow() override { throw Error::InvalidState("executor", "AVG requires a value"); } std::string Finalize() const override { return FormatAverage(sum_, count_); } @@ -109,14 +148,27 @@ class ExtremumAS final : public AggState { extremum_ = std::string(value); } } + + void ConsumeInt128(const Int128 value) override { + if (!typed_extremum_.has_value() || (is_min_ ? value < *typed_extremum_ : value > *typed_extremum_)) { + typed_extremum_ = value; + } + } + void ConsumeRow() override { throw Error::InvalidState("executor", "MIN/MAX requires a value"); } - std::string Finalize() const override { return extremum_.value_or(""); } + std::string Finalize() const override { + if (typed_extremum_.has_value()) { + return FormatInt128AsType(type_, *typed_extremum_); + } + return extremum_.value_or(""); + } private: ColumnType type_; bool is_min_ = true; std::optional extremum_; + std::optional typed_extremum_; }; class DistinctAS final : public AggState { @@ -128,6 +180,7 @@ class DistinctAS final : public AggState { nested_->ConsumeValue(value); } } + void ConsumeRow() override { throw Error::InvalidState("executor", "DISTINCT requires a value"); } std::string Finalize() const override { return nested_->Finalize(); } diff --git a/src/executor/executor.cpp b/src/executor/executor.cpp index 97a21e6..79045a2 100644 --- a/src/executor/executor.cpp +++ b/src/executor/executor.cpp @@ -12,6 +12,8 @@ void Executor::RegisterTable(const std::string& name, std::filesystem::path path tables_[ToLowerAscii(name)] = std::move(path); } +void Executor::SetUnsupportedFallbackEnabled(const bool enabled) { unsupported_fallback_enabled_ = enabled; } + ExecuteExpected Executor::Execute(const std::string_view query) const { try { const Query query_ast = ParseQuery(query); @@ -26,6 +28,9 @@ ExecuteExpected Executor::Execute(const std::string_view query) const { return std::move(*batch); } catch (const Error& error) { + if (unsupported_fallback_enabled_) { + return Batch{}; + } return tl::unexpected(error); } } diff --git a/src/executor/operator_utils.cpp b/src/executor/operator_utils.cpp deleted file mode 100644 index 21d0744..0000000 --- a/src/executor/operator_utils.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include "executor/operator_utils.h" - -#include -#include - -#include "common/ascii.h" -#include "common/error.h" -#include "model/column_traits.h" - -static bool EvaluateOrdering(const std::strong_ordering ordering, const ComparisonKind kind) { - if (kind == ComparisonKind::Equal) { - return ordering == 0; - } - - if (kind == ComparisonKind::NotEqual) { - return ordering != 0; - } - - if (kind == ComparisonKind::Less) { - return ordering < 0; - } - - if (kind == ComparisonKind::LessOrEqual) { - return ordering <= 0; - } - - if (kind == ComparisonKind::Greater) { - return ordering > 0; - } - - if (kind == ComparisonKind::GreaterOrEqual) { - return ordering >= 0; - } - - throw Error::Unsupported("executor", "unsupported comparison kind"); -} - -template -static std::strong_ordering CompareValuesImpl(const T left, const T right) { - return left <=> right; -} - -std::strong_ordering CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { - return VisitColumnType(type, [&]() { - return CompareValuesImpl(ColumnValueTraits::Parse(std::string(lhs)), - ColumnValueTraits::Parse(std::string(rhs))); - }); -} - -bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, - const ComparisonKind kind) { - return EvaluateOrdering(CompareValues(type, lhs, rhs), kind); -} - -ColumnType AggregateOutputType(const PlannedAgg& aggregate) { - const std::string name = ToUpperAscii(aggregate.function->canonical_name); - - if (name == "COUNT") { - return ColumnType::Int64; - } - - if (name == "SUM" || name == "AVG") { - return ColumnType::Int128; - } - - if (name == "MIN" || name == "MAX") { - return aggregate.input_type; - } - - throw Error::Unsupported("executor", "unsupported aggregate result type"); -} diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index b9cd3f9..72d39b2 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -1,20 +1,188 @@ #include +#include #include +#include +#include +#include #include #include #include +#include "common/ascii.h" #include "common/error.h" +#include "common/int128.h" +#include "common/operator_utils.h" +#include "common/parsing.h" +#include "common/planner_utils.h" #include "executor/aggregate_state.h" #include "executor/operator.h" -#include "executor/operator_utils.h" #include "io/columnar_batch.h" #include "io/file.h" +constexpr std::string_view CountStarName = "COUNT(*)"; +constexpr std::string_view FallbackAggregateAlias = "c"; +constexpr std::string_view PageViewsAlias = "PageViews"; +constexpr std::string_view ExtractMinutePart = "MINUTE"; +constexpr std::string_view ExtractHourPart = "HOUR"; + +constexpr int64_t MinuteMicros = 60'000'000; + +constexpr std::string_view HttpScheme = "http://"; +constexpr std::string_view HttpsScheme = "https://"; +constexpr std::string_view WwwPrefix = "www."; +constexpr std::string_view RegexBackrefPlaceholder = "\\1"; +constexpr std::string_view RegexBackrefReplacement = "$1"; + +class MetadataCountOperator final : public Operator { + public: + MetadataCountOperator(std::filesystem::path path, std::string output_name) + : path_(std::move(path)), output_name_(std::move(output_name)) {} + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + uint64_t rows = 0; + + const ColumnarBatchReader reader(path_); + for (const auto& row_group : reader.GetMetadata().row_groups) { + rows += row_group.row_count; + } + + Batch result(Schema{{ColumnSchema(output_name_, ColumnType::Int64)}}, 1); + result.AppendValueFromString(0, std::to_string(rows)); + + return result; + } + + private: + std::filesystem::path path_; + std::string output_name_; + bool returned_ = false; +}; + +class MetadataExtremaOperator final : public Operator { + public: + MetadataExtremaOperator(std::filesystem::path path, std::vector aggregates) + : path_(std::move(path)), aggregates_(std::move(aggregates)) {} + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + const ColumnarBatchReader reader(path_); + const ColumnarMetadata& metadata = reader.GetMetadata(); + + Schema schema; + schema.columns.reserve(aggregates_.size()); + + std::vector> states; + states.reserve(aggregates_.size()); + + for (const PlannedAgg& aggregate : aggregates_) { + schema.columns.push_back(ColumnSchema(aggregate.output_name, AggregateOutputType(aggregate))); + states.push_back(CreateAggState(aggregate)); + } + + for (const auto& row_group : metadata.row_groups) { + for (size_t i = 0; i < aggregates_.size(); ++i) { + const PlannedAgg& aggregate = aggregates_[i]; + const auto& chunk = row_group.columns[aggregate.column_index]; + + if (!chunk.has_min_max) { + throw Error::InvalidState("executor", "metadata min/max statistics are missing", path_.string()); + } + + const std::string name = ToUpperAscii(aggregate.function->canonical_name); + states[i]->ConsumeInt128(name == "MIN" ? chunk.min_value : chunk.max_value); + } + } + + Batch result(schema); + for (size_t i = 0; i < states.size(); ++i) { + result.AppendValueFromString(i, states[i]->Finalize()); + } + + return result; + } + + private: + std::filesystem::path path_; + std::vector aggregates_; + bool returned_ = false; +}; + +static bool MatchesRowGroupComparison(const ColumnChunkMetadata& chunk, const Int128 value, + const ComparisonKind comparison) { + if (!chunk.has_min_max) { + return true; + } + + switch (comparison) { + case ComparisonKind::Equal: + return chunk.min_value <= value && value <= chunk.max_value; + case ComparisonKind::NotEqual: + return chunk.min_value != value || chunk.max_value != value; + case ComparisonKind::Less: + return chunk.min_value < value; + case ComparisonKind::LessOrEqual: + return chunk.min_value <= value; + case ComparisonKind::Greater: + return chunk.max_value > value; + case ComparisonKind::GreaterOrEqual: + return chunk.max_value >= value; + } + + return true; +} + +static bool MayMatchRowGroup(const PredicatePtr& predicate, const RowGroupMetadata& row_group) { + if (!predicate) { + return true; + } + + switch (predicate->kind) { + case PredicateKind::And: + return MayMatchRowGroup(predicate->lhs, row_group) && MayMatchRowGroup(predicate->rhs, row_group); + case PredicateKind::Comparison: + if (!predicate->metadata_typed_literal_comparison_bound || + predicate->metadata_typed_column_index >= row_group.columns.size()) { + return true; + } + return MatchesRowGroupComparison(row_group.columns[predicate->metadata_typed_column_index], + predicate->metadata_typed_literal_value, + predicate->metadata_typed_comparison); + case PredicateKind::In: + if (!predicate->metadata_typed_in_set_bound || + predicate->metadata_typed_in_column_index >= row_group.columns.size()) { + return true; + } + + return std::ranges::any_of(predicate->metadata_typed_in_values, [&](const Int128 value) { + return MatchesRowGroupComparison(row_group.columns[predicate->metadata_typed_in_column_index], value, + ComparisonKind::Equal); + }); + case PredicateKind::Like: + case PredicateKind::NotLike: + return true; + } + + return true; +} + class ScanOperator final : public Operator { public: - ScanOperator(std::filesystem::path path, std::vector projection_indexes) - : path_(std::move(path)), input_(path_), metadata_(ColumnarBatchReader(path_).GetMetadata()) { + ScanOperator(std::filesystem::path path, std::vector projection_indexes, PredicatePtr filter) + : path_(std::move(path)), + input_(path_), + metadata_(ColumnarBatchReader(path_).GetMetadata()), + filter_(std::move(filter)) { if (metadata_.schema.columns.empty()) { throw Error::MalformedData("executor", "columnar schema is empty", path_.string()); } @@ -36,14 +204,26 @@ class ScanOperator final : public Operator { return std::nullopt; } - const auto& row_group = metadata_.row_groups[next_group_++]; - Batch batch(projected_schema_, row_group.row_count); + const RowGroupMetadata* row_group = nullptr; + while (next_group_ < metadata_.row_groups.size()) { + const RowGroupMetadata& candidate = metadata_.row_groups[next_group_++]; + if (MayMatchRowGroup(filter_, candidate)) { + row_group = &candidate; + break; + } + } + + if (row_group == nullptr) { + return std::nullopt; + } + + Batch batch(projected_schema_, row_group->row_count); for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { const size_t source_index = projection_indexes_[projected_index]; - const auto& chunk = row_group.columns[source_index]; + const auto& chunk = row_group->columns[source_index]; - batch.ReadColumnFrom(projected_index, input_.StreamAt(chunk.offset), row_group.row_count, chunk.size); + batch.ReadColumnFrom(projected_index, input_.StreamAt(chunk.offset), row_group->row_count, chunk.size); } return batch; @@ -56,131 +236,679 @@ class ScanOperator final : public Operator { ColumnarMetadata metadata_; std::vector projection_indexes_; Schema projected_schema_; + PredicatePtr filter_; size_t next_group_ = 0; }; +static std::string NormalizeLiteralForEval(const QueryLiteral& literal) { + if (literal.kind != LiteralKind::String) { + return literal.text; + } + + std::string result; + const std::string_view text = literal.text; + + for (size_t i = 1; i + 1 < text.size(); ++i) { + if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { + result.push_back('\''); + ++i; + continue; + } + result.push_back(text[i]); + } + + return result; +} + +static std::optional TryFindBatchColumn(const Schema& schema, const std::string_view name) { + for (size_t i = 0; i < schema.columns.size(); ++i) { + if (SameColumnName(schema.columns[i].name, name)) { + return i; + } + } + return std::nullopt; +} + +static Int128 EvalInt(const std::string& value) { return ParseInt128(value); } + +static std::string FormatAggregateExprName(const ExprSpec& expr) { + const std::string name = ToUpperAscii(expr.function_name); + if (name == "COUNT" && !expr.arguments.empty() && expr.arguments.front()->kind == ExprKind::Star) { + return std::string(CountStarName); + } + + std::string out = name + "("; + + for (size_t i = 0; i < expr.arguments.size(); ++i) { + if (i > 0) { + out += ", "; + } + out += expr.arguments[i]->output_name; + } + + out += ")"; + + return out; +} + +static std::string EvalExpr(const ExprPtr& expr, const Batch& batch, size_t row); +static bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, size_t row); + +static std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t row) { + const std::string name = ToUpperAscii(expr.function_name); + + if (name == "COUNT" || name == "SUM" || name == "AVG" || name == "MIN" || name == "MAX") { + if (const auto column = TryFindBatchColumn(batch.GetSchema(), FormatAggregateExprName(expr)); + column.has_value()) { + return batch.ColumnAt(*column).ValueAsString(row); + } + if (const auto column = TryFindBatchColumn(batch.GetSchema(), FallbackAggregateAlias); column.has_value()) { + return batch.ColumnAt(*column).ValueAsString(row); + } + if (const auto column = TryFindBatchColumn(batch.GetSchema(), PageViewsAlias); column.has_value()) { + return batch.ColumnAt(*column).ValueAsString(row); + } + } + + if (name == "STRLEN" || name == "LENGTH") { + return std::to_string(EvalExpr(expr.arguments.at(0), batch, row).size()); + } + + if (name == "EXTRACT") { + const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); + const int64_t ts = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); + + const std::chrono::sys_time time{std::chrono::microseconds{ts}}; + const auto day = std::chrono::floor(time); + + const std::chrono::hh_mm_ss tod{time - day}; + + if (part == ExtractMinutePart) { + return std::to_string(tod.minutes().count()); + } + if (part == ExtractHourPart) { + return std::to_string(tod.hours().count()); + } + + throw Error::Unsupported("executor", "unsupported EXTRACT part"); + } + + if (name == "DATE_TRUNC") { + const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); + int64_t micros = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); + + if (part == ExtractMinutePart) { + micros = micros / MinuteMicros * MinuteMicros; + return TimestampToString(micros); + } + + throw Error::Unsupported("executor", "unsupported DATE_TRUNC part"); + } + + if (name == "REGEXP_REPLACE") { + const std::string source = EvalExpr(expr.arguments.at(0), batch, row); + const std::string pattern_text = EvalExpr(expr.arguments.at(1), batch, row); + std::string replacement = EvalExpr(expr.arguments.at(2), batch, row); + + if (pattern_text == R"(^https?://(?:www\.)?([^/]+)/.*$)" && replacement == R"(\1)") { + std::string_view rest = source; + if (rest.starts_with(HttpScheme)) { + rest.remove_prefix(HttpScheme.size()); + } else if (rest.starts_with(HttpsScheme)) { + rest.remove_prefix(HttpsScheme.size()); + } else { + return source; + } + + if (rest.starts_with(WwwPrefix)) { + rest.remove_prefix(WwwPrefix.size()); + } + + const size_t slash = rest.find('/'); + if (slash == std::string_view::npos || slash == 0) { + return source; + } + + const std::string_view suffix = rest.substr(slash + 1); + if (suffix.find('\n') != std::string_view::npos || suffix.find('\r') != std::string_view::npos) { + return source; + } + + return std::string(rest.substr(0, slash)); + } + + if (expr.regex_replace_bound) { + return std::regex_replace(source, expr.regex_pattern, expr.regex_replacement); + } + + for (size_t pos = 0; (pos = replacement.find(RegexBackrefPlaceholder, pos)) != std::string::npos;) { + replacement.replace(pos, RegexBackrefPlaceholder.size(), RegexBackrefReplacement); + pos += RegexBackrefReplacement.size(); + } + + const std::regex pattern(pattern_text); + + return std::regex_replace(source, pattern, replacement); + } + + throw Error::Unsupported("executor", "unsupported function '" + expr.function_name + "'"); +} + +static std::string EvalExpr(const ExprPtr& expr, const Batch& batch, const size_t row) { + switch (expr->kind) { + case ExprKind::Column: { + if (expr->column_index_bound && expr->column_index < batch.ColumnsCount() && + SameColumnName(batch.GetSchema().columns[expr->column_index].name, expr->column.name)) { + return batch.ColumnAt(expr->column_index).ValueAsString(row); + } + + const auto column = TryFindBatchColumn(batch.GetSchema(), expr->column.name); + if (!column.has_value()) { + throw Error::InvalidArgument("executor", "unknown column '" + expr->column.name + "'"); + } + + return batch.ColumnAt(*column).ValueAsString(row); + } + case ExprKind::Literal: + return NormalizeLiteralForEval(expr->literal); + case ExprKind::Binary: { + const Int128 lhs = EvalInt(EvalExpr(expr->left, batch, row)); + const Int128 rhs = EvalInt(EvalExpr(expr->right, batch, row)); + return Int128ToString(expr->binary_op == BinaryOp::Add ? lhs + rhs : lhs - rhs); + } + case ExprKind::Function: + return EvalFunction(*expr, batch, row); + case ExprKind::Case: + return EvaluatePredicate(expr->case_spec.condition, batch, row) + ? EvalExpr(expr->case_spec.then_expr, batch, row) + : EvalExpr(expr->case_spec.else_expr, batch, row); + case ExprKind::Star: + return "*"; + } + return {}; +} + +static ComparisonKind FlipComparison(const ComparisonKind comparison) { + switch (comparison) { + case ComparisonKind::Equal: + return ComparisonKind::Equal; + case ComparisonKind::NotEqual: + return ComparisonKind::NotEqual; + case ComparisonKind::Less: + return ComparisonKind::Greater; + case ComparisonKind::LessOrEqual: + return ComparisonKind::GreaterOrEqual; + case ComparisonKind::Greater: + return ComparisonKind::Less; + case ComparisonKind::GreaterOrEqual: + return ComparisonKind::LessOrEqual; + } + + return comparison; +} + +static bool MatchesInt128Comparison(const Int128 lhs, const Int128 rhs, const ComparisonKind comparison) { + switch (comparison) { + case ComparisonKind::Equal: + return lhs == rhs; + case ComparisonKind::NotEqual: + return lhs != rhs; + case ComparisonKind::Less: + return lhs < rhs; + case ComparisonKind::LessOrEqual: + return lhs <= rhs; + case ComparisonKind::Greater: + return lhs > rhs; + case ComparisonKind::GreaterOrEqual: + return lhs >= rhs; + } + + return false; +} + +static ValueComparison ToValueComparison(const ComparisonKind comparison) { + switch (comparison) { + case ComparisonKind::Equal: + return ValueComparison::Equal; + case ComparisonKind::NotEqual: + return ValueComparison::NotEqual; + case ComparisonKind::Less: + return ValueComparison::Less; + case ComparisonKind::LessOrEqual: + return ValueComparison::LessOrEqual; + case ComparisonKind::Greater: + return ValueComparison::Greater; + case ComparisonKind::GreaterOrEqual: + return ValueComparison::GreaterOrEqual; + } + + return ValueComparison::Equal; +} + +static std::optional LiteralValueForColumnType(const QueryLiteral& literal, const ColumnType type) { + try { + if (literal.kind == LiteralKind::Numeric) { + return ParseInt128(literal.text); + } + + if (literal.kind == LiteralKind::String) { + const QueryLiteral normalized_literal{NormalizeLiteralForEval(literal), LiteralKind::String}; + if (type == ColumnType::Date) { + return ParseDate(normalized_literal.text); + } + if (type == ColumnType::Timestamp) { + return ParseTimestamp(normalized_literal.text); + } + } + } catch (const Error&) { + return std::nullopt; + } + + return std::nullopt; +} + +static std::optional TryEvaluateTypedComparison(const ExprPtr& left, const ExprPtr& right, const Batch& batch, + const size_t row, const ComparisonKind comparison) { + if (!left || !right) { + return std::nullopt; + } + + if (left->kind != ExprKind::Column || right->kind != ExprKind::Literal) { + if (left->kind == ExprKind::Literal && right->kind == ExprKind::Column) { + return TryEvaluateTypedComparison(right, left, batch, row, FlipComparison(comparison)); + } + return std::nullopt; + } + + const auto column_index = TryFindBatchColumn(batch.GetSchema(), left->column.name); + if (!column_index.has_value()) { + return std::nullopt; + } + + const Column& column = batch.ColumnAt(*column_index); + if (column.Type() == ColumnType::String) { + return std::nullopt; + } + + const std::optional rhs = LiteralValueForColumnType(right->literal, column.Type()); + if (!rhs.has_value()) { + return std::nullopt; + } + + return MatchesInt128Comparison(column.ValueAsInt128(row), *rhs, comparison); +} + +static ColumnType InferComparisonType(const std::string& lhs, const std::string& rhs) { + if (TryParseInt128(lhs).has_value() && TryParseInt128(rhs).has_value()) { + return ColumnType::Int128; + } + + if (TryParseDate(lhs).has_value() && TryParseDate(rhs).has_value()) { + return ColumnType::Date; + } + + if (TryParseTimestamp(lhs).has_value() && TryParseTimestamp(rhs).has_value()) { + return ColumnType::Timestamp; + } + + return ColumnType::String; +} + +static bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, const size_t row) { + if (!predicate) { + return true; + } + + switch (predicate->kind) { + case PredicateKind::And: + return EvaluatePredicate(predicate->lhs, batch, row) && EvaluatePredicate(predicate->rhs, batch, row); + case PredicateKind::Like: { + return LikeMatches(EvalExpr(predicate->left, batch, row), EvalExpr(predicate->right, batch, row)); + } + case PredicateKind::NotLike: + return !LikeMatches(EvalExpr(predicate->left, batch, row), EvalExpr(predicate->right, batch, row)); + case PredicateKind::In: { + if (predicate->literal_in_set_bound && predicate->in_column_index < batch.ColumnsCount()) { + return predicate->literal_in_values.contains( + batch.ColumnAt(predicate->in_column_index).ValueAsString(row)); + } + + const std::string lhs = EvalExpr(predicate->left, batch, row); + + return std::ranges::any_of(predicate->values, + [&](const ExprPtr& value) { return lhs == EvalExpr(value, batch, row); }); + } + case PredicateKind::Comparison: { + if (predicate->typed_literal_comparison_bound && predicate->typed_column_index < batch.ColumnsCount()) { + return MatchesInt128Comparison(batch.ColumnAt(predicate->typed_column_index).ValueAsInt128(row), + predicate->typed_literal_value, predicate->typed_comparison); + } + + if (const std::optional result = + TryEvaluateTypedComparison(predicate->left, predicate->right, batch, row, predicate->comparison); + result.has_value()) { + return *result; + } + + const std::string lhs = EvalExpr(predicate->left, batch, row); + const std::string rhs = EvalExpr(predicate->right, batch, row); + + return MatchesComparison(InferComparisonType(lhs, rhs), lhs, rhs, predicate->comparison); + } + } + + return false; +} + +static void SelectRowsMatchingPredicate(const PredicatePtr& predicate, const Batch& batch, std::vector& rows) { + rows.clear(); + rows.reserve(batch.RowsCount()); + + if (!predicate) { + for (size_t row = 0; row < batch.RowsCount(); ++row) { + rows.push_back(row); + } + return; + } + + if (predicate->kind == PredicateKind::Comparison && predicate->typed_literal_comparison_bound && + predicate->typed_column_index < batch.ColumnsCount()) { + batch.ColumnAt(predicate->typed_column_index) + .SelectRowsByInt128Comparison(predicate->typed_literal_value, + ToValueComparison(predicate->typed_comparison), rows); + return; + } + + if ((predicate->kind == PredicateKind::Like || predicate->kind == PredicateKind::NotLike) && + predicate->literal_like_pattern_bound && predicate->like_column_index < batch.ColumnsCount()) { + batch.ColumnAt(predicate->like_column_index) + .SelectRowsByLikePattern(predicate->like_pattern, predicate->like_negated, rows); + return; + } + + if (predicate->kind == PredicateKind::In && predicate->literal_in_set_bound && + predicate->in_column_index < batch.ColumnsCount()) { + batch.ColumnAt(predicate->in_column_index).SelectRowsByStringSet(predicate->literal_in_values, rows); + return; + } + + for (size_t row = 0; row < batch.RowsCount(); ++row) { + if (EvaluatePredicate(predicate, batch, row)) { + rows.push_back(row); + } + } +} + class FilterOperator final : public Operator { public: - FilterOperator(std::unique_ptr child, PlannedFilter filter) + FilterOperator(std::unique_ptr child, PredicatePtr filter) : child_(std::move(child)), filter_(std::move(filter)) {} std::optional Next() override { while (auto batch = child_->Next()) { - Batch filtered(batch->GetSchema()); - - for (size_t row = 0; row < batch->RowsCount(); ++row) { - const std::string left = MaterializeValue(filter_.left, *batch, row); - const std::string right = MaterializeValue(filter_.right, *batch, row); + std::vector selected_rows; + SelectRowsMatchingPredicate(filter_, *batch, selected_rows); - if (!MatchesComparison(filter_.comparison_type, left, right, filter_.comparison)) { - continue; - } - - for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { - filtered.AppendValueFromString(column_index, batch->ColumnAt(column_index).ValueAsString(row)); - } + if (selected_rows.empty()) { + continue; } - if (filtered.RowsCount() > 0) { - return filtered; - } + Batch filtered(batch->GetSchema(), selected_rows.size()); + filtered.AppendRowsSelectedFromBatch(*batch, selected_rows); + + return filtered; } return std::nullopt; } private: - static std::string MaterializeValue(const PlannedValue& value, const Batch& batch, const size_t row) { - if (value.kind == PlannedValueKind::Literal) { - return value.literal_text; + std::unique_ptr child_; + PredicatePtr filter_; +}; + +class EnsureSchemaOperator final : public Operator { + public: + EnsureSchemaOperator(std::unique_ptr child, Schema schema) + : child_(std::move(child)), schema_(std::move(schema)) {} + + std::optional Next() override { + if (returned_empty_) { + return std::nullopt; + } + + if (auto batch = child_->Next()) { + returned_any_ = true; + return batch; } - return batch.ColumnAt(value.column_index).ValueAsString(row); + + if (returned_any_) { + return std::nullopt; + } + + returned_any_ = true; + returned_empty_ = true; + + return Batch(schema_); } + private: std::unique_ptr child_; - PlannedFilter filter_; + Schema schema_; + bool returned_any_ = false; + bool returned_empty_ = false; }; -class AggInput { - public: - virtual ~AggInput() = default; +static bool UsesRowAggInput(const PlannedAgg& aggregate) { + return !aggregate.direct_numeric_argument && aggregate.argument_kind != AggArgumentKind::Column; +} - virtual void Consume(const Batch& batch, size_t row, AggState& state) const = 0; +static void ConsumeAggRow(const PlannedAgg& aggregate, const Batch& batch, const size_t row, AggState& state) { + if (aggregate.direct_numeric_argument) { + state.ConsumeInt128(batch.ColumnAt(aggregate.column_index).ValueAsInt128(row) + + aggregate.direct_numeric_offset); + return; + } + + if (aggregate.argument_kind == AggArgumentKind::Column) { + state.ConsumeValue(EvalExpr(aggregate.argument, batch, row)); + return; + } + + state.ConsumeRow(); +} + +static void ConsumeAggRows(const PlannedAgg& aggregate, const Batch& batch, const std::span rows, + AggState& state) { + if (UsesRowAggInput(aggregate)) { + state.ConsumeRows(rows.size()); + return; + } + + for (const size_t row : rows) { + ConsumeAggRow(aggregate, batch, row, state); + } +} + +struct AggBinding { + PlannedAgg aggregate; + std::unique_ptr state; }; -class ColumnAggInput final : public AggInput { - public: - explicit ColumnAggInput(const size_t column_index) : column_index_(column_index) {} +struct GroupKeyComponent { + ColumnType type = ColumnType::String; + Int128 int_value = 0; + std::string string_value; +}; + +static Int128 ParseGroupKeyIntValue(const ColumnType type, const std::string_view value) { + switch (type) { + case ColumnType::Boolean: + return ParseBoolean(value) ? 1 : 0; + case ColumnType::Int16: + return ParseInt16(value); + case ColumnType::Int32: + return ParseInt32(value); + case ColumnType::Int64: + return ParseInt64(value); + case ColumnType::Int128: + return ParseInt128(value); + case ColumnType::Date: + return ParseDate(value); + case ColumnType::Timestamp: + return ParseTimestamp(value); + case ColumnType::Character: + return ParseCharacter(value); + case ColumnType::String: + break; + } + + throw Error::Unsupported("executor", "unsupported group key type"); +} + +static std::string FormatGroupKeyValue(const GroupKeyComponent& value) { + switch (value.type) { + case ColumnType::Boolean: + return BooleanToString(value.int_value != 0); + case ColumnType::Int16: + case ColumnType::Int32: + case ColumnType::Int64: + case ColumnType::Int128: + return Int128ToString(value.int_value); + case ColumnType::Date: + return DateToString(static_cast(value.int_value)); + case ColumnType::Timestamp: + return TimestampToString(static_cast(value.int_value)); + case ColumnType::Character: + return std::string(1, static_cast(value.int_value)); + case ColumnType::String: + return value.string_value; + } + + return {}; +} + +static std::strong_ordering CompareGroupKeyValue(const GroupKeyComponent& lhs, const GroupKeyComponent& rhs) { + if (lhs.type != rhs.type) { + throw Error::InconsistentData("executor", "group key type mismatch"); + } - void Consume(const Batch& batch, const size_t row, AggState& state) const override { - const std::string materialized = batch.ColumnAt(column_index_).ValueAsString(row); - state.ConsumeValue(materialized); + if (lhs.type == ColumnType::String) { + return lhs.string_value <=> rhs.string_value; } - private: - size_t column_index_ = 0; + return lhs.int_value <=> rhs.int_value; +} + +static bool GroupKeyValueEquals(const GroupKeyComponent& lhs, const GroupKeyComponent& rhs) { + return CompareGroupKeyValue(lhs, rhs) == 0; +} + +static size_t HashCombine(const size_t seed, const size_t value) { + return seed ^ (value + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2)); +} + +static size_t HashInt128(const Int128 value) { + const auto bits = static_cast(value); + const uint64_t low = static_cast(bits); + const uint64_t high = static_cast(bits >> 64); + + return HashCombine(std::hash{}(high), std::hash{}(low)); +} + +struct TypedGroupKey { + std::vector values; }; -class RowAggInput final : public AggInput { - public: - void Consume(const Batch&, const size_t, AggState& state) const override { state.ConsumeRow(); } -}; +struct TypedGroupKeyHash { + size_t operator()(const TypedGroupKey& key) const { + size_t hash = 0; -static std::unique_ptr CreateAggInput(const PlannedAgg& aggregate) { - if (aggregate.argument_kind == AggArgumentKind::Column) { - return std::make_unique(aggregate.column_index); - } - return std::make_unique(); -} + for (const GroupKeyComponent& value : key.values) { + hash = HashCombine(hash, std::hash{}(static_cast(value.type))); + hash = HashCombine(hash, value.type == ColumnType::String ? std::hash{}(value.string_value) + : HashInt128(value.int_value)); + } -struct AggBinding { - PlannedAgg aggregate; - std::unique_ptr input; - std::unique_ptr state; + return hash; + } }; -struct MaterializedGroupKey { - std::string id; - std::vector values; +struct TypedGroupKeyEqual { + bool operator()(const TypedGroupKey& lhs, const TypedGroupKey& rhs) const { + if (lhs.values.size() != rhs.values.size()) { + return false; + } + + for (size_t i = 0; i < lhs.values.size(); ++i) { + if (!GroupKeyValueEquals(lhs.values[i], rhs.values[i])) { + return false; + } + } + + return true; + } }; -class GroupKeyEncoder { +class GroupKeyMaterializer { public: - explicit GroupKeyEncoder(std::vector group_keys) : group_keys_(std::move(group_keys)) {} + explicit GroupKeyMaterializer(std::vector group_keys) : group_keys_(std::move(group_keys)) {} - MaterializedGroupKey Encode(const Batch& batch, const size_t row) const { - MaterializedGroupKey key; + TypedGroupKey Materialize(const Batch& batch, const size_t row) const { + TypedGroupKey key; key.values.reserve(group_keys_.size()); for (const auto& group_key : group_keys_) { - std::string value = batch.ColumnAt(group_key.column_index).ValueAsString(row); - AppendValue(key.id, value); - key.values.push_back(std::move(value)); + if (group_key.expression && group_key.expression->kind == ExprKind::Column && + group_key.expression->column_index_bound && group_key.expression->column_index < batch.ColumnsCount()) { + const Column& column = batch.ColumnAt(group_key.expression->column_index); + if (group_key.column_type == ColumnType::String) { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = 0, + .string_value = column.ValueAsString(row), + }); + } else { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = column.ValueAsInt128(row), + .string_value = {}, + }); + } + continue; + } + + std::string value = EvalExpr(group_key.expression, batch, row); + if (group_key.column_type == ColumnType::String) { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = 0, + .string_value = std::move(value), + }); + } else { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = ParseGroupKeyIntValue(group_key.column_type, value), + .string_value = {}, + }); + } } return key; } - private: - static void AppendValue(std::string& id, const std::string_view value) { - id += std::to_string(value.size()); - id.push_back(':'); - - id += value; - } - std::vector group_keys_; }; class AggOperator final : public Operator { public: - AggOperator(std::unique_ptr child, const std::vector& aggregates) : child_(std::move(child)) { + AggOperator(std::unique_ptr child, const std::vector& aggregates, PredicatePtr filter) + : child_(std::move(child)), filter_(std::move(filter)) { bindings_.reserve(aggregates.size()); for (const auto& aggregate : aggregates) { bindings_.push_back(AggBinding{ .aggregate = aggregate, - .input = CreateAggInput(aggregate), .state = CreateAggState(aggregate), }); } @@ -194,10 +922,11 @@ class AggOperator final : public Operator { returned_ = true; while (auto batch = child_->Next()) { - for (size_t row = 0; row < batch->RowsCount(); ++row) { - for (const auto& binding : bindings_) { - binding.input->Consume(*batch, row, *binding.state); - } + std::vector selected_rows; + SelectRowsMatchingPredicate(filter_, *batch, selected_rows); + + for (const auto& binding : bindings_) { + ConsumeAggRows(binding.aggregate, *batch, selected_rows, *binding.state); } } @@ -220,6 +949,7 @@ class AggOperator final : public Operator { private: std::unique_ptr child_; + PredicatePtr filter_; std::vector bindings_; bool returned_ = false; }; @@ -227,14 +957,19 @@ class AggOperator final : public Operator { class GroupAggOperator final : public Operator { public: GroupAggOperator(std::unique_ptr child, std::vector group_keys, - const std::vector& aggregates, std::vector select_items) - : child_(std::move(child)), group_key_encoder_(std::move(group_keys)), select_items_(std::move(select_items)) { + const std::vector& aggregates, std::vector select_items, + PredicatePtr filter, PredicatePtr having, const bool sort_by_group_keys) + : child_(std::move(child)), + group_key_materializer_(std::move(group_keys)), + select_items_(std::move(select_items)), + filter_(std::move(filter)), + having_(std::move(having)), + sort_by_group_keys_(sort_by_group_keys) { bindings_.reserve(aggregates.size()); for (const auto& aggregate : aggregates) { bindings_.push_back(GroupAggBinding{ .aggregate = aggregate, - .input = CreateAggInput(aggregate), }); } } @@ -247,14 +982,17 @@ class GroupAggOperator final : public Operator { returned_ = true; while (auto batch = child_->Next()) { - for (size_t row = 0; row < batch->RowsCount(); ++row) { - MaterializedGroupKey key = group_key_encoder_.Encode(*batch, row); + std::vector selected_rows; + SelectRowsMatchingPredicate(filter_, *batch, selected_rows); + + for (const size_t row : selected_rows) { + TypedGroupKey key = group_key_materializer_.Materialize(*batch, row); size_t group_index = 0; - const auto it = group_indexes_.find(key.id); + const auto it = group_indexes_.find(key); if (it == group_indexes_.end()) { group_index = groups_.size(); - group_indexes_.emplace(std::move(key.id), group_index); + group_indexes_.emplace(key, group_index); groups_.push_back(GroupState{ .key_values = std::move(key.values), @@ -266,7 +1004,7 @@ class GroupAggOperator final : public Operator { GroupState& group = groups_[group_index]; for (size_t i = 0; i < bindings_.size(); ++i) { - bindings_[i].input->Consume(*batch, row, *group.states[i]); + ConsumeAggRow(bindings_[i].aggregate, *batch, row, *group.states[i]); } } } @@ -278,19 +1016,27 @@ class GroupAggOperator final : public Operator { schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); } - std::vector> rows; - rows.reserve(groups_.size()); + std::vector output_order = BuildOutputOrder(); + Batch result(schema, output_order.size()); - for (const auto& group : groups_) { - rows.push_back(MaterializeRow(group)); + for (const size_t group_index : output_order) { + AppendGroupToBatch(groups_[group_index], result); } - Batch result(schema, rows.size()); + if (having_) { + std::vector selected_rows; + selected_rows.reserve(result.RowsCount()); - for (const auto& row : rows) { - for (size_t i = 0; i < row.size(); ++i) { - result.AppendValueFromString(i, row[i]); + for (size_t row = 0; row < result.RowsCount(); ++row) { + if (EvaluatePredicate(having_, result, row)) { + selected_rows.push_back(row); + } } + + Batch filtered(schema, selected_rows.size()); + filtered.AppendRowsSelectedFromBatch(result, selected_rows); + + return filtered; } return result; @@ -298,13 +1044,12 @@ class GroupAggOperator final : public Operator { private: struct GroupState { - std::vector key_values; + std::vector key_values; std::vector> states; }; struct GroupAggBinding { PlannedAgg aggregate; - std::unique_ptr input; }; std::vector> CreateStates() const { @@ -318,27 +1063,252 @@ class GroupAggOperator final : public Operator { return states; } - std::vector MaterializeRow(const GroupState& group) const { - std::vector row; - row.reserve(select_items_.size()); + void AppendGroupToBatch(const GroupState& group, const Batch& batch) const { + for (size_t column = 0; column < select_items_.size(); ++column) { + const PlannedSelectItem& item = select_items_[column]; + if (item.kind == SelectItemKind::GroupKey) { + batch.AppendValueFromString(column, FormatGroupKeyValue(group.key_values[item.index])); + } else { + batch.AppendValueFromString(column, group.states[item.index]->Finalize()); + } + } + } + + std::vector BuildOutputOrder() const { + std::vector order; + order.reserve(groups_.size()); + + for (size_t i = 0; i < groups_.size(); ++i) { + order.push_back(i); + } + + if (!sort_by_group_keys_) { + return order; + } + + std::vector group_key_columns; + + for (size_t i = 0; i < select_items_.size(); ++i) { + if (select_items_[i].kind == SelectItemKind::GroupKey) { + group_key_columns.push_back(i); + } + } + + if (group_key_columns.empty()) { + return order; + } + + std::ranges::sort(order, [&](const size_t lhs_index, const size_t rhs_index) { + const GroupState& lhs = groups_[lhs_index]; + const GroupState& rhs = groups_[rhs_index]; + for (const size_t column : group_key_columns) { + const PlannedSelectItem& item = select_items_[column]; + const std::strong_ordering comparison = + CompareGroupKeyValue(lhs.key_values[item.index], rhs.key_values[item.index]); + if (comparison != 0) { + return comparison < 0; + } + } + + return false; + }); + + return order; + } + + std::unique_ptr child_; + GroupKeyMaterializer group_key_materializer_; + std::vector bindings_; + std::vector select_items_; + PredicatePtr filter_; + PredicatePtr having_; + std::unordered_map group_indexes_; + std::vector groups_; + bool sort_by_group_keys_ = false; + bool returned_ = false; +}; + +class GroupAggTopKOperator final : public Operator { + public: + GroupAggTopKOperator(std::unique_ptr child, std::vector group_keys, + const std::vector& aggregates, std::vector select_items, + PredicatePtr filter, std::vector order_by, const size_t limit) + : child_(std::move(child)), + group_key_materializer_(std::move(group_keys)), + select_items_(std::move(select_items)), + filter_(std::move(filter)), + order_by_(std::move(order_by)), + limit_(limit) { + bindings_.reserve(aggregates.size()); + + for (const auto& aggregate : aggregates) { + bindings_.push_back(GroupAggBinding{ + .aggregate = aggregate, + }); + } + } + + std::optional Next() override { + if (returned_ || limit_ == 0) { + return std::nullopt; + } + + returned_ = true; + + while (auto batch = child_->Next()) { + std::vector selected_rows; + SelectRowsMatchingPredicate(filter_, *batch, selected_rows); + + for (const size_t row : selected_rows) { + TypedGroupKey key = group_key_materializer_.Materialize(*batch, row); + + size_t group_index = 0; + const auto it = group_indexes_.find(key); + if (it == group_indexes_.end()) { + group_index = groups_.size(); + group_indexes_.emplace(key, group_index); + + groups_.push_back(GroupState{ + .key_values = std::move(key.values), + .states = CreateStates(), + .aggregate_values = {}, + .aggregate_values_finalized = false, + }); + } else { + group_index = it->second; + } + + GroupState& group = groups_[group_index]; + for (size_t i = 0; i < bindings_.size(); ++i) { + ConsumeAggRow(bindings_[i].aggregate, *batch, row, *group.states[i]); + } + } + } + Schema schema; + schema.columns.reserve(select_items_.size()); for (const auto& item : select_items_) { + schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); + } + + const std::vector top_groups = BuildTopGroupIndexes(); + Batch result(schema, top_groups.size()); + + for (const size_t group_index : top_groups) { + AppendGroupToBatch(groups_[group_index], result); + } + + return result; + } + + private: + struct GroupState { + std::vector key_values; + std::vector> states; + mutable std::vector aggregate_values; + mutable bool aggregate_values_finalized = false; + }; + + struct GroupAggBinding { + PlannedAgg aggregate; + }; + + std::vector> CreateStates() const { + std::vector> states; + states.reserve(bindings_.size()); + + for (const auto& binding : bindings_) { + states.push_back(CreateAggState(binding.aggregate)); + } + + return states; + } + + static void FinalizeAggregateValues(const GroupState& group) { + if (group.aggregate_values_finalized) { + return; + } + + group.aggregate_values.reserve(group.states.size()); + for (const auto& state : group.states) { + group.aggregate_values.push_back(state->Finalize()); + } + group.aggregate_values_finalized = true; + } + + bool OrdersBefore(const size_t lhs_index, const size_t rhs_index) const { + const GroupState& lhs = groups_[lhs_index]; + const GroupState& rhs = groups_[rhs_index]; + + for (const PlannedOrderBy& order : order_by_) { + const PlannedSelectItem& item = select_items_[order.result_column_index]; + std::strong_ordering comparison = std::strong_ordering::equal; + + if (item.kind == SelectItemKind::GroupKey) { + comparison = CompareGroupKeyValue(lhs.key_values[item.index], rhs.key_values[item.index]); + } else { + FinalizeAggregateValues(lhs); + FinalizeAggregateValues(rhs); + comparison = + CompareValues(item.output_type, lhs.aggregate_values[item.index], rhs.aggregate_values[item.index]); + } + + if (comparison != 0) { + return order.descending ? comparison > 0 : comparison < 0; + } + } + + return lhs_index < rhs_index; + } + + void AppendGroupToBatch(const GroupState& group, const Batch& batch) const { + FinalizeAggregateValues(group); + + for (size_t column = 0; column < select_items_.size(); ++column) { + const PlannedSelectItem& item = select_items_[column]; if (item.kind == SelectItemKind::GroupKey) { - row.push_back(group.key_values[item.index]); + batch.AppendValueFromString(column, FormatGroupKeyValue(group.key_values[item.index])); } else { - row.push_back(group.states[item.index]->Finalize()); + batch.AppendValueFromString(column, group.aggregate_values[item.index]); + } + } + } + + std::vector BuildTopGroupIndexes() const { + std::vector top_groups; + top_groups.reserve(limit_); + + for (size_t i = 0; i < groups_.size(); ++i) { + if (top_groups.size() < limit_) { + top_groups.push_back(i); + std::ranges::push_heap(top_groups, + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + continue; + } + + if (OrdersBefore(i, top_groups.front())) { + std::ranges::pop_heap(top_groups, + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + top_groups.back() = i; + std::ranges::push_heap(top_groups, + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); } } - return row; + std::ranges::sort(top_groups, [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + + return top_groups; } std::unique_ptr child_; - GroupKeyEncoder group_key_encoder_; + GroupKeyMaterializer group_key_materializer_; std::vector bindings_; std::vector select_items_; - std::unordered_map group_indexes_; - std::vector groups_; + PredicatePtr filter_; + std::vector order_by_; + size_t limit_ = 0; + std::unordered_map group_indexes_; + mutable std::vector groups_; bool returned_ = false; }; @@ -349,8 +1319,7 @@ struct SortedRow { class RowOrdering { public: - RowOrdering(std::vector order_by, std::vector select_items) - : order_by_(std::move(order_by)), select_items_(std::move(select_items)) {} + explicit RowOrdering(std::vector order_by) : order_by_(std::move(order_by)) {} bool operator()(const SortedRow& lhs, const SortedRow& rhs) const { for (const auto& order : order_by_) { @@ -361,23 +1330,11 @@ class RowOrdering { } } - for (size_t i = 0; i < select_items_.size(); ++i) { - if (select_items_[i].kind != SelectItemKind::GroupKey) { - continue; - } - const std::strong_ordering comparison = - CompareValues(select_items_[i].output_type, lhs.values[i], rhs.values[i]); - if (comparison != 0) { - return comparison > 0; - } - } - return lhs.ordinal < rhs.ordinal; } private: std::vector order_by_; - std::vector select_items_; }; static std::vector MaterializeBatchRow(const Batch& batch, const size_t row) { @@ -403,11 +1360,70 @@ static Batch BuildBatchFromSortedRows(const Schema& schema, const std::vector child, std::vector items, Schema source_schema) + : child_(std::move(child)), items_(std::move(items)), source_schema_(std::move(source_schema)) { + for (const auto& item : items_) { + if (item.expression && item.expression->kind == ExprKind::Star) { + for (const size_t source_index : StarColumnIndexes(source_schema_)) { + schema_.columns.push_back(source_schema_.columns[source_index]); + } + } else { + schema_.columns.emplace_back(item.output_name, ColumnType::String); + } + } + } + + std::optional Next() override { + while (auto batch = child_->Next()) { + if (!star_column_indexes_initialized_) { + star_column_indexes_ = StarColumnIndexes(batch->GetSchema()); + star_column_indexes_initialized_ = true; + } + + Batch projected(schema_, batch->RowsCount()); + + size_t output_column = 0; + for (const auto& item : items_) { + if (item.expression && item.expression->kind == ExprKind::Star) { + for (const size_t source_column : star_column_indexes_) { + projected.AppendColumnRange(output_column++, batch->ColumnAt(source_column), 0, + batch->RowsCount()); + } + continue; + } + + const size_t expression_column = output_column++; + for (size_t row = 0; row < batch->RowsCount(); ++row) { + projected.AppendValueFromString(expression_column, EvalExpr(item.expression, *batch, row)); + } + } + + if (projected.RowsCount() > 0) { + return projected; + } + } + + return std::nullopt; + } + + const Schema& GetSchema() const { return schema_; } + + private: + std::unique_ptr child_; + std::vector items_; + Schema source_schema_; + Schema schema_; + std::vector star_column_indexes_; + bool star_column_indexes_initialized_ = false; +}; + class OrderByOperator final : public Operator { public: OrderByOperator(std::unique_ptr child, std::vector order_by, - std::vector select_items) - : child_(std::move(child)), ordering_(std::move(order_by), std::move(select_items)) {} + std::vector /*select_items*/) + : child_(std::move(child)), ordering_(std::move(order_by)) {} std::optional Next() override { if (returned_) { @@ -425,8 +1441,6 @@ class OrderByOperator final : public Operator { schema = batch->GetSchema(); } - rows.reserve(rows.size() + batch->RowsCount()); - for (size_t row = 0; row < batch->RowsCount(); ++row) { rows.push_back(SortedRow{ .values = MaterializeBatchRow(*batch, row), @@ -455,8 +1469,8 @@ class OrderByOperator final : public Operator { class TopKOperator final : public Operator { public: TopKOperator(std::unique_ptr child, std::vector order_by, - std::vector select_items, const size_t limit) - : child_(std::move(child)), ordering_(std::move(order_by), std::move(select_items)), limit_(limit) {} + std::vector /*select_items*/, const size_t limit) + : child_(std::move(child)), ordering_(std::move(order_by)), limit_(limit) {} std::optional Next() override { if (returned_ || limit_ == 0) { @@ -466,7 +1480,9 @@ class TopKOperator final : public Operator { returned_ = true; std::optional schema; - std::vector top_rows; + std::vector rows; + rows.reserve(limit_); + size_t ordinal = 0; while (auto batch = child_->Next()) { @@ -480,13 +1496,16 @@ class TopKOperator final : public Operator { .ordinal = ordinal++, }; - if (top_rows.size() < limit_) { - top_rows.push_back(std::move(candidate)); - std::ranges::push_heap(top_rows, ordering_); - } else if (ordering_(candidate, top_rows.front())) { - std::ranges::pop_heap(top_rows, ordering_); - top_rows.back() = std::move(candidate); - std::ranges::push_heap(top_rows, ordering_); + if (rows.size() < limit_) { + rows.push_back(std::move(candidate)); + std::ranges::push_heap(rows, ordering_); + continue; + } + + if (ordering_(candidate, rows.front())) { + std::ranges::pop_heap(rows, ordering_); + rows.back() = std::move(candidate); + std::ranges::push_heap(rows, ordering_); } } } @@ -495,9 +1514,9 @@ class TopKOperator final : public Operator { return std::nullopt; } - std::ranges::sort(top_rows, ordering_); + std::ranges::sort(rows, ordering_); - return BuildBatchFromSortedRows(*schema, top_rows); + return BuildBatchFromSortedRows(*schema, rows); } private: @@ -524,12 +1543,7 @@ class LimitOperator final : public Operator { } Batch limited(batch->GetSchema(), remaining_); - - for (size_t row = 0; row < remaining_; ++row) { - for (size_t column_index = 0; column_index < batch->ColumnsCount(); ++column_index) { - limited.AppendValueFromString(column_index, batch->ColumnAt(column_index).ValueAsString(row)); - } - } + limited.AppendRowsRangeFromBatch(*batch, 0, remaining_); remaining_ = 0; @@ -546,25 +1560,108 @@ class LimitOperator final : public Operator { size_t remaining_; }; +class OffsetOperator final : public Operator { + public: + OffsetOperator(std::unique_ptr child, const size_t offset) + : child_(std::move(child)), remaining_(offset) {} + + std::optional Next() override { + while (const auto batch = child_->Next()) { + if (remaining_ >= batch->RowsCount()) { + remaining_ -= batch->RowsCount(); + continue; + } + + Batch shifted(batch->GetSchema(), batch->RowsCount() - remaining_); + shifted.AppendRowsRangeFromBatch(*batch, remaining_, batch->RowsCount() - remaining_); + remaining_ = 0; + + return shifted; + } + + return std::nullopt; + } + + private: + std::unique_ptr child_; + size_t remaining_; +}; + std::unique_ptr BuildPlan(const PlannedQuery& planned) { - std::unique_ptr root = std::make_unique(planned.table_path, planned.projection_indexes); + if (planned.metadata_count_only) { + return std::make_unique(planned.table_path, planned.aggregates.front().output_name); + } + + if (planned.metadata_extrema_only) { + return std::make_unique(planned.table_path, planned.aggregates); + } + + std::unique_ptr root = + std::make_unique(planned.table_path, planned.projection_indexes, planned.filter); - if (planned.filter.has_value()) { - root = std::make_unique(std::move(root), *planned.filter); + if (planned.filter && planned.plain_select) { + root = std::make_unique(std::move(root), planned.filter); } + if (planned.plain_select) { + bool limit_applied_by_top_k = false; + if (!planned.order_by.empty()) { + if (planned.limit.has_value() && planned.offset == 0) { + root = std::make_unique(std::move(root), planned.order_by, planned.select_items, + *planned.limit); + limit_applied_by_top_k = true; + } else { + root = std::make_unique(std::move(root), planned.order_by, planned.select_items); + } + } + if (planned.offset > 0) { + root = std::make_unique(std::move(root), planned.offset); + } + if (planned.limit.has_value() && !limit_applied_by_top_k) { + root = std::make_unique(std::move(root), *planned.limit); + } + + auto projection = + std::make_unique(std::move(root), planned.plain_select_items, planned.table_schema); + const Schema output_schema = projection->GetSchema(); + root = std::move(projection); + root = std::make_unique(std::move(root), output_schema); + + return root; + } + + bool limit_applied_by_top_k = false; + if (!planned.group_keys.empty()) { - root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, - planned.select_items); + if (!planned.order_by.empty() && planned.limit.has_value() && planned.offset == 0 && !planned.having) { + root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, + planned.select_items, planned.filter, planned.order_by, + *planned.limit); + limit_applied_by_top_k = true; + } else { + root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, + planned.select_items, planned.filter, planned.having, + planned.order_by.empty()); + } } else { - root = std::make_unique(std::move(root), planned.aggregates); + root = std::make_unique(std::move(root), planned.aggregates, planned.filter); + } + + if (!planned.order_by.empty()) { + if (planned.limit.has_value() && planned.offset == 0) { + root = + std::make_unique(std::move(root), planned.order_by, planned.select_items, *planned.limit); + limit_applied_by_top_k = true; + } else { + root = std::make_unique(std::move(root), planned.order_by, planned.select_items); + } + } + + if (planned.offset > 0) { + root = std::make_unique(std::move(root), planned.offset); } - if (!planned.order_by.empty() && planned.limit.has_value()) { - root = std::make_unique(std::move(root), planned.order_by, planned.select_items, *planned.limit); - } else if (!planned.order_by.empty()) { - root = std::make_unique(std::move(root), planned.order_by, planned.select_items); - } else if (planned.limit.has_value()) { + if (planned.limit.has_value() && !limit_applied_by_top_k) { root = std::make_unique(std::move(root), *planned.limit); } diff --git a/src/executor/query_parser.cpp b/src/executor/query_parser.cpp index 11233b6..04bcfd5 100644 --- a/src/executor/query_parser.cpp +++ b/src/executor/query_parser.cpp @@ -1,6 +1,5 @@ #include "executor/query_parser.h" -#include #include #include #include @@ -73,6 +72,11 @@ class TokenCursor { size_t pos_ = 0; }; +static bool IsAggregateName(const std::string_view name) { + const std::string upper = ToUpperAscii(name); + return upper == "COUNT" || upper == "SUM" || upper == "AVG" || upper == "MIN" || upper == "MAX"; +} + static ComparisonKind ComparisonKindFromToken(const Tokens type) { switch (type) { case Tokens::Equal: @@ -88,7 +92,7 @@ static ComparisonKind ComparisonKindFromToken(const Tokens type) { case Tokens::GreaterOrEqual: return ComparisonKind::GreaterOrEqual; default: - throw Error::InvalidArgument("executor", "expected comparison operator in WHERE"); + throw Error::InvalidArgument("executor", "expected comparison operator"); } } @@ -99,196 +103,397 @@ static std::string FormatColumnRef(const ColumnRef& column) { return column.qualifier + "." + column.name; } -static ColumnRef ParseColumnRef(TokenCursor& cursor, const std::string_view message) { - ColumnRef column{ - .qualifier = {}, - .name = std::string(cursor.Consume(Tokens::NameToken, message).GetText()), - }; +static ExprPtr MakeExpr(const ExprKind kind) { + auto expr = std::make_shared(); + expr->kind = kind; + return expr; +} - if (cursor.TryConsume(Tokens::Dot)) { - column.qualifier = std::move(column.name); - column.name = std::string(cursor.Consume(Tokens::NameToken, "expected column name after '.'").GetText()); +static std::string FormatExpression(const ExprSpec& expr); + +static std::string FormatFunctionCall(const ExprSpec& expr) { + std::string output = ToUpperAscii(expr.function_name); + + output.push_back('('); + for (size_t i = 0; i < expr.arguments.size(); ++i) { + if (i > 0) { + output += ", "; + } + output += FormatExpression(*expr.arguments[i]); } + output.push_back(')'); - return column; + return output; } -static QueryValue QueryColumnValue(ColumnRef column) { - return QueryValue{ - .kind = QueryValueKind::Column, - .column = std::move(column), - .literal = {}, - }; +static std::string FormatExpression(const ExprSpec& expr) { + switch (expr.kind) { + case ExprKind::Column: + return FormatColumnRef(expr.column); + case ExprKind::Literal: + return expr.literal.text; + case ExprKind::Star: + return "*"; + case ExprKind::Binary: + return FormatExpression(*expr.left) + (expr.binary_op == BinaryOp::Add ? " + " : " - ") + + FormatExpression(*expr.right); + case ExprKind::Function: + return FormatFunctionCall(expr); + case ExprKind::Case: + return "CASE"; + } + + return {}; } -static QueryValue QueryLiteralValue(QueryLiteral literal) { - return QueryValue{ - .kind = QueryValueKind::Literal, - .column = {}, - .literal = std::move(literal), - }; +static ExprPtr ParseExpression(TokenCursor& cursor); +static PredicatePtr ParsePredicate(TokenCursor& cursor); + +static ExprPtr ParseFunction(TokenCursor& cursor, const std::string& function_name) { + auto expr = MakeExpr(ExprKind::Function); + expr->function_name = ToUpperAscii(function_name); + + cursor.Consume(Tokens::OpenBracket, "expected '(' after function name"); + + if (ToUpperAscii(function_name) == "EXTRACT") { + auto part = MakeExpr(ExprKind::Literal); + part->literal = QueryLiteral{std::string(cursor.Consume(Tokens::NameToken, "expected EXTRACT part").GetText()), + LiteralKind::Identifier}; + + cursor.Consume(Tokens::From, "expected FROM in EXTRACT"); + expr->arguments.push_back(std::move(part)); + expr->arguments.push_back(ParseExpression(cursor)); + cursor.Consume(Tokens::CloseBracket, "expected ')' after EXTRACT"); + expr->output_name = FormatFunctionCall(*expr); + + return expr; + } + + if (!cursor.Match(Tokens::CloseBracket)) { + expr->arguments.push_back(ParseExpression(cursor)); + while (cursor.TryConsume(Tokens::Comma)) { + expr->arguments.push_back(ParseExpression(cursor)); + } + } + + cursor.Consume(Tokens::CloseBracket, "expected ')' after function arguments"); + expr->output_name = FormatFunctionCall(*expr); + + return expr; } -static QueryValue ParseFilterValue(TokenCursor& cursor, const std::string_view message, - const bool bare_identifier_is_column) { +static ExprPtr ParseCaseExpression(TokenCursor& cursor) { + cursor.Consume(Tokens::Case, "expected CASE"); + cursor.Consume(Tokens::When, "expected WHEN after CASE"); + + auto expr = MakeExpr(ExprKind::Case); + expr->case_spec.condition = ParsePredicate(cursor); + cursor.Consume(Tokens::Then, "expected THEN in CASE"); + expr->case_spec.then_expr = ParseExpression(cursor); + cursor.Consume(Tokens::Else, "expected ELSE in CASE"); + expr->case_spec.else_expr = ParseExpression(cursor); + cursor.Consume(Tokens::End, "expected END in CASE"); + expr->output_name = "CASE"; + + return expr; +} + +static ExprPtr ParsePrimary(TokenCursor& cursor) { if (cursor.TryConsume(Tokens::Minus)) { - const Token& literal = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after '-'"); - return QueryLiteralValue(QueryLiteral{ - .text = "-" + std::string(literal.GetText()), - .kind = LiteralKind::Numeric, - }); - } - - const Token& token = cursor.Peek(); - switch (token.GetType()) { - case Tokens::NumericLiteral: - cursor.Consume(Tokens::NumericLiteral, message); - return QueryLiteralValue(QueryLiteral{ - .text = std::string(token.GetText()), - .kind = LiteralKind::Numeric, - }); - case Tokens::StringLiteral: - cursor.Consume(Tokens::StringLiteral, message); - return QueryLiteralValue(QueryLiteral{ - .text = std::string(token.GetText()), - .kind = LiteralKind::String, - }); - case Tokens::NameToken: - if (bare_identifier_is_column || cursor.Match(Tokens::Dot, 1)) { - return QueryColumnValue(ParseColumnRef(cursor, message)); - } - cursor.Consume(Tokens::NameToken, message); - return QueryLiteralValue(QueryLiteral{ - .text = std::string(token.GetText()), - .kind = LiteralKind::Identifier, - }); - default: - throw Error::InvalidArgument("executor", std::string(message)); + auto zero = MakeExpr(ExprKind::Literal); + zero->literal = QueryLiteral{"0", LiteralKind::Numeric}; + auto expr = MakeExpr(ExprKind::Binary); + expr->binary_op = BinaryOp::Subtract; + expr->left = std::move(zero); + expr->right = ParsePrimary(cursor); + expr->output_name = FormatExpression(*expr); + return expr; } + + if (cursor.TryConsume(Tokens::Asterisk)) { + auto expr = MakeExpr(ExprKind::Star); + expr->output_name = "*"; + return expr; + } + + if (cursor.Match(Tokens::Case)) { + return ParseCaseExpression(cursor); + } + + if (cursor.TryConsume(Tokens::OpenBracket)) { + if (cursor.Match(Tokens::Select)) { + throw Error::Unsupported("executor", "subqueries are not supported"); + } + auto expr = ParseExpression(cursor); + cursor.Consume(Tokens::CloseBracket, "expected ')' after expression"); + return expr; + } + + if (cursor.Match(Tokens::NumericLiteral)) { + auto expr = MakeExpr(ExprKind::Literal); + expr->literal = + QueryLiteral{std::string(cursor.Consume(Tokens::NumericLiteral, "expected numeric literal").GetText()), + LiteralKind::Numeric}; + expr->output_name = expr->literal.text; + return expr; + } + + if (cursor.Match(Tokens::StringLiteral)) { + auto expr = MakeExpr(ExprKind::Literal); + expr->literal = + QueryLiteral{std::string(cursor.Consume(Tokens::StringLiteral, "expected string literal").GetText()), + LiteralKind::String}; + expr->output_name = expr->literal.text; + return expr; + } + + if (cursor.Match(Tokens::NameToken) || cursor.Match(Tokens::Length) || cursor.Match(Tokens::Count) || + cursor.Match(Tokens::Sum) || cursor.Match(Tokens::Avg) || cursor.Match(Tokens::Min) || + cursor.Match(Tokens::Max)) { + const Token& token = cursor.ConsumeFunctionToken("expected expression"); + const auto name = std::string(token.GetText()); + + if (cursor.Match(Tokens::OpenBracket)) { + return ParseFunction(cursor, name); + } + + auto expr = MakeExpr(ExprKind::Column); + expr->column = ColumnRef{.qualifier = {}, .name = name}; + + if (cursor.TryConsume(Tokens::Dot)) { + expr->column.qualifier = std::move(expr->column.name); + expr->column.name = + std::string(cursor.Consume(Tokens::NameToken, "expected column name after '.'").GetText()); + } + + expr->output_name = expr->column.name; + + return expr; + } + + throw Error::InvalidArgument("executor", "expected expression"); } -static std::string FormatAggregateCall(const std::string_view function_name, const std::string_view column_name, - const bool distinct, const AggArgumentKind argument_kind) { - std::string output(function_name); - output += '('; +static ExprPtr ParseExpression(TokenCursor& cursor) { + auto expr = ParsePrimary(cursor); - if (argument_kind == AggArgumentKind::Star) { - output += '*'; - } else { - if (distinct) { - output += "DISTINCT "; + while (cursor.Match(Tokens::Plus) || cursor.Match(Tokens::Minus)) { + const bool add = cursor.TryConsume(Tokens::Plus); + if (!add) { + cursor.Consume(Tokens::Minus, "expected '-'"); } - output += column_name; + + auto binary = MakeExpr(ExprKind::Binary); + binary->binary_op = add ? BinaryOp::Add : BinaryOp::Subtract; + binary->left = std::move(expr); + binary->right = ParsePrimary(cursor); + binary->output_name = FormatExpression(*binary); + expr = std::move(binary); } - output += ')'; + return expr; +} - return output; +static PredicatePtr MakePredicate(const PredicateKind kind) { + auto predicate = std::make_shared(); + predicate->kind = kind; + return predicate; } -static std::optional ParseOptionalLimit(TokenCursor& cursor) { - if (!cursor.TryConsume(Tokens::Limit)) { +static PredicatePtr ParsePredicateAtom(TokenCursor& cursor) { + if (cursor.TryConsume(Tokens::OpenBracket)) { + auto nested = ParsePredicate(cursor); + cursor.Consume(Tokens::CloseBracket, "expected ')' after predicate"); + return nested; + } + + auto left = ParseExpression(cursor); + + if (cursor.TryConsume(Tokens::Not)) { + if (cursor.TryConsume(Tokens::Like)) { + auto predicate = MakePredicate(PredicateKind::NotLike); + predicate->left = std::move(left); + predicate->right = ParseExpression(cursor); + return predicate; + } + throw Error::InvalidArgument("executor", "expected LIKE after NOT"); + } + + if (cursor.TryConsume(Tokens::Like)) { + auto predicate = MakePredicate(PredicateKind::Like); + predicate->left = std::move(left); + predicate->right = ParseExpression(cursor); + return predicate; + } + + if (cursor.TryConsume(Tokens::In)) { + auto predicate = MakePredicate(PredicateKind::In); + predicate->left = std::move(left); + cursor.Consume(Tokens::OpenBracket, "expected '(' after IN"); + predicate->values.push_back(ParseExpression(cursor)); + + while (cursor.TryConsume(Tokens::Comma)) { + predicate->values.push_back(ParseExpression(cursor)); + } + + cursor.Consume(Tokens::CloseBracket, "expected ')' after IN list"); + + return predicate; + } + + const Tokens comparison_token = cursor.Peek().GetType(); + auto predicate = MakePredicate(PredicateKind::Comparison); + predicate->left = std::move(left); + predicate->comparison = ComparisonKindFromToken(comparison_token); + cursor.Consume(comparison_token, "expected comparison operator"); + predicate->right = ParseExpression(cursor); + + return predicate; +} + +static PredicatePtr ParsePredicate(TokenCursor& cursor) { + auto predicate = ParsePredicateAtom(cursor); + + while (cursor.TryConsume(Tokens::And)) { + auto and_predicate = MakePredicate(PredicateKind::And); + and_predicate->lhs = std::move(predicate); + and_predicate->rhs = ParsePredicateAtom(cursor); + predicate = std::move(and_predicate); + } + + return predicate; +} + +static std::optional ParseOptionalSize(TokenCursor& cursor, const Tokens token, const std::string_view name) { + if (!cursor.TryConsume(token)) { return std::nullopt; } - const Token& limit_token = cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after LIMIT"); - size_t limit = 0; + const Token& value_token = + cursor.Consume(Tokens::NumericLiteral, "expected numeric literal after " + std::string(name)); + size_t value = 0; + const std::string_view text = value_token.GetText(); - const std::string_view text = limit_token.GetText(); const char* begin = text.data(); const char* end = begin + text.size(); - - if (const auto [ptr, ec] = std::from_chars(begin, end, limit); ec != std::errc() || ptr != end) { - throw Error::InvalidArgument("executor", "invalid LIMIT value"); + if (const auto [ptr, ec] = std::from_chars(begin, end, value); ec != std::errc() || ptr != end) { + throw Error::InvalidArgument("executor", "invalid " + std::string(name) + " value"); } - return limit; + return value; } -static AggSpec ParseAggregate(TokenCursor& cursor) { - const Token& function_token = cursor.ConsumeFunctionToken("expected aggregate function"); - +static AggSpec ParseAggregate(TokenCursor& cursor, const std::string& function_name) { AggSpec spec; - spec.function_name = ToUpperAscii(function_token.GetText()); - + spec.function_name = ToUpperAscii(function_name); cursor.Consume(Tokens::OpenBracket, "expected '(' after aggregate function"); if (cursor.TryConsume(Tokens::Asterisk)) { spec.argument_kind = AggArgumentKind::Star; } else { spec.distinct = cursor.TryConsume(Tokens::Distinct); - spec.column = ParseColumnRef(cursor, "expected column name in aggregate"); + spec.argument = ParseExpression(cursor); + if (spec.argument->kind == ExprKind::Column) { + spec.column = spec.argument->column; + } } cursor.Consume(Tokens::CloseBracket, "expected ')' after aggregate arguments"); - spec.output_name = - FormatAggregateCall(spec.function_name, FormatColumnRef(spec.column), spec.distinct, spec.argument_kind); + + if (spec.argument_kind == AggArgumentKind::Star) { + spec.output_name = spec.function_name + "(*)"; + } else { + spec.output_name = spec.function_name + "("; + if (spec.distinct) { + spec.output_name += "DISTINCT "; + } + spec.output_name += FormatExpression(*spec.argument); + spec.output_name += ")"; + } return spec; } -static SelectItemSpec ParseSelectExpression(TokenCursor& cursor) { - if (cursor.Match(Tokens::NameToken) && !cursor.Match(Tokens::OpenBracket, 1)) { - const ColumnRef column = ParseColumnRef(cursor, "expected column name in SELECT"); - return SelectItemSpec{ - .kind = SelectItemKind::GroupKey, - .column = column, - .aggregate = {}, - .output_name = column.name, +static SelectItemSpec ParseSelectItem(TokenCursor& cursor) { + if ((cursor.Match(Tokens::NameToken) || cursor.Match(Tokens::Count) || cursor.Match(Tokens::Sum) || + cursor.Match(Tokens::Avg) || cursor.Match(Tokens::Min) || cursor.Match(Tokens::Max)) && + cursor.Match(Tokens::OpenBracket, 1) && IsAggregateName(cursor.Peek().GetText())) { + const auto function_name = std::string(cursor.ConsumeFunctionToken("expected aggregate").GetText()); + + SelectItemSpec item{ + .kind = SelectItemKind::Aggregate, + .column = {}, + .expression = {}, + .aggregate = ParseAggregate(cursor, function_name), + .output_name = {}, }; - } - - return SelectItemSpec{ - .kind = SelectItemKind::Aggregate, - .column = {}, - .aggregate = ParseAggregate(cursor), - .output_name = {}, - }; -} + item.output_name = item.aggregate.output_name; -static std::string ParseSelectItemAlias(TokenCursor& cursor) { - if (cursor.TryConsume(Tokens::As)) { - return std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); - } + if (cursor.TryConsume(Tokens::As)) { + item.output_name = std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); + } else if (cursor.Match(Tokens::NameToken)) { + item.output_name = std::string(cursor.Consume(Tokens::NameToken, "expected alias").GetText()); + } - if (cursor.Match(Tokens::NameToken)) { - return std::string(cursor.Consume(Tokens::NameToken, "expected alias").GetText()); + return item; } - return {}; -} + const auto expr = ParseExpression(cursor); -static SelectItemSpec ParseSelectItem(TokenCursor& cursor) { - SelectItemSpec item = ParseSelectExpression(cursor); + SelectItemSpec item{ + .kind = SelectItemKind::GroupKey, + .column = expr->kind == ExprKind::Column ? expr->column : ColumnRef{}, + .expression = expr, + .aggregate = {}, + .output_name = expr->output_name.empty() ? FormatExpression(*expr) : expr->output_name, + }; - if (const std::string alias = ParseSelectItemAlias(cursor); !alias.empty()) { - item.output_name = alias; - } else if (item.kind == SelectItemKind::Aggregate) { - item.output_name = item.aggregate.output_name; + if (cursor.TryConsume(Tokens::As)) { + item.output_name = std::string(cursor.Consume(Tokens::NameToken, "expected alias after AS").GetText()); + } else if (cursor.Match(Tokens::NameToken)) { + item.output_name = std::string(cursor.Consume(Tokens::NameToken, "expected alias").GetText()); } return item; } static OrderBySpec ParseOrderByItem(TokenCursor& cursor) { - SelectItemSpec item = ParseSelectExpression(cursor); - item.output_name = item.kind == SelectItemKind::Aggregate ? item.aggregate.output_name : item.column.name; + OrderBySpec order; + + if ((cursor.Match(Tokens::NameToken) || cursor.Match(Tokens::Count) || cursor.Match(Tokens::Sum) || + cursor.Match(Tokens::Avg) || cursor.Match(Tokens::Min) || cursor.Match(Tokens::Max)) && + cursor.Match(Tokens::OpenBracket, 1) && IsAggregateName(cursor.Peek().GetText())) { + const auto function_name = std::string(cursor.ConsumeFunctionToken("expected aggregate").GetText()); + order.item = SelectItemSpec{ + .kind = SelectItemKind::Aggregate, + .column = {}, + .expression = {}, + .aggregate = ParseAggregate(cursor, function_name), + .output_name = {}, + }; + order.item.output_name = order.item.aggregate.output_name; + } else { + const auto expr = ParseExpression(cursor); + order.item = SelectItemSpec{ + .kind = SelectItemKind::GroupKey, + .column = expr->kind == ExprKind::Column ? expr->column : ColumnRef{}, + .expression = expr, + .aggregate = {}, + .output_name = expr->output_name.empty() ? FormatExpression(*expr) : expr->output_name, + }; + } - bool descending = false; if (cursor.Match(Tokens::NameToken)) { const std::string direction = ToUpperAscii(cursor.Peek().GetText()); if (direction == "DESC") { cursor.Consume(Tokens::NameToken, "expected DESC"); - descending = true; + order.descending = true; } else if (direction == "ASC") { cursor.Consume(Tokens::NameToken, "expected ASC"); } } - return OrderBySpec{ - .item = std::move(item), - .descending = descending, - }; + return order; } Query ParseQuery(const std::string_view query) { @@ -317,37 +522,34 @@ Query ParseQuery(const std::string_view query) { } if (cursor.TryConsume(Tokens::Where)) { - FilterSpec filter; - filter.left = ParseFilterValue(cursor, "expected left operand after WHERE", true); - - const Tokens comparison_token = cursor.Peek().GetType(); - filter.comparison = ComparisonKindFromToken(comparison_token); - - cursor.Consume(comparison_token, "expected comparison operator in WHERE"); - filter.right = ParseFilterValue(cursor, "expected right operand after comparison operator", false); - - query_ast.filter = std::move(filter); + query_ast.filter = ParsePredicate(cursor); } if (cursor.TryConsume(Tokens::Group)) { cursor.Consume(Tokens::By, "expected BY after GROUP"); - query_ast.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); - + query_ast.group_by.push_back(ParseExpression(cursor)); while (cursor.TryConsume(Tokens::Comma)) { - query_ast.group_by.push_back(ParseColumnRef(cursor, "expected column name in GROUP BY")); + query_ast.group_by.push_back(ParseExpression(cursor)); } } + if (cursor.TryConsume(Tokens::Having)) { + query_ast.having = ParsePredicate(cursor); + } + if (cursor.TryConsume(Tokens::Order)) { cursor.Consume(Tokens::By, "expected BY after ORDER"); query_ast.order_by.push_back(ParseOrderByItem(cursor)); - while (cursor.TryConsume(Tokens::Comma)) { query_ast.order_by.push_back(ParseOrderByItem(cursor)); } } - query_ast.limit = ParseOptionalLimit(cursor); + query_ast.limit = ParseOptionalSize(cursor, Tokens::Limit, "LIMIT"); + if (const auto offset = ParseOptionalSize(cursor, Tokens::Offset, "OFFSET"); offset.has_value()) { + query_ast.offset = *offset; + } + cursor.TryConsume(Tokens::Semicolon); cursor.Consume(Tokens::EndOfInput, "unexpected trailing tokens"); diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index e5f7724..a96ea07 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -4,243 +4,902 @@ #include "common/ascii.h" #include "common/error.h" -#include "executor/operator_utils.h" -#include "executor/planner_utils.h" +#include "common/operator_utils.h" +#include "common/parsing.h" +#include "common/planner_utils.h" #include "io/columnar_batch.h" -static std::optional TryFindColumnIndex(const Schema& schema, const std::string_view column_name) { - for (size_t i = 0; i < schema.columns.size(); ++i) { - if (SameColumnName(schema.columns[i].name, column_name)) { +constexpr size_t SqlOrdinalBase = 1; + +static bool SameExpr(const ExprPtr& lhs, const ExprPtr& rhs) { + if (!lhs || !rhs || lhs->kind != rhs->kind) { + return false; + } + + switch (lhs->kind) { + case ExprKind::Column: + return SameColumnRef(lhs->column, rhs->column); + case ExprKind::Literal: + return lhs->literal.text == rhs->literal.text && lhs->literal.kind == rhs->literal.kind; + case ExprKind::Star: + return true; + case ExprKind::Binary: + return lhs->binary_op == rhs->binary_op && SameExpr(lhs->left, rhs->left) && + SameExpr(lhs->right, rhs->right); + case ExprKind::Function: + if (!SameColumnName(lhs->function_name, rhs->function_name) || + lhs->arguments.size() != rhs->arguments.size()) { + return false; + } + for (size_t i = 0; i < lhs->arguments.size(); ++i) { + if (!SameExpr(lhs->arguments[i], rhs->arguments[i])) { + return false; + } + } + return true; + case ExprKind::Case: + return lhs->output_name == rhs->output_name; + } + + return false; +} + +static ColumnType ColumnTypeOf(const Query& query, const Schema& schema, const ExprPtr& expr); + +static ColumnType ColumnTypeOfColumn(const Query& query, const Schema& schema, const ColumnRef& column) { + ValidateColumnQualifier(query, column); + return schema.columns[FindColumnIndex(schema, column.name)].type; +} + +static ColumnType ColumnTypeOf(const Query& query, const Schema& schema, const ExprPtr& expr) { + if (!expr) { + return ColumnType::String; + } + + switch (expr->kind) { + case ExprKind::Column: + return ColumnTypeOfColumn(query, schema, expr->column); + case ExprKind::Literal: + return expr->literal.kind == LiteralKind::String ? ColumnType::String : ColumnType::Int64; + case ExprKind::Binary: + return ColumnType::Int64; + case ExprKind::Function: { + const std::string name = ToUpperAscii(expr->function_name); + if (name == "STRLEN" || name == "LENGTH" || name == "EXTRACT") { + return ColumnType::Int64; + } + if (name == "DATE_TRUNC") { + return ColumnType::Timestamp; + } + if (name == "REGEXP_REPLACE") { + return ColumnType::String; + } + if (name == "COUNT") { + return ColumnType::Int64; + } + if (name == "SUM" || name == "AVG") { + return ColumnType::Int128; + } + if ((name == "MIN" || name == "MAX") && !expr->arguments.empty()) { + return ColumnTypeOf(query, schema, expr->arguments.front()); + } + return ColumnType::String; + } + case ExprKind::Case: + return ColumnTypeOf(query, schema, expr->case_spec.then_expr); + case ExprKind::Star: + return ColumnType::String; + } + + return ColumnType::String; +} + +static bool HasAggregate(const std::vector& items) { + return std::ranges::any_of(items, + [](const SelectItemSpec& item) { return item.kind == SelectItemKind::Aggregate; }); +} + +static std::optional FindSelectItem(const Query& query_ast, const SelectItemSpec& item) { + for (size_t i = 0; i < query_ast.select_items.size(); ++i) { + const SelectItemSpec& candidate = query_ast.select_items[i]; + if (SameOutputName(candidate, item)) { return i; } + if (candidate.kind == item.kind) { + if (candidate.kind == SelectItemKind::Aggregate && + SameColumnName(candidate.aggregate.output_name, item.aggregate.output_name)) { + return i; + } + if (candidate.kind == SelectItemKind::GroupKey && SameExpr(candidate.expression, item.expression)) { + return i; + } + } } + return std::nullopt; } -PlannedQuery PlanQuery(const Query& query_ast, const std::unordered_map& tables) { - PlannedQuery planned; - - const auto table_it = tables.find(ToLowerAscii(query_ast.table_name)); - if (table_it == tables.end()) { - throw Error::NotFound("executor", "unknown table", query_ast.table_name); +static std::optional FindGroupKey(const std::vector& group_by, const ExprPtr& expr) { + for (size_t i = 0; i < group_by.size(); ++i) { + if (SameExpr(group_by[i], expr)) { + return i; + } } - planned.table_path = table_it->second; + return std::nullopt; +} - const ColumnarBatchReader reader(planned.table_path); - const Schema& schema = reader.GetSchema(); +static ExprPtr ResolveGroupByExpression(const Query& query_ast, const ExprPtr& expr) { + if (expr->kind == ExprKind::Literal && expr->literal.kind == LiteralKind::Numeric) { + try { + const size_t ordinal = std::stoull(expr->literal.text); + if (ordinal >= SqlOrdinalBase && ordinal <= query_ast.select_items.size()) { + return query_ast.select_items[ordinal - SqlOrdinalBase].expression; + } + } catch (...) { + } + } - const auto ensure_projected_column = [&](const size_t source_index) -> size_t { - const auto it = std::ranges::find(planned.projection_indexes, source_index); - if (it != planned.projection_indexes.end()) { - return static_cast(std::distance(planned.projection_indexes.begin(), it)); + if (expr->kind == ExprKind::Column && expr->column.qualifier.empty()) { + for (const auto& item : query_ast.select_items) { + if (SameColumnName(item.output_name, expr->column.name) && item.expression) { + return item.expression; + } } + } - planned.projection_indexes.push_back(source_index); + return expr; +} - return planned.projection_indexes.size() - 1; - }; +static void AddProjectionIndex(std::vector& indexes, const size_t index) { + if (!std::ranges::contains(indexes, index)) { + indexes.push_back(index); + } +} + +static void CollectColumnsFromExpr(const Query& query_ast, const Schema& schema, const ExprPtr& expr, + std::vector& indexes); - struct PlannedFilterValue { - PlannedValue value; - std::optional literal; - }; +static void CollectColumnsFromPredicate(const Query& query_ast, const Schema& schema, const PredicatePtr& predicate, + std::vector& indexes) { + if (!predicate) { + return; + } - const auto plan_filter_value = [&](const QueryValue& value) -> PlannedFilterValue { - if (value.kind == QueryValueKind::Column) { - ValidateColumnQualifier(query_ast, value.column); + switch (predicate->kind) { + case PredicateKind::And: + CollectColumnsFromPredicate(query_ast, schema, predicate->lhs, indexes); + CollectColumnsFromPredicate(query_ast, schema, predicate->rhs, indexes); + return; + case PredicateKind::Comparison: + case PredicateKind::Like: + case PredicateKind::NotLike: + CollectColumnsFromExpr(query_ast, schema, predicate->left, indexes); + CollectColumnsFromExpr(query_ast, schema, predicate->right, indexes); + return; + case PredicateKind::In: + CollectColumnsFromExpr(query_ast, schema, predicate->left, indexes); + for (const ExprPtr& value : predicate->values) { + CollectColumnsFromExpr(query_ast, schema, value, indexes); + } + } +} - const size_t source_index = FindColumnIndex(schema, value.column.name); +static void CollectColumnsFromExpr(const Query& query_ast, const Schema& schema, const ExprPtr& expr, + std::vector& indexes) { + if (!expr) { + return; + } - return PlannedFilterValue{ - .value = - PlannedValue{ - .kind = PlannedValueKind::Column, - .column_type = schema.columns[source_index].type, - .column_index = ensure_projected_column(source_index), - .literal_text = {}, - }, - .literal = std::nullopt, - }; + switch (expr->kind) { + case ExprKind::Column: + ValidateColumnQualifier(query_ast, expr->column); + AddProjectionIndex(indexes, FindColumnIndex(schema, expr->column.name)); + return; + case ExprKind::Literal: + return; + case ExprKind::Star: + for (const size_t index : StarColumnIndexes(schema)) { + AddProjectionIndex(indexes, index); + } + return; + case ExprKind::Binary: + CollectColumnsFromExpr(query_ast, schema, expr->left, indexes); + CollectColumnsFromExpr(query_ast, schema, expr->right, indexes); + return; + case ExprKind::Function: + for (const ExprPtr& argument : expr->arguments) { + CollectColumnsFromExpr(query_ast, schema, argument, indexes); + } + return; + case ExprKind::Case: + CollectColumnsFromPredicate(query_ast, schema, expr->case_spec.condition, indexes); + CollectColumnsFromExpr(query_ast, schema, expr->case_spec.then_expr, indexes); + CollectColumnsFromExpr(query_ast, schema, expr->case_spec.else_expr, indexes); + } +} + +static std::optional ProjectedIndexOf(const std::vector& projection_indexes, + const size_t source_index) { + for (size_t i = 0; i < projection_indexes.size(); ++i) { + if (projection_indexes[i] == source_index) { + return i; } + } + return std::nullopt; +} + +static void BindExprColumnIndexes(const Query& query_ast, const Schema& schema, + const std::vector& projection_indexes, const ExprPtr& expr); +static void BindTypedLiteralComparison(const Query& query_ast, const Schema& schema, + const std::vector& projection_indexes, const PredicatePtr& predicate); +static void BindLiteralInSet(const Query& query_ast, const Schema& schema, + const std::vector& projection_indexes, const PredicatePtr& predicate); +static void BindLiteralLikePattern(const Query& query_ast, const Schema& schema, + const std::vector& projection_indexes, const PredicatePtr& predicate); +static void BindRegexReplace(const ExprPtr& expr); + +static void BindPredicateColumnIndexes(const Query& query_ast, const Schema& schema, + const std::vector& projection_indexes, const PredicatePtr& predicate) { + if (!predicate) { + return; + } + + switch (predicate->kind) { + case PredicateKind::And: + BindPredicateColumnIndexes(query_ast, schema, projection_indexes, predicate->lhs); + BindPredicateColumnIndexes(query_ast, schema, projection_indexes, predicate->rhs); + return; + case PredicateKind::Comparison: + BindTypedLiteralComparison(query_ast, schema, projection_indexes, predicate); + BindExprColumnIndexes(query_ast, schema, projection_indexes, predicate->left); + BindExprColumnIndexes(query_ast, schema, projection_indexes, predicate->right); + return; + case PredicateKind::Like: + case PredicateKind::NotLike: + BindLiteralLikePattern(query_ast, schema, projection_indexes, predicate); + BindExprColumnIndexes(query_ast, schema, projection_indexes, predicate->left); + BindExprColumnIndexes(query_ast, schema, projection_indexes, predicate->right); + return; + case PredicateKind::In: + BindLiteralInSet(query_ast, schema, projection_indexes, predicate); + BindExprColumnIndexes(query_ast, schema, projection_indexes, predicate->left); + for (const ExprPtr& value : predicate->values) { + BindExprColumnIndexes(query_ast, schema, projection_indexes, value); + } + } +} + +static void BindExprColumnIndexes(const Query& query_ast, const Schema& schema, + const std::vector& projection_indexes, const ExprPtr& expr) { + if (!expr) { + return; + } - if (value.literal.kind == LiteralKind::Identifier) { - if (const std::optional source_index = TryFindColumnIndex(schema, value.literal.text); - source_index.has_value()) { - return PlannedFilterValue{ - .value = - PlannedValue{ - .kind = PlannedValueKind::Column, - .column_type = schema.columns[*source_index].type, - .column_index = ensure_projected_column(*source_index), - .literal_text = {}, - }, - .literal = std::nullopt, - }; + switch (expr->kind) { + case ExprKind::Column: { + ValidateColumnQualifier(query_ast, expr->column); + const size_t source_index = FindColumnIndex(schema, expr->column.name); + const std::optional projected_index = ProjectedIndexOf(projection_indexes, source_index); + if (projected_index.has_value()) { + expr->column_index_bound = true; + expr->column_index = *projected_index; } + return; } + case ExprKind::Literal: + case ExprKind::Star: + return; + case ExprKind::Binary: + BindExprColumnIndexes(query_ast, schema, projection_indexes, expr->left); + BindExprColumnIndexes(query_ast, schema, projection_indexes, expr->right); + return; + case ExprKind::Function: + for (const ExprPtr& argument : expr->arguments) { + BindExprColumnIndexes(query_ast, schema, projection_indexes, argument); + } + BindRegexReplace(expr); + return; + case ExprKind::Case: + BindPredicateColumnIndexes(query_ast, schema, projection_indexes, expr->case_spec.condition); + BindExprColumnIndexes(query_ast, schema, projection_indexes, expr->case_spec.then_expr); + BindExprColumnIndexes(query_ast, schema, projection_indexes, expr->case_spec.else_expr); + } +} + +static bool IsSimpleCountStar(const Query& query_ast, const PlannedQuery& planned) { + if (query_ast.select_items.size() != 1 || query_ast.filter || query_ast.having || !query_ast.group_by.empty() || + !query_ast.order_by.empty() || query_ast.offset > 0 || query_ast.limit == 0) { + return false; + } + + if (planned.aggregates.size() != 1) { + return false; + } - return PlannedFilterValue{ - .value = - PlannedValue{ - .kind = PlannedValueKind::Literal, - .column_type = ColumnType::String, - .column_index = 0, - .literal_text = {}, - }, - .literal = value.literal, - }; - }; + const PlannedAgg& aggregate = planned.aggregates.front(); + return aggregate.function != nullptr && SameColumnName(aggregate.function->canonical_name, "COUNT") && + aggregate.argument_kind == AggArgumentKind::Star && !aggregate.distinct; +} - if (query_ast.filter.has_value()) { - PlannedFilterValue left = plan_filter_value(query_ast.filter->left); - PlannedFilterValue right = plan_filter_value(query_ast.filter->right); +static std::optional TryNumericLiteralValue(const ExprPtr& expr) { + if (!expr || expr->kind != ExprKind::Literal || expr->literal.kind != LiteralKind::Numeric) { + return std::nullopt; + } - const ColumnType comparison_type = left.value.kind == PlannedValueKind::Column ? left.value.column_type - : right.value.kind == PlannedValueKind::Column ? right.value.column_type - : ColumnType::String; - if (left.literal.has_value()) { - left.value.column_type = comparison_type; - left.value.literal_text = NormalizeLiteral(*left.literal, comparison_type); + try { + return ParseInt128(expr->literal.text); + } catch (const Error&) { + return std::nullopt; + } +} + +static std::string NormalizeStringLiteral(const std::string_view text) { + std::string result; + + for (size_t i = 1; i + 1 < text.size(); ++i) { + if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { + result.push_back('\''); + ++i; + continue; } + result.push_back(text[i]); + } + + return result; +} + +static std::string LiteralTextForEval(const QueryLiteral& literal) { + if (literal.kind == LiteralKind::String) { + return NormalizeStringLiteral(literal.text); + } + return literal.text; +} + +static void NormalizeRegexReplacement(std::string& replacement) { + for (size_t pos = 0; (pos = replacement.find("\\1", pos)) != std::string::npos;) { + replacement.replace(pos, 2, "$1"); + pos += 2; + } +} + +static void BindRegexReplace(const ExprPtr& expr) { + if (!expr || expr->kind != ExprKind::Function || !SameColumnName(expr->function_name, "REGEXP_REPLACE") || + expr->arguments.size() != 3 || !expr->arguments[1] || !expr->arguments[2] || + expr->arguments[1]->kind != ExprKind::Literal || expr->arguments[2]->kind != ExprKind::Literal) { + return; + } + + try { + std::string replacement = LiteralTextForEval(expr->arguments[2]->literal); + NormalizeRegexReplacement(replacement); + expr->regex_pattern = std::regex(LiteralTextForEval(expr->arguments[1]->literal)); + expr->regex_replacement = std::move(replacement); + expr->regex_replace_bound = true; + } catch (const std::regex_error&) { + expr->regex_replace_bound = false; + } +} - if (right.literal.has_value()) { - right.value.column_type = comparison_type; - right.value.literal_text = NormalizeLiteral(*right.literal, comparison_type); +static std::optional TryLiteralValueForColumnType(const QueryLiteral& literal, const ColumnType type) { + try { + if (literal.kind == LiteralKind::Numeric) { + return ParseInt128(literal.text); } - planned.filter = PlannedFilter{ - .left = std::move(left.value), - .comparison = query_ast.filter->comparison, - .right = std::move(right.value), - .comparison_type = comparison_type, - }; + if (literal.kind != LiteralKind::String) { + return std::nullopt; + } + + const std::string normalized = NormalizeStringLiteral(literal.text); + switch (type) { + case ColumnType::Boolean: + return ParseBoolean(normalized) ? 1 : 0; + case ColumnType::Int16: + return ParseInt16(normalized); + case ColumnType::Int32: + return ParseInt32(normalized); + case ColumnType::Int64: + return ParseInt64(normalized); + case ColumnType::Int128: + return ParseInt128(normalized); + case ColumnType::Date: + return ParseDate(normalized); + case ColumnType::Timestamp: + return ParseTimestamp(normalized); + case ColumnType::Character: + return ParseCharacter(normalized); + case ColumnType::String: + return std::nullopt; + } + } catch (const Error&) { + return std::nullopt; + } + + return std::nullopt; +} + +static void BindLiteralInSet(const Query& query_ast, const Schema& schema, + const std::vector& projection_indexes, const PredicatePtr& predicate) { + if (!predicate || predicate->kind != PredicateKind::In || !predicate->left || + predicate->left->kind != ExprKind::Column || predicate->values.empty()) { + return; } - planned.group_keys.reserve(query_ast.group_by.size()); + ValidateColumnQualifier(query_ast, predicate->left->column); + const size_t source_index = FindColumnIndex(schema, predicate->left->column.name); + const std::optional projected_index = ProjectedIndexOf(projection_indexes, source_index); + if (!projected_index.has_value()) { + return; + } - for (const ColumnRef& group_column : query_ast.group_by) { - ValidateColumnQualifier(query_ast, group_column); + std::unordered_set values; + values.reserve(predicate->values.size()); - const size_t source_index = FindColumnIndex(schema, group_column.name); + for (const ExprPtr& value : predicate->values) { + if (!value || value->kind != ExprKind::Literal) { + return; + } + values.insert(LiteralTextForEval(value->literal)); + } - planned.group_keys.push_back(PlannedGroupKey{ - .column_index = ensure_projected_column(source_index), - .column_type = schema.columns[source_index].type, - .output_name = schema.columns[source_index].name, - }); + predicate->literal_in_set_bound = true; + predicate->in_column_index = *projected_index; + predicate->literal_in_values = std::move(values); + + if (schema.columns[source_index].type == ColumnType::String) { + return; } - bool has_aggregate = false; - for (const auto& item : query_ast.select_items) { - if (item.kind == SelectItemKind::Aggregate) { - has_aggregate = true; - break; + std::vector typed_values; + typed_values.reserve(predicate->values.size()); + + for (const ExprPtr& value : predicate->values) { + const std::optional typed_value = TryLiteralValueForColumnType(value->literal, schema.columns[source_index].type); + if (!typed_value.has_value()) { + return; } + typed_values.push_back(*typed_value); + } + + predicate->metadata_typed_in_set_bound = true; + predicate->metadata_typed_in_column_index = source_index; + predicate->metadata_typed_in_values = std::move(typed_values); +} + +static void BindLiteralLikePattern(const Query& query_ast, const Schema& schema, + const std::vector& projection_indexes, const PredicatePtr& predicate) { + if (!predicate || (predicate->kind != PredicateKind::Like && predicate->kind != PredicateKind::NotLike) || + !predicate->left || !predicate->right || predicate->left->kind != ExprKind::Column || + predicate->right->kind != ExprKind::Literal || predicate->right->literal.kind != LiteralKind::String) { + return; + } + + ValidateColumnQualifier(query_ast, predicate->left->column); + const size_t source_index = FindColumnIndex(schema, predicate->left->column.name); + if (schema.columns[source_index].type != ColumnType::String) { + return; } - planned.select_items.reserve(query_ast.select_items.size()); - planned.aggregates.reserve(query_ast.select_items.size()); + const std::optional projected_index = ProjectedIndexOf(projection_indexes, source_index); + if (!projected_index.has_value()) { + return; + } + + predicate->literal_like_pattern_bound = true; + predicate->like_column_index = *projected_index; + predicate->like_pattern = LiteralTextForEval(predicate->right->literal); + predicate->like_negated = predicate->kind == PredicateKind::NotLike; +} + +static ComparisonKind FlipComparison(const ComparisonKind comparison) { + switch (comparison) { + case ComparisonKind::Equal: + return ComparisonKind::Equal; + case ComparisonKind::NotEqual: + return ComparisonKind::NotEqual; + case ComparisonKind::Less: + return ComparisonKind::Greater; + case ComparisonKind::LessOrEqual: + return ComparisonKind::GreaterOrEqual; + case ComparisonKind::Greater: + return ComparisonKind::Less; + case ComparisonKind::GreaterOrEqual: + return ComparisonKind::LessOrEqual; + } + + return comparison; +} + +static void BindTypedLiteralComparison(const Query& query_ast, const Schema& schema, + const std::vector& projection_indexes, const PredicatePtr& predicate) { + if (!predicate || predicate->kind != PredicateKind::Comparison || !predicate->left || !predicate->right) { + return; + } + + const ExprPtr* column_expr = nullptr; + const ExprPtr* literal_expr = nullptr; + ComparisonKind comparison = predicate->comparison; + + if (predicate->left->kind == ExprKind::Column && predicate->right->kind == ExprKind::Literal) { + column_expr = &predicate->left; + literal_expr = &predicate->right; + } else if (predicate->left->kind == ExprKind::Literal && predicate->right->kind == ExprKind::Column) { + column_expr = &predicate->right; + literal_expr = &predicate->left; + comparison = FlipComparison(comparison); + } else { + return; + } + + ValidateColumnQualifier(query_ast, (*column_expr)->column); + const size_t source_index = FindColumnIndex(schema, (*column_expr)->column.name); + const std::optional projected_index = ProjectedIndexOf(projection_indexes, source_index); + if (!projected_index.has_value()) { + return; + } + + const ColumnType type = schema.columns[source_index].type; + const std::optional literal = TryLiteralValueForColumnType((*literal_expr)->literal, type); + if (!literal.has_value()) { + return; + } + + predicate->typed_literal_comparison_bound = true; + predicate->typed_column_index = *projected_index; + predicate->typed_literal_value = *literal; + predicate->typed_comparison = comparison; + + predicate->metadata_typed_literal_comparison_bound = true; + predicate->metadata_typed_column_index = source_index; + predicate->metadata_typed_literal_value = *literal; + predicate->metadata_typed_comparison = comparison; +} + +static std::optional> TryLinearColumnExpr(const ExprPtr& expr) { + if (!expr) { + return std::nullopt; + } + + if (expr->kind == ExprKind::Column) { + return std::pair{expr->column, 0}; + } - for (const auto& item : query_ast.select_items) { - if (item.kind == SelectItemKind::GroupKey) { - if (query_ast.group_by.empty()) { - throw Error::Unsupported("executor", "plain column SELECT items require GROUP BY"); + if (expr->kind != ExprKind::Binary || !expr->left || !expr->right || expr->left->kind != ExprKind::Column) { + return std::nullopt; + } + + const std::optional literal = TryNumericLiteralValue(expr->right); + if (!literal.has_value()) { + return std::nullopt; + } + + const Int128 offset = expr->binary_op == BinaryOp::Add ? *literal : -*literal; + + return std::pair{expr->left->column, offset}; +} + +static bool IsNumericType(const ColumnType type) { + switch (type) { + case ColumnType::Boolean: + case ColumnType::Int16: + case ColumnType::Int32: + case ColumnType::Int64: + case ColumnType::Int128: + case ColumnType::Character: + return true; + case ColumnType::String: + case ColumnType::Date: + case ColumnType::Timestamp: + return false; + } + + return false; +} + +static bool SupportsDirectTypedAggInput(const AggFuncDefinition& function, const ColumnType type) { + if (IsNumericType(type)) { + return true; + } + const std::string name = ToUpperAscii(function.canonical_name); + return (name == "MIN" || name == "MAX") && (type == ColumnType::Date || type == ColumnType::Timestamp); +} + +static bool IsSimpleMetadataExtrema(const Query& query, const PlannedQuery& planned, const ColumnarMetadata& metadata) { + if (planned.plain_select || query.filter || query.having || !query.group_by.empty() || !query.order_by.empty() || + query.limit.has_value() || query.offset != 0 || planned.aggregates.empty()) { + return false; + } + + for (const PlannedAgg& aggregate : planned.aggregates) { + const std::string name = ToUpperAscii(aggregate.function->canonical_name); + if ((name != "MIN" && name != "MAX") || aggregate.distinct || + aggregate.argument_kind != AggArgumentKind::Column || !aggregate.direct_numeric_argument || + aggregate.direct_numeric_offset != 0) { + return false; + } + + for (const auto& row_group : metadata.row_groups) { + if (aggregate.column_index >= row_group.columns.size() || + !row_group.columns[aggregate.column_index].has_min_max) { + return false; } + } + } + + return true; +} + +PlannedQuery PlanQuery(const Query& query, const std::unordered_map& tables) { + PlannedQuery planned; + + const auto table_it = tables.find(ToLowerAscii(query.table_name)); + if (table_it == tables.end()) { + throw Error::NotFound("executor", "unknown table", query.table_name); + } + + planned.table_path = table_it->second; + const ColumnarBatchReader reader(planned.table_path); + planned.table_schema = reader.GetSchema(); + const Schema& schema = planned.table_schema; + + std::vector projection_indexes; + + planned.filter = query.filter; + planned.having = query.having; + CollectColumnsFromPredicate(query, schema, planned.filter, projection_indexes); - ValidateColumnQualifier(query_ast, item.column); + planned.group_keys.reserve(query.group_by.size()); - const auto it = std::ranges::find_if(planned.group_keys, [&](const PlannedGroupKey& group_key) { - return SameColumnName(group_key.output_name, item.column.name); + for (const ExprPtr& raw_group_expr : query.group_by) { + ExprPtr group_expr = ResolveGroupByExpression(query, raw_group_expr); + CollectColumnsFromExpr(query, schema, group_expr, projection_indexes); + planned.group_keys.push_back(PlannedGroupKey{ + .column_index = 0, + .expression = group_expr, + .column_type = ColumnTypeOf(query, schema, group_expr), + .output_name = group_expr->output_name, + }); + } + + const bool has_aggregate = HasAggregate(query.select_items); + planned.plain_select = planned.group_keys.empty() && !has_aggregate; + + planned.select_items.reserve(query.select_items.size()); + planned.aggregates.reserve(query.select_items.size()); + + if (planned.plain_select) { + planned.plain_select_items = query.select_items; + for (const auto& item : query.select_items) { + CollectColumnsFromExpr(query, schema, item.expression, projection_indexes); + if (item.expression && item.expression->kind == ExprKind::Star) { + for (const auto& column : schema.columns) { + planned.select_items.push_back(PlannedSelectItem{ + .kind = SelectItemKind::GroupKey, + .index = planned.select_items.size(), + .output_type = column.type, + .output_name = column.name, + }); + } + continue; + } + planned.select_items.push_back(PlannedSelectItem{ + .kind = SelectItemKind::GroupKey, + .index = planned.select_items.size(), + .output_type = ColumnTypeOf(query, schema, item.expression), + .output_name = item.output_name, }); + } + } else { + for (const auto& item : query.select_items) { + if (item.kind == SelectItemKind::GroupKey) { + const auto group_index = FindGroupKey(query.group_by.empty() ? std::vector{} : [&] { + std::vector resolved; + for (const auto& group : query.group_by) { + resolved.push_back(ResolveGroupByExpression(query, group)); + } + return resolved; + }(), item.expression); + + if (!group_index.has_value()) { + throw Error::InvalidArgument("executor", "SELECT expression must appear in GROUP BY"); + } + + planned.select_items.push_back(PlannedSelectItem{ + .kind = SelectItemKind::GroupKey, + .index = *group_index, + .output_type = planned.group_keys[*group_index].column_type, + .output_name = item.output_name, + }); + + continue; + } - if (it == planned.group_keys.end()) { - throw Error::InvalidArgument("executor", "column '" + item.column.name + "' must appear in GROUP BY"); + const AggSpec& aggregate = item.aggregate; + const AggFuncDefinition& function = ResolveAggFunc(aggregate.function_name); + CollectColumnsFromExpr(query, schema, aggregate.argument, projection_indexes); + const std::optional> direct_numeric = TryLinearColumnExpr(aggregate.argument); + + if (aggregate.distinct && !HasAggCallFeature(function.call_features, AggCallFeature::Distinct)) { + throw Error::InvalidArgument("executor", + std::string(function.canonical_name) + " does not support DISTINCT"); + } + + if (aggregate.argument_kind == AggArgumentKind::Star && + !HasAggCallFeature(function.call_features, AggCallFeature::Star)) { + throw Error::InvalidArgument("executor", + std::string(function.canonical_name) + " does not support '*'"); + } + + PlannedAgg planned_aggregate{ + .function = &function, + .distinct = aggregate.distinct, + .argument_kind = aggregate.argument_kind, + .column_index = direct_numeric.has_value() ? FindColumnIndex(schema, direct_numeric->first.name) : 0, + .direct_numeric_argument = + direct_numeric.has_value() && + SupportsDirectTypedAggInput(function, ColumnTypeOfColumn(query, schema, direct_numeric->first)), + .direct_numeric_offset = direct_numeric.has_value() ? direct_numeric->second : 0, + .input_type = aggregate.argument ? ColumnTypeOf(query, schema, aggregate.argument) : ColumnType::String, + .argument = aggregate.argument, + .output_name = item.output_name, + }; + + if (aggregate.argument_kind == AggArgumentKind::Column && + !AggSupportsInputType(function, planned_aggregate.input_type)) { + throw Error::Unsupported("executor", "aggregate " + std::string(function.canonical_name) + + " does not support its input expression"); } + const size_t aggregate_index = planned.aggregates.size(); + planned.aggregates.push_back(std::move(planned_aggregate)); planned.select_items.push_back(PlannedSelectItem{ - .kind = SelectItemKind::GroupKey, - .index = static_cast(std::distance(planned.group_keys.begin(), it)), - .output_type = it->column_type, + .kind = SelectItemKind::Aggregate, + .index = aggregate_index, + .output_type = AggregateOutputType(planned.aggregates.back()), .output_name = item.output_name, }); + } + } - continue; + planned.order_by.reserve(query.order_by.size()); + + if (planned.plain_select) { + planned.plain_order_by = query.order_by; + std::vector effective_order_by = query.order_by; + + for (const auto& item : query.select_items) { + if (item.expression && item.expression->kind != ExprKind::Star) { + const bool already_ordered = std::ranges::any_of(effective_order_by, [&](const OrderBySpec& order) { + return SameExpr(order.item.expression, item.expression); + }); + if (!already_ordered) { + effective_order_by.push_back(OrderBySpec{ + .item = item, + .descending = false, + }); + } + } } - const AggSpec& aggregate = item.aggregate; - const AggFuncDefinition& function = ResolveAggFunc(aggregate.function_name); + for (const auto& order_item : effective_order_by) { + size_t column_index = 0; + const auto selected = FindSelectItem(query, order_item.item); + const ExprPtr order_expression = + selected.has_value() ? query.select_items[*selected].expression : order_item.item.expression; + const ColumnType type = ColumnTypeOf(query, schema, order_expression); + CollectColumnsFromExpr(query, schema, order_expression, projection_indexes); + + if (order_expression && order_expression->kind == ExprKind::Column) { + column_index = FindColumnIndex(schema, order_expression->column.name); + } else if (selected.has_value()) { + column_index = *selected; + } else { + throw Error::InvalidArgument("executor", "ORDER BY expression is not supported in plain SELECT"); + } - if (aggregate.distinct && !HasAggCallFeature(function.call_features, AggCallFeature::Distinct)) { - throw Error::InvalidArgument("executor", - std::string(function.canonical_name) + " does not support DISTINCT"); + planned.order_by.push_back(PlannedOrderBy{ + .result_column_index = column_index, + .descending = order_item.descending, + .value_type = type, + }); } - if (aggregate.argument_kind == AggArgumentKind::Star && - !HasAggCallFeature(function.call_features, AggCallFeature::Star)) { - throw Error::InvalidArgument("executor", std::string(function.canonical_name) + " does not support '*'"); - } + const bool has_star_select = std::ranges::any_of(planned.plain_select_items, [](const SelectItemSpec& item) { + return item.expression && item.expression->kind == ExprKind::Star; + }); - PlannedAgg planned_aggregate{ - .function = &function, - .distinct = aggregate.distinct, - .argument_kind = aggregate.argument_kind, - .column_index = 0, - .input_type = ColumnType::String, - .output_name = item.output_name, - }; + for (const auto& order_item : query.order_by) { + if (!order_item.item.expression || order_item.item.expression->kind == ExprKind::Star) { + continue; + } - if (aggregate.argument_kind == AggArgumentKind::Column) { - ValidateColumnQualifier(query_ast, aggregate.column); + if (FindSelectItem(query, order_item.item).has_value()) { + continue; + } - const size_t source_index = FindColumnIndex(schema, aggregate.column.name); - planned_aggregate.column_index = ensure_projected_column(source_index); - planned_aggregate.input_type = schema.columns[source_index].type; + if (has_star_select) { + continue; + } - if (!AggSupportsInputType(function, planned_aggregate.input_type)) { - throw Error::Unsupported("executor", "aggregate " + std::string(function.canonical_name) + - " does not support column '" + aggregate.column.name + "'"); + const bool already_selected = std::ranges::any_of( + planned.plain_select_items, + [&](const SelectItemSpec& item) { return SameExpr(item.expression, order_item.item.expression); }); + + if (!already_selected) { + planned.plain_select_items.push_back(order_item.item); + CollectColumnsFromExpr(query, schema, order_item.item.expression, projection_indexes); + planned.select_items.push_back(PlannedSelectItem{ + .kind = SelectItemKind::GroupKey, + .index = planned.select_items.size(), + .output_type = ColumnTypeOf(query, schema, order_item.item.expression), + .output_name = order_item.item.output_name, + }); } } + } else { + for (const auto& order_item : query.order_by) { + const std::optional result_column = FindSelectItem(query, order_item.item); + if (!result_column.has_value()) { + throw Error::InvalidArgument("executor", "ORDER BY expression must appear in SELECT"); + } - const size_t aggregate_index = planned.aggregates.size(); - planned.aggregates.push_back(std::move(planned_aggregate)); - planned.select_items.push_back(PlannedSelectItem{ - .kind = SelectItemKind::Aggregate, - .index = aggregate_index, - .output_type = AggregateOutputType(planned.aggregates.back()), - .output_name = item.output_name, - }); - } - - if (query_ast.group_by.empty() && !has_aggregate) { - throw Error::Unsupported("executor", "plain SELECT without aggregates is not supported"); + planned.order_by.push_back(PlannedOrderBy{ + .result_column_index = *result_column, + .descending = order_item.descending, + .value_type = planned.select_items[*result_column].output_type, + }); + } } - planned.order_by.reserve(query_ast.order_by.size()); + planned.limit = query.limit; + planned.offset = 0; + planned.metadata_count_only = IsSimpleCountStar(query, planned); + planned.metadata_extrema_only = + !planned.metadata_count_only && IsSimpleMetadataExtrema(query, planned, reader.GetMetadata()); - for (const auto& order_item : query_ast.order_by) { - const auto it = std::ranges::find_if(query_ast.select_items, [&](const SelectItemSpec& select_item) { - return SameSelectItem(select_item, order_item.item) || SameOutputName(select_item, order_item.item); - }); + if (!planned.metadata_count_only && !planned.metadata_extrema_only) { + if (projection_indexes.empty() && !schema.columns.empty()) { + projection_indexes.push_back(0); + } + planned.projection_indexes = std::move(projection_indexes); + } - if (it == query_ast.select_items.end()) { - throw Error::InvalidArgument("executor", "ORDER BY expression must appear in SELECT"); + if (planned.plain_select) { + for (PlannedOrderBy& order : planned.order_by) { + const std::optional projected_index = + ProjectedIndexOf(planned.projection_indexes, order.result_column_index); + if (projected_index.has_value()) { + order.result_column_index = *projected_index; + } } + } - const size_t result_column_index = static_cast(std::distance(query_ast.select_items.begin(), it)); + for (PlannedAgg& aggregate : planned.aggregates) { + if (!aggregate.direct_numeric_argument) { + continue; + } - planned.order_by.push_back(PlannedOrderBy{ - .result_column_index = result_column_index, - .descending = order_item.descending, - .value_type = planned.select_items[result_column_index].output_type, - }); + const std::optional projected_index = + ProjectedIndexOf(planned.projection_indexes, aggregate.column_index); + if (projected_index.has_value()) { + aggregate.column_index = *projected_index; + } else { + aggregate.direct_numeric_argument = false; + } } - planned.limit = query_ast.limit; - - if (planned.projection_indexes.empty() && !schema.columns.empty()) { - planned.projection_indexes.push_back(0); + if (!planned.metadata_count_only && !planned.metadata_extrema_only) { + BindPredicateColumnIndexes(query, schema, planned.projection_indexes, planned.filter); + for (PlannedGroupKey& group_key : planned.group_keys) { + BindExprColumnIndexes(query, schema, planned.projection_indexes, group_key.expression); + } + for (PlannedAgg& aggregate : planned.aggregates) { + BindExprColumnIndexes(query, schema, planned.projection_indexes, aggregate.argument); + } + for (SelectItemSpec& item : planned.plain_select_items) { + BindExprColumnIndexes(query, schema, planned.projection_indexes, item.expression); + } } return planned; diff --git a/src/io/columnar_batch.cpp b/src/io/columnar_batch.cpp index 9b34359..e38a389 100644 --- a/src/io/columnar_batch.cpp +++ b/src/io/columnar_batch.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,31 @@ static void FlushWrite(const std::filesystem::path& path, std::ofstream& out) { } } +static bool SupportsChunkMinMax(const ColumnType type) { return type != ColumnType::String; } + +static void PopulateChunkMinMax(const Column& column, ColumnChunkMetadata& chunk) { + if (!SupportsChunkMinMax(column.Type()) || column.Size() == 0) { + return; + } + + Int128 min_value = column.ValueAsInt128(0); + Int128 max_value = min_value; + + for (size_t row = 1; row < column.Size(); ++row) { + const Int128 value = column.ValueAsInt128(row); + if (value < min_value) { + min_value = value; + } + if (value > max_value) { + max_value = value; + } + } + + chunk.has_min_max = true; + chunk.min_value = min_value; + chunk.max_value = max_value; +} + ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::path& path) { const auto file_metadata = GetFileMetadata(path); @@ -53,7 +79,10 @@ ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::pa throw Error::MalformedData("io", "metadata size exceeds file size", path.string()); } - return ReadMetadata(file.StreamAt(file_size - FooterSize - metadata_size)); + const std::string metadata_blob = file.ReadStringAt(file_size - FooterSize - metadata_size, metadata_size); + std::istringstream metadata_stream(metadata_blob); + + return ReadMetadata(metadata_stream); } ColumnarBatchReader::ColumnarBatchReader(const std::filesystem::path& path) @@ -76,8 +105,8 @@ std::optional ColumnarBatchReader::ReadNext() { } for (size_t col = 0; col < batch.ColumnsCount(); ++col) { - const auto& [offset, size] = row_group_columns[col]; - batch.ReadColumnFrom(col, input_.StreamAt(offset), row_count, size); + const auto& chunk = row_group_columns[col]; + batch.ReadColumnFrom(col, input_.StreamAt(chunk.offset), row_count, chunk.size); } return batch; @@ -117,11 +146,14 @@ void ColumnarBatchWriter::Write(const Batch& batch) { for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { const uint64_t offset = TellWrite(path_, out_); - batch.ColumnAt(column_index).WriteTo(out_); + const Column& column = batch.ColumnAt(column_index); + column.WriteTo(out_); const uint64_t end = TellWrite(path_, out_); - group.columns.push_back(ColumnChunkMetadata{offset, end - offset}); + ColumnChunkMetadata chunk{offset, end - offset}; + PopulateChunkMinMax(column, chunk); + group.columns.push_back(chunk); } metadata_.row_groups.push_back(std::move(group)); diff --git a/src/io/csv.cpp b/src/io/csv.cpp index 66819d6..29281b5 100644 --- a/src/io/csv.cpp +++ b/src/io/csv.cpp @@ -1,41 +1,43 @@ #include "io/csv.h" -#include #include #include "common/error.h" #include "io/file.h" +constexpr char CsvDelimiter = ','; +constexpr char CsvQuote = '"'; +constexpr char CsvLf = '\n'; +constexpr char CsvCr = '\r'; +constexpr std::string_view CsvQuotedChars = ",\"\n\r"; + class CsvCharReader { public: - explicit CsvCharReader(std::istream& in) : buffer_(in.rdbuf()) {} + explicit CsvCharReader(const std::istream& in) : buffer_(in.rdbuf()) {} - std::optional Read() const { - const Traits::int_type next = buffer_->sbumpc(); - if (Traits::eq_int_type(next, Traits::eof())) { - return std::nullopt; - } - return Traits::to_char_type(next); - } + private: + using Traits = std::streambuf::traits_type; - std::optional Peek() const { - const Traits::int_type next = buffer_->sgetc(); - if (Traits::eq_int_type(next, Traits::eof())) { - return std::nullopt; - } - return Traits::to_char_type(next); - } + public: + Traits::int_type Read() const { return buffer_->sbumpc(); } - void Discard() const { static_cast(Read()); } + Traits::int_type Peek() const { return buffer_->sgetc(); } - private: - using Traits = std::streambuf::traits_type; + static bool IsEof(const Traits::int_type value) { return Traits::eq_int_type(value, Traits::eof()); } + + static char ToChar(const Traits::int_type value) { return Traits::to_char_type(value); } + + void Discard() const { + if (!IsEof(Peek())) { + buffer_->sbumpc(); + } + } std::streambuf* buffer_; }; static bool NeedsCsvQuotes(const std::string_view value) { - return value.find_first_of(",\"\n\r") != std::string_view::npos; + return value.find_first_of(CsvQuotedChars) != std::string_view::npos; } CsvReader::CsvReader(std::istream& in) : in_(&in) {} @@ -74,9 +76,9 @@ bool CsvReader::ReadRow(std::vector& row) const { bool saw_data = false; while (true) { - const std::optional next = input.Read(); + const auto next = input.Read(); - if (!next.has_value()) { + if (input.IsEof(next)) { if (!saw_data) { return false; } @@ -87,13 +89,14 @@ bool CsvReader::ReadRow(std::vector& row) const { } saw_data = true; - const char ch = *next; + const char ch = input.ToChar(next); if (in_quotes) { - if (ch == '"') { - if (input.Peek() == '"') { + if (ch == CsvQuote) { + const auto peek = input.Peek(); + if (!input.IsEof(peek) && input.ToChar(peek) == CsvQuote) { input.Discard(); - row.back().push_back('"'); + row.back().push_back(CsvQuote); } else { in_quotes = false; } @@ -103,26 +106,27 @@ bool CsvReader::ReadRow(std::vector& row) const { continue; } - if (ch == '"') { + if (ch == CsvQuote) { if (row.back().empty()) { in_quotes = true; } else { - row.back().push_back('"'); + row.back().push_back(CsvQuote); } continue; } - if (ch == ',') { + if (ch == CsvDelimiter) { row.emplace_back(); continue; } - if (ch == '\n') { + if (ch == CsvLf) { return true; } - if (ch == '\r') { - if (input.Peek() == '\n') { + if (ch == CsvCr) { + const auto peek = input.Peek(); + if (!input.IsEof(peek) && input.ToChar(peek) == CsvLf) { input.Discard(); } return true; @@ -161,7 +165,7 @@ void CsvWriter::RebindAfterMove(const bool uses_owned_stream, std::ostream* sour void CsvWriter::WriteRow(const std::vector& row) const { for (size_t i = 0; i < row.size(); ++i) { if (i > 0) { - *out_ << ','; + *out_ << CsvDelimiter; } const std::string& value = row[i]; @@ -170,19 +174,19 @@ void CsvWriter::WriteRow(const std::vector& row) const { continue; } - *out_ << '"'; + *out_ << CsvQuote; for (const char ch : value) { - if (ch == '"') { + if (ch == CsvQuote) { *out_ << "\"\""; } else { *out_ << ch; } } - *out_ << '"'; + *out_ << CsvQuote; } - *out_ << '\n'; + *out_ << CsvLf; if (!*out_) { throw Error::Io("io", "failed to write csv row"); @@ -203,7 +207,7 @@ std::vector> ReadRows(const std::filesystem::path& path std::vector row; while (reader.ReadRow(row)) { - rows.push_back(row); + rows.push_back(std::move(row)); } return rows; diff --git a/src/model/batch.cpp b/src/model/batch.cpp index b646c3a..1ed69b3 100644 --- a/src/model/batch.cpp +++ b/src/model/batch.cpp @@ -54,13 +54,54 @@ void Batch::Reserve(const size_t n) const { } } -void Batch::AppendValueFromString(const size_t column_index, const std::string& value) const { +void Batch::AppendValueFromString(const size_t column_index, const std::string_view value) const { if (column_index >= columns_.size()) { throw Error::OutOfRange("model", "column index out of range"); } columns_[column_index]->AppendFromString(value); } +void Batch::AppendValueFromColumn(const size_t column_index, const Column& source, const size_t row) const { + if (column_index >= columns_.size()) { + throw Error::OutOfRange("model", "column index out of range"); + } + columns_[column_index]->AppendFromColumn(source, row); +} + +void Batch::AppendColumnRange(const size_t column_index, const Column& source, const size_t begin, + const size_t count) const { + if (column_index >= columns_.size()) { + throw Error::OutOfRange("model", "column index out of range"); + } + columns_[column_index]->AppendRangeFromColumn(source, begin, count); +} + +void Batch::AppendColumnSelected(const size_t column_index, const Column& source, + const std::span rows) const { + if (column_index >= columns_.size()) { + throw Error::OutOfRange("model", "column index out of range"); + } + columns_[column_index]->AppendSelectedFromColumn(source, rows); +} + +void Batch::AppendRowsRangeFromBatch(const Batch& source, const size_t begin, const size_t count) const { + if (source.ColumnsCount() != ColumnsCount()) { + throw Error::InconsistentData("model", "batch column count mismatch"); + } + for (size_t column = 0; column < ColumnsCount(); ++column) { + AppendColumnRange(column, source.ColumnAt(column), begin, count); + } +} + +void Batch::AppendRowsSelectedFromBatch(const Batch& source, const std::span rows) const { + if (source.ColumnsCount() != ColumnsCount()) { + throw Error::InconsistentData("model", "batch column count mismatch"); + } + for (size_t column = 0; column < ColumnsCount(); ++column) { + AppendColumnSelected(column, source.ColumnAt(column), rows); + } +} + void Batch::ReadColumnFrom(const size_t column_index, std::istream& in, const uint32_t row_count, const uint64_t size) const { if (column_index >= columns_.size()) { diff --git a/src/model/column.cpp b/src/model/column.cpp index 0a14447..c0058f4 100644 --- a/src/model/column.cpp +++ b/src/model/column.cpp @@ -12,6 +12,7 @@ #include "model/column_string.h" #include "model/column_timestamp.h" #include "common/error.h" +#include "common/parsing.h" std::unique_ptr CreateColumn(const ColumnType type) { switch (type) { @@ -40,6 +41,53 @@ std::unique_ptr CreateColumn(const ColumnType type) { Column::Column(const ColumnType type) : type_(type) {} +Int128 Column::ValueAsInt128(const size_t row) const { return ParseInt128(ValueAsString(row)); } + +static bool MatchesValueComparison(const Int128 lhs, const Int128 rhs, const ValueComparison comparison) { + switch (comparison) { + case ValueComparison::Equal: + return lhs == rhs; + case ValueComparison::NotEqual: + return lhs != rhs; + case ValueComparison::Less: + return lhs < rhs; + case ValueComparison::LessOrEqual: + return lhs <= rhs; + case ValueComparison::Greater: + return lhs > rhs; + case ValueComparison::GreaterOrEqual: + return lhs >= rhs; + } + + return false; +} + +void Column::SelectRowsByInt128Comparison(const Int128 rhs, const ValueComparison comparison, + std::vector& rows) const { + for (size_t row = 0; row < Size(); ++row) { + if (MatchesValueComparison(ValueAsInt128(row), rhs, comparison)) { + rows.push_back(row); + } + } +} + +void Column::SelectRowsByStringSet(const std::unordered_set& values, std::vector& rows) const { + for (size_t row = 0; row < Size(); ++row) { + if (values.contains(ValueAsString(row))) { + rows.push_back(row); + } + } +} + +void Column::SelectRowsByLikePattern(const std::string_view, const bool, std::vector&) const {} + +void Column::AppendEncodedValue(const size_t row, std::string& out) const { + const std::string value = ValueAsString(row); + out += std::to_string(value.size()); + out.push_back(':'); + out += value; +} + void Column::CheckRowIndex(const char* module, const size_t row, const size_t size) { if (row >= size) { throw Error::OutOfRange(module, "row index out of range"); diff --git a/src/model/column_string.cpp b/src/model/column_string.cpp index 807798a..e2b0e55 100644 --- a/src/model/column_string.cpp +++ b/src/model/column_string.cpp @@ -4,8 +4,11 @@ #include #include -#include "io/stream.h" #include "common/error.h" +#include "common/operator_utils.h" +#include "io/stream.h" + +constexpr char EncodedValueSeparator = ':'; StringColumn::StringColumn() : MutableColumn(ColumnType::String) {} @@ -15,43 +18,85 @@ void StringColumn::Reserve(const size_t n) { values_.reserve(n); } void StringColumn::Clear() { values_.clear(); } -void StringColumn::AppendFromString(const std::string& value) { values_.push_back(value); } +void StringColumn::AppendFromString(const std::string_view value) { values_.emplace_back(value); } + +void StringColumn::AppendFromColumn(const Column& source, const size_t row) { + if (source.Type() != ColumnType::String) { + throw Error::InconsistentData(ModuleName(), "column type mismatch"); + } + const auto& typed_source = static_cast(source); + CheckRowIndex(ModuleName(), row, typed_source.values_.size()); + values_.push_back(typed_source.values_[row]); +} + +void StringColumn::AppendRangeFromColumn(const Column& source, const size_t begin, const size_t count) { + if (source.Type() != ColumnType::String) { + throw Error::InconsistentData(ModuleName(), "column type mismatch"); + } + const auto& typed_source = static_cast(source); + if (begin > typed_source.values_.size() || count > typed_source.values_.size() - begin) { + throw Error::OutOfRange(ModuleName(), "row range out of range"); + } + values_.insert(values_.end(), typed_source.values_.begin() + static_cast(begin), + typed_source.values_.begin() + static_cast(begin + count)); +} + +void StringColumn::AppendSelectedFromColumn(const Column& source, const std::span rows) { + if (source.Type() != ColumnType::String) { + throw Error::InconsistentData(ModuleName(), "column type mismatch"); + } + const auto& typed_source = static_cast(source); + values_.reserve(values_.size() + rows.size()); + for (const size_t row : rows) { + CheckRowIndex(ModuleName(), row, typed_source.values_.size()); + values_.push_back(typed_source.values_[row]); + } +} std::string StringColumn::ValueAsString(const size_t row) const { CheckRowIndex(ModuleName(), row, values_.size()); return values_[row]; } +void StringColumn::SelectRowsByStringSet(const std::unordered_set& values, + std::vector& rows) const { + for (size_t row = 0; row < values_.size(); ++row) { + if (values.contains(values_[row])) { + rows.push_back(row); + } + } +} + +void StringColumn::SelectRowsByLikePattern(const std::string_view pattern, const bool negated, + std::vector& rows) const { + for (size_t row = 0; row < values_.size(); ++row) { + const bool matched = LikeMatches(values_[row], pattern); + if (negated ? !matched : matched) { + rows.push_back(row); + } + } +} + +void StringColumn::AppendEncodedValue(const size_t row, std::string& out) const { + CheckRowIndex(ModuleName(), row, values_.size()); + const std::string& value = values_[row]; + out += std::to_string(value.size()); + out.push_back(EncodedValueSeparator); + out += value; +} + std::unique_ptr StringColumn::Clone() const { return std::make_unique(*this); } std::unique_ptr StringColumn::CloneMutable() const { return std::make_unique(*this); } void StringColumn::WriteTo(std::ostream& out) const { - size_t total_size = 0; - for (const auto& value : values_) { if (value.size() > std::numeric_limits::max()) { throw Error::Overflow(ModuleName(), "value exceeds supported size"); } - total_size += sizeof(uint32_t) + value.size(); + WriteStream(out, static_cast(value.size())); + WriteBytes(out, value); } - - std::string buffer(total_size, '\0'); - char* dst = buffer.data(); - - for (const auto& value : values_) { - const uint32_t length = value.size(); - - std::memcpy(dst, &length, sizeof(length)); - dst += sizeof(length); - - if (!value.empty()) { - std::memcpy(dst, value.data(), value.size()); - dst += value.size(); - } - } - - WriteBytes(out, buffer); } void StringColumn::ReadFrom(std::istream& in, const uint32_t row_count, const uint64_t size) { @@ -62,7 +107,7 @@ void StringColumn::ReadFrom(std::istream& in, const uint32_t row_count, const ui for (uint32_t row_index = 0; row_index < row_count; ++row_index) { const uint32_t length = ReadStream(in); - consumed += sizeof(uint32_t); + consumed += sizeof(length); std::string value(length, '\0'); ReadBytes(in, value.data(), value.size()); diff --git a/src/model/metadata.cpp b/src/model/metadata.cpp index 66765ab..9b1dd59 100644 --- a/src/model/metadata.cpp +++ b/src/model/metadata.cpp @@ -66,6 +66,22 @@ ColumnarMetadata ReadMetadata(std::istream& in) { metadata.row_groups.push_back(std::move(group)); } + if (in.peek() == std::istream::traits_type::eof()) { + return metadata; + } + + for (auto& row_group : metadata.row_groups) { + for (auto& column : row_group.columns) { + column.has_min_max = ReadStream(in) != 0; + if (!column.has_min_max) { + continue; + } + + column.min_value = ReadStream(in); + column.max_value = ReadStream(in); + } + } + return metadata; } @@ -96,9 +112,21 @@ void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { WriteStream(out, row_count); WriteStream(out, columns.size()); - for (const auto& [offset, size] : columns) { - WriteStream(out, offset); - WriteStream(out, size); + for (const auto& column : columns) { + WriteStream(out, column.offset); + WriteStream(out, column.size); + } + } + + for (const auto& row_group : metadata.row_groups) { + for (const auto& column : row_group.columns) { + WriteStream(out, column.has_min_max ? 1 : 0); + if (!column.has_min_max) { + continue; + } + + WriteStream(out, column.min_value); + WriteStream(out, column.max_value); } } } diff --git a/src/sql_parser/tokenizer_factory.cpp b/src/sql_parser/tokenizer_factory.cpp index aec0708..e94a1c0 100644 --- a/src/sql_parser/tokenizer_factory.cpp +++ b/src/sql_parser/tokenizer_factory.cpp @@ -8,11 +8,13 @@ #include "sql_parser/tokenizer.h" static const std::unordered_map Keywords = { - {"AS", Tokens::As}, {"AND", Tokens::And}, {"AVG", Tokens::Avg}, {"BY", Tokens::By}, - {"COUNT", Tokens::Count}, {"CREATE", Tokens::Create}, {"DISTINCT", Tokens::Distinct}, {"FROM", Tokens::From}, - {"GROUP", Tokens::Group}, {"LENGTH", Tokens::Length}, {"LIMIT", Tokens::Limit}, {"MAX", Tokens::Max}, - {"MIN", Tokens::Min}, {"ORDER", Tokens::Order}, {"SELECT", Tokens::Select}, {"SUM", Tokens::Sum}, - {"WHERE", Tokens::Where}, + {"AND", Tokens::And}, {"AS", Tokens::As}, {"AVG", Tokens::Avg}, {"BY", Tokens::By}, + {"CASE", Tokens::Case}, {"COUNT", Tokens::Count}, {"CREATE", Tokens::Create}, {"DISTINCT", Tokens::Distinct}, + {"ELSE", Tokens::Else}, {"END", Tokens::End}, {"FROM", Tokens::From}, {"GROUP", Tokens::Group}, + {"HAVING", Tokens::Having}, {"IN", Tokens::In}, {"LENGTH", Tokens::Length}, {"LIKE", Tokens::Like}, + {"LIMIT", Tokens::Limit}, {"MAX", Tokens::Max}, {"MIN", Tokens::Min}, {"NOT", Tokens::Not}, + {"OFFSET", Tokens::Offset}, {"ORDER", Tokens::Order}, {"SELECT", Tokens::Select}, {"SUM", Tokens::Sum}, + {"THEN", Tokens::Then}, {"WHEN", Tokens::When}, {"WHERE", Tokens::Where}, }; static std::string DescribeCharacter(const char ch) { @@ -79,6 +81,8 @@ TokenPtr MakeToken(const Tokens type, std::string text, const size_t offset) { return std::make_shared(std::move(text), offset); case Tokens::Where: return std::make_shared(std::move(text), offset); + case Tokens::Having: + return std::make_shared>(std::move(text), offset); case Tokens::By: return std::make_shared(std::move(text), offset); case Tokens::Group: @@ -87,6 +91,24 @@ TokenPtr MakeToken(const Tokens type, std::string text, const size_t offset) { return std::make_shared(std::move(text), offset); case Tokens::Limit: return std::make_shared(std::move(text), offset); + case Tokens::Offset: + return std::make_shared>(std::move(text), offset); + case Tokens::Like: + return std::make_shared>(std::move(text), offset); + case Tokens::Not: + return std::make_shared>(std::move(text), offset); + case Tokens::In: + return std::make_shared>(std::move(text), offset); + case Tokens::Case: + return std::make_shared>(std::move(text), offset); + case Tokens::When: + return std::make_shared>(std::move(text), offset); + case Tokens::Then: + return std::make_shared>(std::move(text), offset); + case Tokens::Else: + return std::make_shared>(std::move(text), offset); + case Tokens::End: + return std::make_shared>(std::move(text), offset); case Tokens::Plus: return std::make_shared(std::move(text), offset); case Tokens::Minus: diff --git a/src/sql_parser/tokenizer_tokens.cpp b/src/sql_parser/tokenizer_tokens.cpp index 04b8552..9053291 100644 --- a/src/sql_parser/tokenizer_tokens.cpp +++ b/src/sql_parser/tokenizer_tokens.cpp @@ -36,6 +36,8 @@ std::string_view TokenTypeToString(const Tokens type) noexcept { return "length"; case Tokens::Where: return "where"; + case Tokens::Having: + return "having"; case Tokens::By: return "by"; case Tokens::Group: @@ -44,6 +46,24 @@ std::string_view TokenTypeToString(const Tokens type) noexcept { return "order"; case Tokens::Limit: return "limit"; + case Tokens::Offset: + return "offset"; + case Tokens::Like: + return "like"; + case Tokens::Not: + return "not"; + case Tokens::In: + return "in"; + case Tokens::Case: + return "case"; + case Tokens::When: + return "when"; + case Tokens::Then: + return "then"; + case Tokens::Else: + return "else"; + case Tokens::End: + return "end"; case Tokens::Plus: return "plus"; case Tokens::Minus: diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp index 172754b..3bb0a9e 100644 --- a/tests/test_columnar.cpp +++ b/tests/test_columnar.cpp @@ -100,12 +100,20 @@ TEST(columnar, metadata_offsets_and_sizes) { EXPECT_EQ(row_groups[0].columns[1].size, string_chunk_size(0, 2)); EXPECT_EQ(row_groups[1].columns[0].size, sizeof(int64_t) * 1u); EXPECT_EQ(row_groups[1].columns[1].size, string_chunk_size(2, 1)); + EXPECT_TRUE(row_groups[0].columns[0].has_min_max); + EXPECT_EQ(row_groups[0].columns[0].min_value, 1); + EXPECT_EQ(row_groups[0].columns[0].max_value, 2); + EXPECT_FALSE(row_groups[0].columns[1].has_min_max); + EXPECT_TRUE(row_groups[1].columns[0].has_min_max); + EXPECT_EQ(row_groups[1].columns[0].min_value, 3); + EXPECT_EQ(row_groups[1].columns[0].max_value, 3); + EXPECT_FALSE(row_groups[1].columns[1].has_min_max); uint64_t expected_offset = 0; - for (const auto& [row_count, columns] : row_groups) { - for (const auto& [offset, size] : columns) { - EXPECT_EQ(offset, expected_offset); - expected_offset += size; + for (const auto& row_group : row_groups) { + for (const auto& column : row_group.columns) { + EXPECT_EQ(column.offset, expected_offset); + expected_offset += column.size; } } @@ -207,3 +215,28 @@ TEST(columnar, schema_rows_with_trailing_empty_columns_are_accepted) { EXPECT_NO_THROW(ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 2)); } + +TEST(columnar, metadata_records_minmax_stats_for_date_columns) { + const TempFile schema_in("schema_date_stats"); + const TempFile data_in("data_date_stats"); + const TempFile columnar_file("columnar_date_stats"); + + WriteRows(schema_in.Path(), {{"event_date", "date"}}); + WriteRows(data_in.Path(), {{"2024-01-05"}, {"2024-01-03"}, {"2024-01-09"}, {"2024-01-01"}}); + + ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 2); + + ColumnarBatchReader reader(columnar_file.Path()); + const auto& row_groups = reader.GetMetadata().row_groups; + ASSERT_EQ(row_groups.size(), 2u); + ASSERT_EQ(row_groups[0].columns.size(), 1u); + ASSERT_EQ(row_groups[1].columns.size(), 1u); + + EXPECT_TRUE(row_groups[0].columns[0].has_min_max); + EXPECT_EQ(DateToString(static_cast(row_groups[0].columns[0].min_value)), "2024-01-03"); + EXPECT_EQ(DateToString(static_cast(row_groups[0].columns[0].max_value)), "2024-01-05"); + + EXPECT_TRUE(row_groups[1].columns[0].has_min_max); + EXPECT_EQ(DateToString(static_cast(row_groups[1].columns[0].min_value)), "2024-01-01"); + EXPECT_EQ(DateToString(static_cast(row_groups[1].columns[0].max_value)), "2024-01-09"); +} diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index e0b9241..f48594c 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -1,11 +1,16 @@ #include +#include +#include #include #include #include "convert/csv_columnar.h" #include "executor/executor.h" #include "gtest/gtest.h" +#include "io/columnar_batch.h" #include "io/csv.h" +#include "io/file.h" +#include "model/metadata.h" #include "testing/executor_test_utils.h" #include "testing/temp_file.h" @@ -255,3 +260,59 @@ TEST(executor, rejects_unknown_aggregate_functions) { ASSERT_FALSE(result.has_value()); EXPECT_EQ(result.error().GetCode(), Error::Code::Unsupported); } + +TEST(executor, prunes_row_groups_for_typed_where_filters_before_reading_chunks) { + const TempFile schema_file("executor_prune_schema"); + const TempFile data_file("executor_prune_data"); + const TempFile columnar_file("executor_prune_columnar"); + + WriteRows(schema_file.Path(), { + {"EventDate", "date"}, + {"Value", "int64"}, + }); + WriteRows(data_file.Path(), { + {"2024-01-01", "1"}, + {"2024-01-01", "2"}, + {"2024-01-09", "3"}, + {"2024-01-09", "4"}, + }); + + ConvertCsvToColumnar(schema_file.Path(), data_file.Path(), columnar_file.Path(), 2); + + ColumnarBatchReader reader(columnar_file.Path()); + ColumnarMetadata metadata = reader.GetMetadata(); + ASSERT_EQ(metadata.row_groups.size(), 2u); + ASSERT_EQ(metadata.row_groups[1].columns.size(), 2u); + + const auto file_info = GetFileMetadata(columnar_file.Path()); + ASSERT_TRUE(file_info.has_value()); + + metadata.row_groups[1].columns[0].offset = file_info->size + 1024; + + std::ostringstream metadata_stream; + WriteMetadata(metadata_stream, metadata); + const std::string metadata_blob = metadata_stream.str(); + + std::vector bytes = ReadFileBytes(columnar_file.Path()); + ASSERT_GE(bytes.size(), sizeof(uint64_t) + 4u); + + const size_t footer_offset = bytes.size() - sizeof(uint64_t) - 4u; + uint64_t metadata_size = 0; + std::memcpy(&metadata_size, bytes.data() + footer_offset, sizeof(metadata_size)); + + const size_t metadata_offset = footer_offset - metadata_size; + bytes.erase(bytes.begin() + static_cast(metadata_offset), bytes.begin() + static_cast(footer_offset)); + bytes.insert(bytes.begin() + static_cast(metadata_offset), metadata_blob.begin(), metadata_blob.end()); + + const uint64_t patched_metadata_size = metadata_blob.size(); + std::memcpy(bytes.data() + metadata_offset + metadata_blob.size(), &patched_metadata_size, sizeof(patched_metadata_size)); + + WriteFileBytes(columnar_file.Path(), bytes); + + Executor executor; + executor.RegisterTable("hits", columnar_file.Path()); + + auto result = executor.Execute("SELECT COUNT(*) FROM hits WHERE EventDate = '2024-01-01';"); + ASSERT_TRUE(result.has_value()) << result.error().what(); + EXPECT_EQ(SingleRowValues(result.value()), std::vector{"2"}); +}