diff --git a/README.md b/README.md index 4456393..7220807 100644 --- a/README.md +++ b/README.md @@ -5,38 +5,50 @@ ## Benchmark Dashboard -`generated 2026-05-28 03:03:29` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) - -### Summary - -| Metric | Value | -| --- | ---: | -| queries ok | 43 / 43 | -| queries failed | 0 | -| median warm time | 237.07 ms | -| average warm time | 268.20 ms | -| p95 warm time | 777.08 ms | -| cold/warm delta | 11.3% | -| total output size | 20.3 KB | -| max output size | 6.0 KB | -| fastest query | Q06 · 3.68 ms | -| slowest query | Q18 · 1112.19 ms | - -### Storage - -| Metric | Value | -| --- | ---: | -| source csv | `benchmarks/hits_sample.csv` | -| schema | `benchmarks/scheme.csv` | -| source size | 802.6 MB | -| columnar size | 694.9 MB | -| roundtrip csv size | 743.0 MB | -| compression ratio | 1.16x | -| columnar / csv | 86.6% | -| csv -> columnar | 5.77s | -| columnar -> csv | 9.67s | -| convert throughput | 139.2 MB/s | -| roundtrip throughput | 71.9 MB/s | +`generated 2026-05-28 21:28:51` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) + + + + + + +
+

Summary

+ + + + + + + + + + + + + + +
MetricValue
queries ok43 / 43
queries failed0
median warm time220.81 ms
average warm time231.63 ms
p95 warm time476.62 ms
cold/warm delta5.3%
total output size20.3 KB
max output size6.0 KB
fastest queryQ06 · 4.12 ms
slowest queryQ28 · 969.54 ms
+		 +

Storage

+ + + + + + + + + + + + + + + + +
MetricValue
source csvbenchmarks/hits_sample.csv
schemabenchmarks/scheme.csv
compressionlz4
source size802.6 MB
columnar size119.7 MB
roundtrip csv size743.0 MB
compression ratio6.71x
columnar / csv14.9%
csv -> columnar5.81s
columnar -> csv9.84s
convert throughput138.1 MB/s
roundtrip throughput12.2 MB/s
+
### Heatmap @@ -44,89 +56,60 @@ | slot 1 | slot 2 | slot 3 | slot 4 | slot 5 | slot 6 | | ---: | ---: | ---: | ---: | ---: | ---: | -| Q00 🟩 `3.91` | Q01 🟩 `6.32` | Q02 🟩 `17.25` | Q03 🟩 `13.37` | Q04 🟩 `148.12` | Q05 🟩 `130.96` | -| Q06 🟩 `3.68` | Q07 🟩 `6.69` | Q08 🟩 `193.36` | Q09 🟩 `209.45` | Q10 🟥 `268.08` | Q11 🟥 `269.48` | -| Q12 🟦 `233.32` | Q13 🟦 `245.55` | Q14 🟦 `238.18` | Q15 🟩 `73.32` | Q16 🟩 `122.39` | Q17 🟩 `144.28` | -| Q18 🟥 `1112.19` | Q19 🟩 `4.11` | Q20 🟩 `134.97` | Q21 🟥 `268.11` | Q22 🟥 `357.07` | Q23 🟦 `236.84` | -| Q24 🟥 `273.73` | Q25 🟦 `226.36` | Q26 🟥 `289.97` | Q27 🟥 `398.98` | Q28 🟥 `907.32` | Q29 🟥 `414.52` | -| Q30 🟦 `237.07` | Q31 🟥 `262.05` | Q32 🟥 `787.69` | Q33 🟥 `419.25` | Q34 🟥 `423.99` | Q35 🟥 `681.60` | -| Q36 🟥 `337.69` | Q37 🟦 `244.01` | Q38 🟩 `67.28` | Q39 🟥 `426.31` | Q40 🟩 `92.18` | Q41 🟩 `33.18` | -| Q42 🟥 `568.52` | | | | | | -### Largest Result Sets - -| Query | Output | Warm median, ms | SQL | -| --- | ---: | ---: | --- | -| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 357.07 | SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT… | -| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 236.84 | SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime… | -| [Q29](benchmarks/queries/query_29.sql) | 990 B | 414.52 | SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(Reso… | -| [Q39](benchmarks/queries/query_39.sql) | 835 B | 426.31 | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (… | -| [Q24](benchmarks/queries/query_24.sql) | 658 B | 273.73 | SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY… | - -### Fastest Queries - -| Query | SQL | Warm median, ms | First run, ms | -| --- | --- | ---: | ---: | -| [Q06](benchmarks/queries/query_6.sql) | SELECT MIN(EventDate), MAX(EventDate) FROM hits; | 3.68 | 4.43 | -| [Q00](benchmarks/queries/query_0.sql) | SELECT COUNT(*) FROM hits; | 3.91 | 8.14 | -| [Q19](benchmarks/queries/query_19.sql) | SELECT UserID FROM hits WHERE UserID = 435090932899640449; | 4.11 | 4.50 | -| [Q01](benchmarks/queries/query_1.sql) | SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; | 6.32 | 34.17 | -| [Q07](benchmarks/queries/query_7.sql) | SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY … | 6.69 | 6.40 | - -### Slowest Queries - -| Query | SQL | Warm median, ms | Warm max, ms | -| --- | --- | ---: | ---: | -| [Q18](benchmarks/queries/query_18.sql) | SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT… | 1112.19 | 1178.59 | -| [Q28](benchmarks/queries/query_28.sql) | SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1')… | 907.32 | 910.64 | -| [Q32](benchmarks/queries/query_32.sql) | SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(Resolution… | 787.69 | 790.48 | -| [Q35](benchmarks/queries/query_35.sql) | SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS … | 681.60 | 683.99 | -| [Q42](benchmarks/queries/query_42.sql) | SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM… | 568.52 | 569.02 | +| Q00 🟩 `7.14` | Q01 🟩 `12.85` | Q02 🟩 `21.68` | Q03 🟩 `14.30` | Q04 🟩 `147.02` | Q05 🟩 `128.76` | +| Q06 🟩 `4.12` | Q07 🟩 `11.69` | Q08 🟩 `189.51` | Q09 🟦 `208.53` | Q10 🟥 `267.84` | Q11 🟥 `267.06` | +| Q12 🟦 `225.37` | Q13 🟥 `253.71` | Q14 🟦 `227.55` | Q15 🟩 `74.16` | Q16 🟩 `116.97` | Q17 🟩 `146.00` | +| Q18 🟥 `349.51` | Q19 🟩 `4.39` | Q20 🟥 `252.69` | Q21 🟥 `277.31` | Q22 🟥 `364.71` | Q23 🟦 `240.62` | +| Q24 🟦 `220.81` | Q25 🟦 `219.69` | Q26 🟦 `220.03` | Q27 🟥 `450.51` | Q28 🟥 `969.54` | Q29 🟥 `427.54` | +| Q30 🟦 `235.10` | Q31 🟥 `262.43` | Q32 🟥 `881.20` | Q33 🟥 `349.73` | Q34 🟥 `420.41` | Q35 🟩 `149.57` | +| Q36 🟥 `339.52` | Q37 🟥 `247.46` | Q38 🟩 `73.93` | Q39 🟥 `479.52` | Q40 🟩 `95.10` | Q41 🟩 `34.88` | +| Q42 🟩 `69.69` | | | | | | ### Query Table | Query | Output CSV | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | -| [Q00](benchmarks/queries/query_0.sql) | 7 B | 8.14 | 3.95 | 3.91 | 3.78 | 4.22 | ok | -| [Q01](benchmarks/queries/query_1.sql) | 6 B | 34.17 | 6.33 | 6.32 | 6.01 | 6.67 | ok | -| [Q02](benchmarks/queries/query_2.sql) | 18 B | 48.00 | 17.26 | 17.25 | 16.83 | 17.71 | ok | -| [Q03](benchmarks/queries/query_3.sql) | 20 B | 49.77 | 13.43 | 13.37 | 13.21 | 13.76 | ok | -| [Q04](benchmarks/queries/query_4.sql) | 6 B | 148.05 | 149.27 | 148.12 | 147.47 | 153.38 | ok | -| [Q05](benchmarks/queries/query_5.sql) | 6 B | 165.31 | 130.88 | 130.96 | 130.57 | 131.02 | ok | -| [Q06](benchmarks/queries/query_6.sql) | 22 B | 4.43 | 3.69 | 3.68 | 3.60 | 3.82 | ok | -| [Q07](benchmarks/queries/query_7.sql) | 31 B | 6.40 | 6.80 | 6.69 | 6.46 | 7.35 | ok | -| [Q08](benchmarks/queries/query_8.sql) | 81 B | 230.36 | 193.81 | 193.36 | 191.19 | 197.33 | ok | -| [Q09](benchmarks/queries/query_9.sql) | 237 B | 210.07 | 210.02 | 209.45 | 209.09 | 212.08 | ok | -| [Q10](benchmarks/queries/query_10.sql) | 102 B | 299.72 | 268.08 | 268.08 | 267.32 | 268.86 | ok | -| [Q11](benchmarks/queries/query_11.sql) | 116 B | 287.55 | 270.30 | 269.48 | 268.57 | 273.67 | ok | -| [Q12](benchmarks/queries/query_12.sql) | 387 B | 229.11 | 232.94 | 233.32 | 230.58 | 234.54 | ok | -| [Q13](benchmarks/queries/query_13.sql) | 350 B | 247.57 | 246.07 | 245.55 | 244.77 | 248.40 | ok | -| [Q14](benchmarks/queries/query_14.sql) | 400 B | 249.88 | 237.64 | 238.18 | 235.56 | 238.64 | ok | -| [Q15](benchmarks/queries/query_15.sql) | 236 B | 75.58 | 73.07 | 73.32 | 71.09 | 74.55 | ok | -| [Q16](benchmarks/queries/query_16.sql) | 244 B | 130.68 | 122.37 | 122.39 | 121.98 | 122.72 | ok | -| [Q17](benchmarks/queries/query_17.sql) | 414 B | 155.01 | 144.52 | 144.28 | 142.55 | 146.97 | ok | -| [Q18](benchmarks/queries/query_18.sql) | 264 B | 1288.46 | 1127.01 | 1112.19 | 1105.08 | 1178.59 | ok | -| [Q19](benchmarks/queries/query_19.sql) | 0 B | 4.50 | 4.34 | 4.11 | 4.01 | 5.15 | ok | -| [Q20](benchmarks/queries/query_20.sql) | 3 B | 227.21 | 135.07 | 134.97 | 134.86 | 135.50 | ok | -| [Q21](benchmarks/queries/query_21.sql) | 162 B | 266.42 | 268.70 | 268.11 | 264.90 | 273.70 | ok | -| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 471.23 | 356.98 | 357.07 | 354.97 | 358.80 | ok | -| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 272.00 | 236.75 | 236.84 | 235.63 | 237.68 | ok | -| [Q24](benchmarks/queries/query_24.sql) | 658 B | 280.44 | 274.08 | 273.73 | 272.01 | 276.87 | ok | -| [Q25](benchmarks/queries/query_25.sql) | 382 B | 226.93 | 226.56 | 226.36 | 226.06 | 227.44 | ok | -| [Q26](benchmarks/queries/query_26.sql) | 658 B | 272.76 | 300.66 | 289.97 | 271.68 | 351.01 | ok | -| [Q27](benchmarks/queries/query_27.sql) | 26 B | 449.09 | 398.29 | 398.98 | 394.65 | 400.55 | ok | -| [Q28](benchmarks/queries/query_28.sql) | 556 B | 1015.82 | 907.49 | 907.32 | 904.67 | 910.64 | ok | -| [Q29](benchmarks/queries/query_29.sql) | 990 B | 414.40 | 414.81 | 414.52 | 414.18 | 416.03 | ok | -| [Q30](benchmarks/queries/query_30.sql) | 244 B | 290.88 | 236.69 | 237.07 | 234.72 | 237.89 | ok | -| [Q31](benchmarks/queries/query_31.sql) | 398 B | 271.91 | 261.89 | 262.05 | 258.68 | 264.77 | ok | -| [Q32](benchmarks/queries/query_32.sql) | 376 B | 1001.96 | 785.02 | 787.69 | 774.24 | 790.48 | ok | -| [Q33](benchmarks/queries/query_33.sql) | 464 B | 391.70 | 454.24 | 419.25 | 390.48 | 587.97 | ok | -| [Q34](benchmarks/queries/query_34.sql) | 484 B | 436.48 | 424.10 | 423.99 | 413.11 | 435.32 | ok | -| [Q35](benchmarks/queries/query_35.sql) | 493 B | 689.66 | 681.69 | 681.60 | 679.59 | 683.99 | ok | -| [Q36](benchmarks/queries/query_36.sql) | 569 B | 379.10 | 338.62 | 337.69 | 333.24 | 345.84 | ok | -| [Q37](benchmarks/queries/query_37.sql) | 550 B | 256.58 | 244.32 | 244.01 | 243.86 | 245.39 | ok | -| [Q38](benchmarks/queries/query_38.sql) | 480 B | 83.42 | 67.53 | 67.28 | 67.05 | 68.52 | ok | -| [Q39](benchmarks/queries/query_39.sql) | 835 B | 536.54 | 426.07 | 426.31 | 423.11 | 428.55 | ok | -| [Q40](benchmarks/queries/query_40.sql) | 356 B | 113.53 | 92.15 | 92.18 | 91.78 | 92.46 | ok | -| [Q41](benchmarks/queries/query_41.sql) | 132 B | 46.85 | 33.59 | 33.18 | 32.37 | 35.65 | ok | -| [Q42](benchmarks/queries/query_42.sql) | 240 B | 572.27 | 568.26 | 568.52 | 566.96 | 569.02 | ok | +| [Q00](benchmarks/queries/query_0.sql) | 7 B | 372.47 | 7.12 | 7.14 | 5.32 | 8.89 | ok | +| [Q01](benchmarks/queries/query_1.sql) | 6 B | 16.18 | 13.17 | 12.85 | 11.37 | 15.61 | ok | +| [Q02](benchmarks/queries/query_2.sql) | 18 B | 28.44 | 21.85 | 21.68 | 21.58 | 22.45 | ok | +| [Q03](benchmarks/queries/query_3.sql) | 20 B | 14.56 | 14.35 | 14.30 | 14.24 | 14.55 | ok | +| [Q04](benchmarks/queries/query_4.sql) | 6 B | 151.94 | 148.55 | 147.02 | 146.46 | 153.70 | ok | +| [Q05](benchmarks/queries/query_5.sql) | 6 B | 131.23 | 128.78 | 128.76 | 128.60 | 129.00 | ok | +| [Q06](benchmarks/queries/query_6.sql) | 22 B | 4.63 | 4.11 | 4.12 | 4.00 | 4.22 | ok | +| [Q07](benchmarks/queries/query_7.sql) | 31 B | 11.54 | 11.68 | 11.69 | 11.61 | 11.71 | ok | +| [Q08](benchmarks/queries/query_8.sql) | 81 B | 189.62 | 189.70 | 189.51 | 189.29 | 190.49 | ok | +| [Q09](benchmarks/queries/query_9.sql) | 237 B | 209.27 | 209.04 | 208.53 | 207.81 | 211.30 | ok | +| [Q10](benchmarks/queries/query_10.sql) | 102 B | 273.63 | 268.25 | 267.84 | 264.68 | 272.65 | ok | +| [Q11](benchmarks/queries/query_11.sql) | 116 B | 267.74 | 267.06 | 267.06 | 266.30 | 267.81 | ok | +| [Q12](benchmarks/queries/query_12.sql) | 387 B | 225.38 | 225.50 | 225.37 | 225.15 | 226.12 | ok | +| [Q13](benchmarks/queries/query_13.sql) | 350 B | 255.52 | 253.82 | 253.71 | 253.17 | 254.72 | ok | +| [Q14](benchmarks/queries/query_14.sql) | 400 B | 227.71 | 227.58 | 227.55 | 227.02 | 228.18 | ok | +| [Q15](benchmarks/queries/query_15.sql) | 236 B | 73.92 | 74.08 | 74.16 | 73.23 | 74.78 | ok | +| [Q16](benchmarks/queries/query_16.sql) | 244 B | 116.27 | 117.15 | 116.97 | 116.74 | 117.90 | ok | +| [Q17](benchmarks/queries/query_17.sql) | 414 B | 146.45 | 145.98 | 146.00 | 145.32 | 146.60 | ok | +| [Q18](benchmarks/queries/query_18.sql) | 264 B | 371.65 | 354.57 | 349.51 | 345.73 | 373.53 | ok | +| [Q19](benchmarks/queries/query_19.sql) | 0 B | 5.11 | 4.40 | 4.39 | 4.35 | 4.45 | ok | +| [Q20](benchmarks/queries/query_20.sql) | 3 B | 253.57 | 252.47 | 252.69 | 251.74 | 252.77 | ok | +| [Q21](benchmarks/queries/query_21.sql) | 162 B | 275.26 | 277.07 | 277.31 | 274.21 | 279.46 | ok | +| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 364.65 | 364.26 | 364.71 | 362.10 | 365.51 | ok | +| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 247.58 | 240.46 | 240.62 | 239.89 | 240.72 | ok | +| [Q24](benchmarks/queries/query_24.sql) | 672 B | 221.33 | 220.77 | 220.81 | 220.51 | 220.95 | ok | +| [Q25](benchmarks/queries/query_25.sql) | 382 B | 219.30 | 219.65 | 219.69 | 218.74 | 220.49 | ok | +| [Q26](benchmarks/queries/query_26.sql) | 658 B | 221.11 | 220.32 | 220.03 | 219.98 | 221.26 | ok | +| [Q27](benchmarks/queries/query_27.sql) | 26 B | 448.24 | 450.32 | 450.51 | 448.06 | 452.19 | ok | +| [Q28](benchmarks/queries/query_28.sql) | 556 B | 972.11 | 969.67 | 969.54 | 964.82 | 974.79 | ok | +| [Q29](benchmarks/queries/query_29.sql) | 990 B | 436.39 | 427.32 | 427.54 | 426.26 | 427.95 | ok | +| [Q30](benchmarks/queries/query_30.sql) | 244 B | 237.16 | 235.05 | 235.10 | 234.09 | 235.90 | ok | +| [Q31](benchmarks/queries/query_31.sql) | 398 B | 264.83 | 262.59 | 262.43 | 261.71 | 263.79 | ok | +| [Q32](benchmarks/queries/query_32.sql) | 376 B | 1089.10 | 866.39 | 881.20 | 808.51 | 894.64 | ok | +| [Q33](benchmarks/queries/query_33.sql) | 464 B | 354.03 | 349.81 | 349.73 | 349.37 | 350.43 | ok | +| [Q34](benchmarks/queries/query_34.sql) | 484 B | 369.46 | 435.58 | 420.41 | 370.29 | 531.22 | ok | +| [Q35](benchmarks/queries/query_35.sql) | 493 B | 151.95 | 149.11 | 149.57 | 142.89 | 154.43 | ok | +| [Q36](benchmarks/queries/query_36.sql) | 569 B | 331.43 | 345.67 | 339.52 | 333.21 | 370.41 | ok | +| [Q37](benchmarks/queries/query_37.sql) | 550 B | 249.16 | 247.76 | 247.46 | 246.64 | 249.47 | ok | +| [Q38](benchmarks/queries/query_38.sql) | 480 B | 79.44 | 74.20 | 73.93 | 73.32 | 75.60 | ok | +| [Q39](benchmarks/queries/query_39.sql) | 835 B | 400.94 | 541.20 | 479.52 | 391.72 | 814.03 | ok | +| [Q40](benchmarks/queries/query_40.sql) | 356 B | 103.53 | 95.10 | 95.10 | 94.63 | 95.58 | ok | +| [Q41](benchmarks/queries/query_41.sql) | 132 B | 39.71 | 34.90 | 34.88 | 34.50 | 35.36 | ok | +| [Q42](benchmarks/queries/query_42.sql) | 240 B | 68.37 | 69.46 | 69.69 | 68.04 | 70.42 | ok | diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index 0a6a1d8..673aaa1 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,44 +1,44 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,output_csv_bytes,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,8.14,3.95,3.91,3.78,4.22,7,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,34.17,6.33,6.32,6.01,6.67,6,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",48.00,17.26,17.25,16.83,17.71,18,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,49.77,13.43,13.37,13.21,13.76,20,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,148.05,149.27,148.12,147.47,153.38,6,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,165.31,130.88,130.96,130.57,131.02,6,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",4.43,3.69,3.68,3.60,3.82,22,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",6.40,6.80,6.69,6.46,7.35,31,ok, -8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",230.36,193.81,193.36,191.19,197.33,81,ok, -9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",210.07,210.02,209.45,209.09,212.08,237,ok, -10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",299.72,268.08,268.08,267.32,268.86,102,ok, -11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",287.55,270.30,269.48,268.57,273.67,116,ok, -12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",229.11,232.94,233.32,230.58,234.54,387,ok, -13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",247.57,246.07,245.55,244.77,248.40,350,ok, -14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",249.88,237.64,238.18,235.56,238.64,400,ok, -15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",75.58,73.07,73.32,71.09,74.55,236,ok, -16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",130.68,122.37,122.39,121.98,122.72,244,ok, -17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",155.01,144.52,144.28,142.55,146.97,414,ok, -18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",1288.46,1127.01,1112.19,1105.08,1178.59,264,ok, -19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,4.50,4.34,4.11,4.01,5.15,0,ok, -20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,227.21,135.07,134.97,134.86,135.50,3,ok, -21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",266.42,268.70,268.11,264.90,273.70,162,ok, -22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",471.23,356.98,357.07,354.97,358.80,6188,ok, -23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,272.00,236.75,236.84,235.63,237.68,2633,ok, -24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,280.44,274.08,273.73,272.01,276.87,658,ok, -25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,226.93,226.56,226.36,226.06,227.44,382,ok, -26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",272.76,300.66,289.97,271.68,351.01,658,ok, -27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",449.09,398.29,398.98,394.65,400.55,26,ok, -28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",1015.82,907.49,907.32,904.67,910.64,556,ok, -29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",414.40,414.81,414.52,414.18,416.03,990,ok, -30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",290.88,236.69,237.07,234.72,237.89,244,ok, -31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",271.91,261.89,262.05,258.68,264.77,398,ok, -32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",1001.96,785.02,787.69,774.24,790.48,376,ok, -33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",391.70,454.24,419.25,390.48,587.97,464,ok, -34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",436.48,424.10,423.99,413.11,435.32,484,ok, -35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",689.66,681.69,681.60,679.59,683.99,493,ok, -36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",379.10,338.62,337.69,333.24,345.84,569,ok, -37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",256.58,244.32,244.01,243.86,245.39,550,ok, -38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",83.42,67.53,67.28,67.05,68.52,480,ok, -39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",536.54,426.07,426.31,423.11,428.55,835,ok, -40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",113.53,92.15,92.18,91.78,92.46,356,ok, -41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",46.85,33.59,33.18,32.37,35.65,132,ok, -42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",572.27,568.26,568.52,566.96,569.02,240,ok, +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,372.47,7.12,7.14,5.32,8.89,7,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,16.18,13.17,12.85,11.37,15.61,6,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",28.44,21.85,21.68,21.58,22.45,18,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,14.56,14.35,14.30,14.24,14.55,20,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,151.94,148.55,147.02,146.46,153.70,6,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,131.23,128.78,128.76,128.60,129.00,6,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",4.63,4.11,4.12,4.00,4.22,22,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",11.54,11.68,11.69,11.61,11.71,31,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",189.62,189.70,189.51,189.29,190.49,81,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",209.27,209.04,208.53,207.81,211.30,237,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",273.63,268.25,267.84,264.68,272.65,102,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",267.74,267.06,267.06,266.30,267.81,116,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",225.38,225.50,225.37,225.15,226.12,387,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",255.52,253.82,253.71,253.17,254.72,350,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",227.71,227.58,227.55,227.02,228.18,400,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",73.92,74.08,74.16,73.23,74.78,236,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",116.27,117.15,116.97,116.74,117.90,244,ok, +17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",146.45,145.98,146.00,145.32,146.60,414,ok, +18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",371.65,354.57,349.51,345.73,373.53,264,ok, +19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,5.11,4.40,4.39,4.35,4.45,0,ok, +20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,253.57,252.47,252.69,251.74,252.77,3,ok, +21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",275.26,277.07,277.31,274.21,279.46,162,ok, +22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",364.65,364.26,364.71,362.10,365.51,6188,ok, +23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,247.58,240.46,240.62,239.89,240.72,2633,ok, +24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,221.33,220.77,220.81,220.51,220.95,672,ok, +25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,219.30,219.65,219.69,218.74,220.49,382,ok, +26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",221.11,220.32,220.03,219.98,221.26,658,ok, +27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",448.24,450.32,450.51,448.06,452.19,26,ok, +28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",972.11,969.67,969.54,964.82,974.79,556,ok, +29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",436.39,427.32,427.54,426.26,427.95,990,ok, +30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",237.16,235.05,235.10,234.09,235.90,244,ok, +31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",264.83,262.59,262.43,261.71,263.79,398,ok, +32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",1089.10,866.39,881.20,808.51,894.64,376,ok, +33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",354.03,349.81,349.73,349.37,350.43,464,ok, +34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",369.46,435.58,420.41,370.29,531.22,484,ok, +35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",151.95,149.11,149.57,142.89,154.43,493,ok, +36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",331.43,345.67,339.52,333.21,370.41,569,ok, +37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",249.16,247.76,247.46,246.64,249.47,550,ok, +38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",79.44,74.20,73.93,73.32,75.60,480,ok, +39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",400.94,541.20,479.52,391.72,814.03,835,ok, +40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",103.53,95.10,95.10,94.63,95.58,356,ok, +41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",39.71,34.90,34.88,34.50,35.36,132,ok, +42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",68.37,69.46,69.69,68.04,70.42,240,ok, diff --git a/benchmarks/support/query_result_checks.cpp b/benchmarks/support/query_result_checks.cpp index bbdcc03..78549a1 100644 --- a/benchmarks/support/query_result_checks.cpp +++ b/benchmarks/support/query_result_checks.cpp @@ -9,8 +9,8 @@ #include "common/ascii.h" #include "common/error.h" -#include "common/operator_utils.h" -#include "common/planner_utils.h" +#include "executor/comparison_utils.h" +#include "executor/query_utils.h" #include "io/csv.h" #include "io/csv_batch.h" diff --git a/include/common/string_arena.h b/include/common/string_arena.h new file mode 100644 index 0000000..5411ae7 --- /dev/null +++ b/include/common/string_arena.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include +#include +#include + +class StringArena { + public: + explicit StringArena(size_t chunk_size = 4096); + + std::string_view Store(std::string_view value); + void Clear(); + + private: + struct Chunk { + std::unique_ptr data; + size_t capacity = 0; + size_t used = 0; + }; + + Chunk& EnsureChunk(size_t size); + + size_t chunk_size_ = 0; + std::vector chunks_; +}; + +struct StringViewHash { + size_t operator()(std::string_view value) const noexcept; +}; + +struct StringViewEqual { + bool operator()(std::string_view lhs, std::string_view rhs) const noexcept; +}; + +using StringViewSet = std::unordered_set; diff --git a/include/common/string_pattern_utils.h b/include/common/string_pattern_utils.h new file mode 100644 index 0000000..7a70759 --- /dev/null +++ b/include/common/string_pattern_utils.h @@ -0,0 +1,5 @@ +#pragma once + +#include + +bool LikeMatches(std::string_view value, std::string_view pattern); diff --git a/include/convert/csv_columnar.h b/include/convert/csv_columnar.h index 058af34..a069e1e 100644 --- a/include/convert/csv_columnar.h +++ b/include/convert/csv_columnar.h @@ -3,10 +3,12 @@ #include #include +#include "io/compression.h" + inline constexpr size_t DefaultMaxRowsPerGroup = 1 << 14; void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, - const std::filesystem::path& output_path, - size_t max_rows_per_group = DefaultMaxRowsPerGroup); + const std::filesystem::path& output_path, size_t max_rows_per_group = DefaultMaxRowsPerGroup, + Compression compression = Compression::None); void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, const std::filesystem::path& data_path); diff --git a/include/common/operator_utils.h b/include/executor/comparison_utils.h similarity index 83% rename from include/common/operator_utils.h rename to include/executor/comparison_utils.h index 3d1a93d..43f13d4 100644 --- a/include/common/operator_utils.h +++ b/include/executor/comparison_utils.h @@ -7,5 +7,5 @@ std::strong_ordering CompareValues(ColumnType type, std::string_view lhs, std::string_view rhs); bool MatchesComparison(ColumnType type, std::string_view lhs, std::string_view rhs, ComparisonKind kind); -bool LikeMatches(std::string_view value, std::string_view pattern); + ColumnType AggregateOutputType(const PlannedAgg& aggregate); diff --git a/include/executor/operators_internal.h b/include/executor/operators_internal.h new file mode 100644 index 0000000..c0fd102 --- /dev/null +++ b/include/executor/operators_internal.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "executor/operator.h" +#include "executor/query_utils.h" + +std::unique_ptr CreateMetadataCountOperator(std::filesystem::path path, std::string output_name); +std::unique_ptr CreateMetadataExtremaOperator(std::filesystem::path path, std::vector aggregates); +std::unique_ptr CreateScanOperator(std::filesystem::path path, std::vector projection_indexes, + PredicatePtr filter); +std::unique_ptr CreateFilterOperator(std::unique_ptr child, PredicatePtr filter); +std::unique_ptr CreateEnsureSchemaOperator(std::unique_ptr child, Schema schema); +std::unique_ptr CreateProjectionOperator(std::unique_ptr child, std::vector items, + Schema source_schema); +std::unique_ptr CreateAggOperator(std::unique_ptr child, std::vector aggregates, + PredicatePtr filter); +std::unique_ptr CreateGroupAggOperator(std::unique_ptr child, + std::vector group_keys, + const std::vector& aggregates, + std::vector select_items, PredicatePtr filter, + PredicatePtr having, bool sort_by_group_keys); +std::unique_ptr CreateGroupAggTopKOperator(std::unique_ptr child, + std::vector group_keys, + const std::vector& aggregates, + std::vector select_items, PredicatePtr filter, + std::vector order_by, size_t limit); + +void ApplyOrderOffsetLimit(std::unique_ptr& root, const PlannedQuery& planned, bool& limit_applied_by_top_k); +Schema ProjectionOutputSchema(const std::vector& items, const Schema& source_schema); + +std::string EvalExpr(const ExprPtr& expr, const Batch& batch, size_t row); +bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, size_t row); +void SelectRowsMatchingPredicate(const PredicatePtr& predicate, const Batch& batch, std::vector& rows); +Schema BuildSelectOutputSchema(const std::vector& select_items); diff --git a/include/executor/query.h b/include/executor/query.h index 6ed56ad..36ca0e3 100644 --- a/include/executor/query.h +++ b/include/executor/query.h @@ -42,6 +42,7 @@ enum class QueryValueKind { struct QueryValue { QueryValueKind kind = QueryValueKind::Literal; + ColumnRef column; QueryLiteral literal; }; @@ -76,6 +77,7 @@ struct ExprSpec { ExprKind kind = ExprKind::Literal; ColumnRef column; + bool column_index_bound = false; size_t column_index = 0; @@ -109,8 +111,8 @@ struct PredicateSpec { PredicateKind kind = PredicateKind::Comparison; ExprPtr left; - ComparisonKind comparison = ComparisonKind::Equal; ExprPtr right; + ComparisonKind comparison = ComparisonKind::Equal; std::vector values; @@ -118,24 +120,29 @@ struct PredicateSpec { PredicatePtr rhs; bool typed_literal_comparison_bound = false; + size_t typed_column_index = 0; Int128 typed_literal_value = 0; ComparisonKind typed_comparison = ComparisonKind::Equal; bool metadata_typed_literal_comparison_bound = false; + size_t metadata_typed_column_index = 0; Int128 metadata_typed_literal_value = 0; ComparisonKind metadata_typed_comparison = ComparisonKind::Equal; bool literal_in_set_bound = false; + size_t in_column_index = 0; std::unordered_set literal_in_values; bool metadata_typed_in_set_bound = false; + size_t metadata_typed_in_column_index = 0; std::vector metadata_typed_in_values; bool literal_like_pattern_bound = false; + size_t like_column_index = 0; std::string like_pattern; bool like_negated = false; @@ -149,9 +156,12 @@ enum class AggArgumentKind { struct AggSpec { std::string function_name; bool distinct = false; + AggArgumentKind argument_kind = AggArgumentKind::Column; + ColumnRef column; ExprPtr argument; + std::string output_name; }; @@ -162,9 +172,11 @@ enum class SelectItemKind { struct SelectItemSpec { SelectItemKind kind = SelectItemKind::GroupKey; + ColumnRef column; ExprPtr expression; AggSpec aggregate; + std::string output_name; }; diff --git a/include/executor/query_plan.h b/include/executor/query_plan.h index fcebdcd..9fc7f32 100644 --- a/include/executor/query_plan.h +++ b/include/executor/query_plan.h @@ -40,13 +40,16 @@ struct PlannedGroupKey { struct PlannedSelectItem { SelectItemKind kind = SelectItemKind::GroupKey; size_t index = 0; + ColumnType output_type = ColumnType::String; std::string output_name; }; struct PlannedOrderBy { size_t result_column_index = 0; + bool descending = false; + ColumnType value_type = ColumnType::String; }; @@ -66,8 +69,9 @@ struct PlannedValue { struct PlannedFilter { PlannedValue left; - ComparisonKind comparison = ComparisonKind::Equal; PlannedValue right; + + ComparisonKind comparison = ComparisonKind::Equal; ColumnType comparison_type = ColumnType::String; }; @@ -89,6 +93,7 @@ struct PlannedQuery { std::vector select_items; std::vector plain_select_items; + std::vector order_by; std::vector plain_order_by; diff --git a/include/common/planner_utils.h b/include/executor/query_utils.h similarity index 67% rename from include/common/planner_utils.h rename to include/executor/query_utils.h index 1e5bc12..e8ff824 100644 --- a/include/common/planner_utils.h +++ b/include/executor/query_utils.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -9,10 +10,16 @@ size_t FindColumnIndex(const Schema& schema, std::string_view column_name); void ValidateColumnQualifier(const Query& query, const ColumnRef& column); +std::string UnescapeSqlString(std::string_view text); std::string NormalizeLiteral(const QueryLiteral& literal, ColumnType type); +std::string NormalizeLiteralForEval(const QueryLiteral& literal); +void NormalizeRegexReplacement(std::string& replacement); bool SameColumnName(std::string_view lhs, std::string_view rhs); bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs); bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs); bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs); +bool SameExpr(const ExprPtr& lhs, const ExprPtr& rhs); + std::vector StarColumnIndexes(const Schema& schema); +std::optional TryFindBatchColumn(const Schema& schema, std::string_view name); diff --git a/include/executor/typed_value_utils.h b/include/executor/typed_value_utils.h new file mode 100644 index 0000000..95aa863 --- /dev/null +++ b/include/executor/typed_value_utils.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include +#include + +#include "executor/query.h" +#include "model/schema.h" + +bool IsNumericColumnType(ColumnType type); +Int128 ParseColumnValueAsInt128(ColumnType type, std::string_view value); +std::optional TryParseLiteralValueAsInt128(const QueryLiteral& literal, ColumnType type); +std::string FormatInt128Value(ColumnType type, Int128 value); +ComparisonKind FlipComparison(ComparisonKind comparison); diff --git a/include/io/columnar_batch.h b/include/io/columnar_batch.h index 9dfebb0..ae8c0b8 100644 --- a/include/io/columnar_batch.h +++ b/include/io/columnar_batch.h @@ -2,8 +2,11 @@ #include #include +#include +#include #include "io/batch.h" +#include "io/compression.h" #include "io/file.h" #include "model/metadata.h" @@ -33,7 +36,7 @@ class ColumnarBatchReader final : public BatchReader { class ColumnarBatchWriter final : public BatchWriter { public: - ColumnarBatchWriter(const std::filesystem::path& path, Schema schema); + ColumnarBatchWriter(const std::filesystem::path& path, Schema schema, Compression compression = Compression::None); ColumnarBatchWriter(const ColumnarBatchWriter&) = delete; ColumnarBatchWriter(ColumnarBatchWriter&&) noexcept = default; ColumnarBatchWriter& operator=(const ColumnarBatchWriter&) = delete; @@ -43,6 +46,7 @@ class ColumnarBatchWriter final : public BatchWriter { void Write(const Batch& batch) override; void Flush() override; + void Finalize() &; void Finalize() &&; const ColumnarMetadata& GetMetadata() const { return metadata_; } @@ -52,5 +56,11 @@ class ColumnarBatchWriter final : public BatchWriter { std::ofstream out_; ColumnarMetadata metadata_; + Compression compression_ = Compression::None; bool finalized_ = false; }; + +std::vector ReadColumnChunk(const std::filesystem::path& path, InputFile& input, + const ColumnChunkMetadata& chunk); +void ReadBatchColumnChunk(const std::filesystem::path& path, InputFile& input, const ColumnChunkMetadata& chunk, + uint32_t row_count, const Batch& batch, size_t column_index); diff --git a/include/io/compression.h b/include/io/compression.h new file mode 100644 index 0000000..4f9d0f1 --- /dev/null +++ b/include/io/compression.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include +#include + +enum class Compression : uint8_t { + None = 0, + Lz4 = 1, +}; + +const char* CompressionName(Compression compression); +Compression CompressionFromName(std::string_view name); + +std::vector Compress(std::span input, Compression compression); +std::vector Decompress(std::span input, Compression compression, uint64_t uncompressed_size); diff --git a/include/io/csv_batch.h b/include/io/csv_batch.h index 97dbab2..de63f9a 100644 --- a/include/io/csv_batch.h +++ b/include/io/csv_batch.h @@ -22,6 +22,7 @@ class CsvBatchReader final : public BatchReader { private: CsvReader csv_reader_; + Schema schema_; BatchSizing sizing_; diff --git a/include/model/metadata.h b/include/model/metadata.h index dbc3844..9db52f7 100644 --- a/include/model/metadata.h +++ b/include/model/metadata.h @@ -5,12 +5,18 @@ #include #include "common/int128.h" +#include "io/compression.h" #include "model/schema.h" struct ColumnChunkMetadata { uint64_t offset = 0; - uint64_t size = 0; + + uint64_t compressed_size = 0; + uint64_t uncompressed_size = 0; + Compression compression = Compression::None; + bool has_min_max = false; + Int128 min_value = 0; Int128 max_value = 0; }; diff --git a/scripts/dashboard.py b/scripts/dashboard.py index 4f8f3eb..01c4254 100644 --- a/scripts/dashboard.py +++ b/scripts/dashboard.py @@ -76,6 +76,12 @@ def parse_args() -> argparse.Namespace: default=1 << 14, help="Row group size for the CSV -> columnar conversion benchmark.", ) + parser.add_argument( + "--compression", + default="none", + choices=("none", "lz4"), + help="Compression codec for the CSV -> columnar conversion benchmark.", + ) return parser.parse_args() @@ -128,6 +134,30 @@ def markdown_escape(text: str) -> str: return text.replace("|", "\\|") +def html_escape(text: str) -> str: + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + ) + + +def render_html_metric_table(title: str, rows: list[tuple[str, str]]) -> list[str]: + lines = [ + f"

{html_escape(title)}

", + '', + '', + "", + ] + for metric, value in rows: + lines.append( + f"" + ) + lines.extend(["", "
MetricValue
{html_escape(str(metric))}{html_escape(str(value))}
"]) + return lines + + def parse_ms(raw: str) -> float | None: if not raw: return None @@ -161,6 +191,7 @@ def collect_conversion_stats( schema_path: pathlib.Path, source_csv_path: pathlib.Path, row_group_size: int, + compression: str, ) -> dict[str, float | int | str] | None: if not schema_path.exists() or not source_csv_path.exists(): return None @@ -182,6 +213,8 @@ def collect_conversion_stats( str(output_columnar), "--row-group-size", str(row_group_size), + "--compression", + compression, ] roundtrip_command = [ str(binary), @@ -215,6 +248,7 @@ def collect_conversion_stats( return { "source_csv_path": str(source_csv_path.relative_to(root)), "schema_path": str(schema_path.relative_to(root)), + "compression": compression, "source_csv_bytes": source_csv_bytes, "columnar_bytes": columnar_bytes, "roundtrip_csv_bytes": roundtrip_csv_bytes, @@ -425,98 +459,75 @@ def render_markdown( cold_penalty = ((statistics.fmean(first_run_values) / avg_value) - 1.0) * 100.0 if avg_value and first_run_values else 0.0 fastest = min(ok_rows, key=query_metric) if ok_rows else None slowest = max(ok_rows, key=query_metric) if ok_rows else None - top_slowest = sorted(ok_rows, key=query_metric, reverse=True)[:5] - top_fastest = sorted(ok_rows, key=query_metric)[:5] - largest_outputs = sorted(ok_rows, key=lambda row: int(row["output_csv_bytes"]), reverse=True)[:5] + summary_rows = [ + ("queries ok", f"{len(ok_rows)} / {len(rows)}"), + ("queries failed", str(len(failed_rows))), + ("median warm time", f"{format_ms(median_value)} ms"), + ("average warm time", f"{format_ms(avg_value)} ms"), + ("p95 warm time", f"{format_ms(p95_value)} ms"), + ("cold/warm delta", format_percent(cold_penalty)), + ("total output size", format_size(sum(output_sizes))), + ("max output size", format_size(max(output_sizes)) if output_sizes else "—"), + ( + "fastest query", + f"Q{int(fastest['query_id']):02d} · {format_ms(query_metric(fastest))} ms" if fastest else "—", + ), + ( + "slowest query", + f"Q{int(slowest['query_id']):02d} · {format_ms(query_metric(slowest))} ms" if slowest else "—", + ), + ] lines = [ "## Benchmark Dashboard", "", f"`generated {generated_at}` · `queries {len(rows)}/{count}` · `warm runs {warm_runs}` · [`csv`]({csv_path})", "", - "### Summary", - "", - "| Metric | Value |", - "| --- | ---: |", - f"| queries ok | {len(ok_rows)} / {len(rows)} |", - f"| queries failed | {len(failed_rows)} |", - f"| median warm time | {format_ms(median_value)} ms |", - f"| average warm time | {format_ms(avg_value)} ms |", - f"| p95 warm time | {format_ms(p95_value)} ms |", - f"| cold/warm delta | {format_percent(cold_penalty)} |", - f"| total output size | {format_size(sum(output_sizes))} |", - f"| max output size | {format_size(max(output_sizes)) if output_sizes else '—'} |", - f"| fastest query | Q{int(fastest['query_id']):02d} · {format_ms(query_metric(fastest))} ms |" if fastest else "| fastest query | — |", - f"| slowest query | Q{int(slowest['query_id']):02d} · {format_ms(query_metric(slowest))} ms |" if slowest else "| slowest query | — |", - "", ] if conversion_stats is not None: + storage_rows = [ + ("source csv", conversion_stats["source_csv_path"]), + ("schema", conversion_stats["schema_path"]), + ("compression", conversion_stats["compression"]), + ("source size", format_size(int(conversion_stats["source_csv_bytes"]))), + ("columnar size", format_size(int(conversion_stats["columnar_bytes"]))), + ("roundtrip csv size", format_size(int(conversion_stats["roundtrip_csv_bytes"]))), + ("compression ratio", format_ratio(float(conversion_stats["compression_ratio"]))), + ("columnar / csv", format_percent(float(conversion_stats["columnar_vs_csv_percent"]))), + ("csv -> columnar", format_seconds(float(conversion_stats["convert_elapsed_seconds"]))), + ("columnar -> csv", format_seconds(float(conversion_stats["roundtrip_elapsed_seconds"]))), + ("convert throughput", f"{float(conversion_stats['convert_throughput_mb_s']):.1f} MB/s"), + ("roundtrip throughput", f"{float(conversion_stats['roundtrip_throughput_mb_s']):.1f} MB/s"), + ] + lines.extend( + [ + '', + '', + '", + '", + "", + "
', + *render_html_metric_table("Summary", summary_rows), + "', + *render_html_metric_table("Storage", storage_rows), + "
", + "", + ] + ) + else: lines.extend( [ - "### Storage", + "### Summary", "", "| Metric | Value |", "| --- | ---: |", - f"| source csv | `{conversion_stats['source_csv_path']}` |", - f"| schema | `{conversion_stats['schema_path']}` |", - f"| source size | {format_size(int(conversion_stats['source_csv_bytes']))} |", - f"| columnar size | {format_size(int(conversion_stats['columnar_bytes']))} |", - f"| roundtrip csv size | {format_size(int(conversion_stats['roundtrip_csv_bytes']))} |", - f"| compression ratio | {format_ratio(float(conversion_stats['compression_ratio']))} |", - f"| columnar / csv | {format_percent(float(conversion_stats['columnar_vs_csv_percent']))} |", - f"| csv -> columnar | {format_seconds(float(conversion_stats['convert_elapsed_seconds']))} |", - f"| columnar -> csv | {format_seconds(float(conversion_stats['roundtrip_elapsed_seconds']))} |", - f"| convert throughput | {float(conversion_stats['convert_throughput_mb_s']):.1f} MB/s |", - f"| roundtrip throughput | {float(conversion_stats['roundtrip_throughput_mb_s']):.1f} MB/s |", + *[f"| {markdown_escape(metric)} | {markdown_escape(str(value))} |" for metric, value in summary_rows], "", ] ) lines.extend(render_heatmap(rows, median_value)) - lines.extend( - [ - "### Largest Result Sets", - "", - "| Query | Output | Warm median, ms | SQL |", - "| --- | ---: | ---: | --- |", - ] - ) - for row in largest_outputs: - lines.append( - f"| [Q{int(row['query_id']):02d}]({row['query_file']}) | {format_size(int(row['output_csv_bytes']))} | " - f"{format_ms(query_metric(row))} | {markdown_escape(compact_sql(row['sql'], 64))} |" - ) - - lines.extend( - [ - "", - "### Fastest Queries", - "", - "| Query | SQL | Warm median, ms | First run, ms |", - "| --- | --- | ---: | ---: |", - ] - ) - for row in top_fastest: - lines.append( - f"| [Q{int(row['query_id']):02d}]({row['query_file']}) | {markdown_escape(compact_sql(row['sql'], 72))} | " - f"{format_ms(query_metric(row))} | {row['first_run_ms'] or '—'} |" - ) - - lines.extend( - [ - "", - "### Slowest Queries", - "", - "| Query | SQL | Warm median, ms | Warm max, ms |", - "| --- | --- | ---: | ---: |", - ] - ) - for row in top_slowest: - lines.append( - f"| [Q{int(row['query_id']):02d}]({row['query_file']}) | {markdown_escape(compact_sql(row['sql'], 72))} | " - f"{format_ms(query_metric(row))} | {row['warm_max_ms'] or '—'} |" - ) lines.extend( [ @@ -612,6 +623,7 @@ def main() -> int: schema_path=schema_path, source_csv_path=source_csv_path, row_group_size=args.row_group_size, + compression=args.compression, ) write_csv(csv_path, rows) markdown_block = render_markdown( diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b81e294..79c6e65 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,19 +1,32 @@ +find_path(LZ4_INCLUDE_DIR lz4.h) +find_library(LZ4_LIBRARY lz4) + +if (NOT LZ4_INCLUDE_DIR OR NOT LZ4_LIBRARY) + message(FATAL_ERROR "lz4 headers/library not found") +endif () + +add_library(lz4_external INTERFACE) +target_include_directories(lz4_external INTERFACE ${LZ4_INCLUDE_DIR}) +target_link_libraries(lz4_external INTERFACE ${LZ4_LIBRARY}) + add_library(columnar_engine_core model/batch.cpp model/column.cpp model/column_string.cpp model/metadata.cpp io/batch.cpp + io/compression.cpp io/file.cpp io/stream.cpp common/error.cpp common/ascii.cpp common/parsing.cpp - common/operator_utils.cpp - common/planner_utils.cpp + common/string_pattern_utils.cpp + common/string_arena.cpp ) target_include_directories(columnar_engine_core PUBLIC ${COLUMNAR_INCLUDE_DIRS}) +target_link_libraries(columnar_engine_core PUBLIC lz4_external) add_library(columnar_engine_columnar io/columnar_batch.cpp @@ -40,11 +53,17 @@ target_link_libraries(columnar_engine_convert ) add_library(columnar_engine_executor + executor/comparison_utils.cpp + executor/query_utils.cpp + executor/typed_value_utils.cpp executor/executor.cpp executor/query_parser.cpp executor/query_planner.cpp executor/aggregate_states.cpp executor/operators.cpp + executor/operators_aggregate.cpp + executor/operators_runtime.cpp + executor/operators_scan.cpp sql_parser/tokenizer_factory.cpp sql_parser/tokenizer_tokens.cpp sql_parser/tokenizer.cpp diff --git a/src/app/main.cpp b/src/app/main.cpp index 9277427..75ef337 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -23,6 +23,7 @@ void ConfigureConvertCommand(argparse::ArgumentParser& command) { command.add_argument("--input").required(); command.add_argument("--output").required(); command.add_argument("--row-group-size").scan<'u', size_t>().default_value(size_t{1 << 14}); + command.add_argument("--compression").default_value(std::string("none")); } void ConfigureRoundtripCommand(argparse::ArgumentParser& command) { @@ -57,7 +58,8 @@ int RunConvert(const argparse::ArgumentParser& command) { EnsureParentDirectory(output_path); ConvertCsvToColumnar(std::filesystem::path(command.get("--schema")), std::filesystem::path(command.get("--input")), output_path, - command.get("--row-group-size")); + command.get("--row-group-size"), + CompressionFromName(command.get("--compression"))); return 0; } diff --git a/src/common/string_arena.cpp b/src/common/string_arena.cpp new file mode 100644 index 0000000..b1165dd --- /dev/null +++ b/src/common/string_arena.cpp @@ -0,0 +1,48 @@ +#include "common/string_arena.h" + +#include +#include +#include + +StringArena::StringArena(const size_t chunk_size) : chunk_size_(chunk_size) {} + +std::string_view StringArena::Store(const std::string_view value) { + if (value.empty()) { + return {}; + } + + Chunk& chunk = EnsureChunk(value.size()); + const size_t offset = chunk.used; + std::memcpy(chunk.data.get() + static_cast(offset), value.data(), value.size()); + chunk.used = offset + value.size(); + return {chunk.data.get() + static_cast(offset), value.size()}; +} + +void StringArena::Clear() { + for (Chunk& chunk : chunks_) { + chunk.used = 0; + } +} + +StringArena::Chunk& StringArena::EnsureChunk(const size_t size) { + if (!chunks_.empty() && chunks_.back().capacity - chunks_.back().used >= size) { + return chunks_.back(); + } + + const size_t capacity = std::max(chunk_size_, size); + chunks_.push_back(Chunk{ + .data = std::make_unique(capacity), + .capacity = capacity, + .used = 0, + }); + + return chunks_.back(); +} + +size_t StringViewHash::operator()(const std::string_view value) const noexcept { + return std::hash{}(value); +} + +bool StringViewEqual::operator()(const std::string_view lhs, const std::string_view rhs) const noexcept { + return lhs == rhs; +} diff --git a/src/common/operator_utils.cpp b/src/common/string_pattern_utils.cpp similarity index 58% rename from src/common/operator_utils.cpp rename to src/common/string_pattern_utils.cpp index 9639f50..48252cd 100644 --- a/src/common/operator_utils.cpp +++ b/src/common/string_pattern_utils.cpp @@ -1,49 +1,11 @@ -#include "common/operator_utils.h" +#include "common/string_pattern_utils.h" -#include -#include +#include #include -#include "common/ascii.h" -#include "common/error.h" -#include "model/column_traits.h" - constexpr char LikeWildcardAny = '%'; constexpr char LikeWildcardSingle = '_'; -static bool EvaluateOrdering(const std::strong_ordering ordering, const ComparisonKind kind) { - if (kind == ComparisonKind::Equal) { - return ordering == 0; - } - - if (kind == ComparisonKind::NotEqual) { - return ordering != 0; - } - - if (kind == ComparisonKind::Less) { - return ordering < 0; - } - - if (kind == ComparisonKind::LessOrEqual) { - return ordering <= 0; - } - - if (kind == ComparisonKind::Greater) { - return ordering > 0; - } - - if (kind == ComparisonKind::GreaterOrEqual) { - return ordering >= 0; - } - - throw Error::Unsupported("executor", "unsupported comparison kind"); -} - -template -static std::strong_ordering CompareValuesImpl(const T left, const T right) { - return left <=> right; -} - static bool LikeMatchesPercentOnly(const std::string_view value, const std::string_view pattern) { std::vector segments; @@ -118,17 +80,6 @@ static bool LikeMatchesWithUnderscore(const std::string_view value, const std::s return pattern_pos == pattern.size(); } -std::strong_ordering CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { - return VisitColumnType(type, [&]() { - return CompareValuesImpl(ColumnValueTraits::Parse(lhs), ColumnValueTraits::Parse(rhs)); - }); -} - -bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, - const ComparisonKind kind) { - return EvaluateOrdering(CompareValues(type, lhs, rhs), kind); -} - bool LikeMatches(const std::string_view value, const std::string_view pattern) { if (pattern.find(LikeWildcardSingle) == std::string_view::npos) { return LikeMatchesPercentOnly(value, pattern); @@ -136,21 +87,3 @@ bool LikeMatches(const std::string_view value, const std::string_view pattern) { return LikeMatchesWithUnderscore(value, pattern); } - -ColumnType AggregateOutputType(const PlannedAgg& aggregate) { - const std::string name = ToUpperAscii(aggregate.function->canonical_name); - - if (name == "COUNT") { - return ColumnType::Int64; - } - - if (name == "SUM" || name == "AVG") { - return ColumnType::Int128; - } - - if (name == "MIN" || name == "MAX") { - return aggregate.input_type; - } - - throw Error::Unsupported("executor", "unsupported aggregate result type"); -} diff --git a/src/convert/csv_columnar.cpp b/src/convert/csv_columnar.cpp index eff1ace..ef7f599 100644 --- a/src/convert/csv_columnar.cpp +++ b/src/convert/csv_columnar.cpp @@ -8,7 +8,8 @@ #include "model/schema_csv.h" void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, - const std::filesystem::path& output_path, size_t max_rows_per_group) { + const std::filesystem::path& output_path, const size_t max_rows_per_group, + const Compression compression) { if (max_rows_per_group == 0) { throw Error::InvalidArgument("convert", "row group size must be > 0"); } @@ -19,13 +20,13 @@ void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::f sizing.max_rows = max_rows_per_group; CsvBatchReader batch_reader(data_path, schema, sizing); - ColumnarBatchWriter batch_writer(output_path, schema); + ColumnarBatchWriter batch_writer(output_path, schema, compression); while (auto batch = batch_reader.ReadNext()) { batch_writer.Write(*batch); } - std::move(batch_writer).Finalize(); + batch_writer.Finalize(); } void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index d86669f..6083cb3 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -8,8 +8,10 @@ #include "common/error.h" #include "common/int128.h" #include "common/parsing.h" +#include "common/string_arena.h" #include "executor/aggregate_function.h" #include "executor/aggregate_state.h" +#include "executor/typed_value_utils.h" #include "model/column_traits.h" void AggState::ConsumeInt128(const Int128 value) { ConsumeValue(Int128ToString(value)); } @@ -30,29 +32,6 @@ static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view }); } -static Int128 ParseToInt128(const ColumnType type, const std::string_view value) { - switch (type) { - case ColumnType::Boolean: - return ParseBoolean(value) ? 1 : 0; - case ColumnType::Int16: - return ParseInt16(value); - case ColumnType::Int32: - return ParseInt32(value); - case ColumnType::Int64: - return ParseInt64(value); - case ColumnType::Int128: - return ParseInt128(value); - case ColumnType::Character: - return ParseCharacter(value); - case ColumnType::String: - case ColumnType::Date: - case ColumnType::Timestamp: - break; - } - - throw Error::Unsupported("executor", "unsupported numeric conversion"); -} - static std::string FormatAverage(const Int128 sum, const uint64_t count) { if (count == 0) { return "0"; @@ -60,28 +39,6 @@ static std::string FormatAverage(const Int128 sum, const uint64_t count) { return Int128ToString(sum / static_cast(count)); } -static std::string FormatInt128AsType(const ColumnType type, const Int128 value) { - switch (type) { - case ColumnType::Boolean: - return BooleanToString(value != 0); - case ColumnType::Int16: - case ColumnType::Int32: - case ColumnType::Int64: - case ColumnType::Int128: - return Int128ToString(value); - case ColumnType::Date: - return DateToString(static_cast(value)); - case ColumnType::Timestamp: - return TimestampToString(static_cast(value)); - case ColumnType::Character: - return std::string(1, static_cast(value)); - case ColumnType::String: - break; - } - - throw Error::Unsupported("executor", "unsupported typed aggregate result"); -} - class CountAS final : public AggState { public: void ConsumeValue(const std::string_view) override { ConsumeRow(); } @@ -99,7 +56,7 @@ class SumAS final : public AggState { public: explicit SumAS(const ColumnType type) : type_(type) {} - void ConsumeValue(const std::string_view value) override { sum_ += ParseToInt128(type_, value); } + void ConsumeValue(const std::string_view value) override { sum_ += ParseColumnValueAsInt128(type_, value); } void ConsumeInt128(const Int128 value) override { sum_ += value; } void ConsumeRow() override { throw Error::InvalidState("executor", "SUM requires a value"); } @@ -115,7 +72,7 @@ class AvgAS final : public AggState { explicit AvgAS(const ColumnType type) : type_(type) {} void ConsumeValue(const std::string_view value) override { - sum_ += ParseToInt128(type_, value); + sum_ += ParseColumnValueAsInt128(type_, value); ++count_; } @@ -159,7 +116,7 @@ class ExtremumAS final : public AggState { std::string Finalize() const override { if (typed_extremum_.has_value()) { - return FormatInt128AsType(type_, *typed_extremum_); + return FormatInt128Value(type_, *typed_extremum_); } return extremum_.value_or(""); } @@ -176,7 +133,7 @@ class DistinctAS final : public AggState { explicit DistinctAS(std::unique_ptr nested) : nested_(std::move(nested)) {} void ConsumeValue(const std::string_view value) override { - if (seen_.insert(std::string(value)).second) { + if (seen_.insert(arena_.Store(value)).second) { nested_->ConsumeValue(value); } } @@ -187,29 +144,12 @@ class DistinctAS final : public AggState { private: std::unique_ptr nested_; - std::unordered_set seen_; + StringArena arena_; + StringViewSet seen_; }; static bool SupportsAnyType(const ColumnType) { return true; } -static bool SupportsNumericType(const ColumnType type) { - switch (type) { - case ColumnType::Boolean: - case ColumnType::Int16: - case ColumnType::Int32: - case ColumnType::Int64: - case ColumnType::Int128: - case ColumnType::Character: - return true; - case ColumnType::String: - case ColumnType::Date: - case ColumnType::Timestamp: - return false; - } - - return false; -} - static std::unique_ptr CreateCountState(const PlannedAgg&) { return std::make_unique(); } static std::unique_ptr CreateSumState(const PlannedAgg& aggregate) { @@ -254,13 +194,13 @@ AggRegistrar::AggRegistrar(const AggFuncDefinition& def) { AggRegistry::Instance [[maybe_unused]] static AggRegistrar register_sum({ .canonical_name = "SUM", - .supports_type = &SupportsNumericType, + .supports_type = &IsNumericColumnType, .factory = &CreateSumState, }); [[maybe_unused]] static AggRegistrar register_avg({ .canonical_name = "AVG", - .supports_type = &SupportsNumericType, + .supports_type = &IsNumericColumnType, .factory = &CreateAvgState, }); diff --git a/src/executor/comparison_utils.cpp b/src/executor/comparison_utils.cpp new file mode 100644 index 0000000..50565fa --- /dev/null +++ b/src/executor/comparison_utils.cpp @@ -0,0 +1,70 @@ +#include "executor/comparison_utils.h" + +#include +#include + +#include "common/ascii.h" +#include "common/error.h" +#include "model/column_traits.h" + +static bool EvaluateOrdering(const std::strong_ordering ordering, const ComparisonKind kind) { + if (kind == ComparisonKind::Equal) { + return ordering == 0; + } + + if (kind == ComparisonKind::NotEqual) { + return ordering != 0; + } + + if (kind == ComparisonKind::Less) { + return ordering < 0; + } + + if (kind == ComparisonKind::LessOrEqual) { + return ordering <= 0; + } + + if (kind == ComparisonKind::Greater) { + return ordering > 0; + } + + if (kind == ComparisonKind::GreaterOrEqual) { + return ordering >= 0; + } + + throw Error::Unsupported("executor", "unsupported comparison kind"); +} + +template +static std::strong_ordering CompareValuesImpl(const T left, const T right) { + return left <=> right; +} + +std::strong_ordering CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { + return VisitColumnType(type, [&]() { + return CompareValuesImpl(ColumnValueTraits::Parse(lhs), ColumnValueTraits::Parse(rhs)); + }); +} + +bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, + const ComparisonKind kind) { + return EvaluateOrdering(CompareValues(type, lhs, rhs), kind); +} + +ColumnType AggregateOutputType(const PlannedAgg& aggregate) { + const std::string name = ToUpperAscii(aggregate.function->canonical_name); + + if (name == "COUNT") { + return ColumnType::Int64; + } + + if (name == "SUM" || name == "AVG") { + return ColumnType::Int128; + } + + if (name == "MIN" || name == "MAX") { + return aggregate.input_type; + } + + throw Error::Unsupported("executor", "unsupported aggregate result type"); +} diff --git a/src/executor/executor.cpp b/src/executor/executor.cpp index 79045a2..1b6e087 100644 --- a/src/executor/executor.cpp +++ b/src/executor/executor.cpp @@ -26,7 +26,7 @@ ExecuteExpected Executor::Execute(const std::string_view query) const { return Batch{}; } - return std::move(*batch); + return std::move(batch).value(); } catch (const Error& error) { if (unsupported_fallback_enabled_) { return Batch{}; diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index 72d39b2..23d2aac 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -1,1631 +1,29 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/ascii.h" -#include "common/error.h" -#include "common/int128.h" -#include "common/operator_utils.h" -#include "common/parsing.h" -#include "common/planner_utils.h" -#include "executor/aggregate_state.h" #include "executor/operator.h" -#include "io/columnar_batch.h" -#include "io/file.h" - -constexpr std::string_view CountStarName = "COUNT(*)"; -constexpr std::string_view FallbackAggregateAlias = "c"; -constexpr std::string_view PageViewsAlias = "PageViews"; -constexpr std::string_view ExtractMinutePart = "MINUTE"; -constexpr std::string_view ExtractHourPart = "HOUR"; - -constexpr int64_t MinuteMicros = 60'000'000; - -constexpr std::string_view HttpScheme = "http://"; -constexpr std::string_view HttpsScheme = "https://"; -constexpr std::string_view WwwPrefix = "www."; -constexpr std::string_view RegexBackrefPlaceholder = "\\1"; -constexpr std::string_view RegexBackrefReplacement = "$1"; - -class MetadataCountOperator final : public Operator { - public: - MetadataCountOperator(std::filesystem::path path, std::string output_name) - : path_(std::move(path)), output_name_(std::move(output_name)) {} - - std::optional Next() override { - if (returned_) { - return std::nullopt; - } - - returned_ = true; - - uint64_t rows = 0; - - const ColumnarBatchReader reader(path_); - for (const auto& row_group : reader.GetMetadata().row_groups) { - rows += row_group.row_count; - } - - Batch result(Schema{{ColumnSchema(output_name_, ColumnType::Int64)}}, 1); - result.AppendValueFromString(0, std::to_string(rows)); - - return result; - } - - private: - std::filesystem::path path_; - std::string output_name_; - bool returned_ = false; -}; - -class MetadataExtremaOperator final : public Operator { - public: - MetadataExtremaOperator(std::filesystem::path path, std::vector aggregates) - : path_(std::move(path)), aggregates_(std::move(aggregates)) {} - - std::optional Next() override { - if (returned_) { - return std::nullopt; - } - - returned_ = true; - - const ColumnarBatchReader reader(path_); - const ColumnarMetadata& metadata = reader.GetMetadata(); - - Schema schema; - schema.columns.reserve(aggregates_.size()); - - std::vector> states; - states.reserve(aggregates_.size()); - - for (const PlannedAgg& aggregate : aggregates_) { - schema.columns.push_back(ColumnSchema(aggregate.output_name, AggregateOutputType(aggregate))); - states.push_back(CreateAggState(aggregate)); - } - - for (const auto& row_group : metadata.row_groups) { - for (size_t i = 0; i < aggregates_.size(); ++i) { - const PlannedAgg& aggregate = aggregates_[i]; - const auto& chunk = row_group.columns[aggregate.column_index]; - - if (!chunk.has_min_max) { - throw Error::InvalidState("executor", "metadata min/max statistics are missing", path_.string()); - } - - const std::string name = ToUpperAscii(aggregate.function->canonical_name); - states[i]->ConsumeInt128(name == "MIN" ? chunk.min_value : chunk.max_value); - } - } - - Batch result(schema); - for (size_t i = 0; i < states.size(); ++i) { - result.AppendValueFromString(i, states[i]->Finalize()); - } - - return result; - } - - private: - std::filesystem::path path_; - std::vector aggregates_; - bool returned_ = false; -}; - -static bool MatchesRowGroupComparison(const ColumnChunkMetadata& chunk, const Int128 value, - const ComparisonKind comparison) { - if (!chunk.has_min_max) { - return true; - } - - switch (comparison) { - case ComparisonKind::Equal: - return chunk.min_value <= value && value <= chunk.max_value; - case ComparisonKind::NotEqual: - return chunk.min_value != value || chunk.max_value != value; - case ComparisonKind::Less: - return chunk.min_value < value; - case ComparisonKind::LessOrEqual: - return chunk.min_value <= value; - case ComparisonKind::Greater: - return chunk.max_value > value; - case ComparisonKind::GreaterOrEqual: - return chunk.max_value >= value; - } - - return true; -} - -static bool MayMatchRowGroup(const PredicatePtr& predicate, const RowGroupMetadata& row_group) { - if (!predicate) { - return true; - } - - switch (predicate->kind) { - case PredicateKind::And: - return MayMatchRowGroup(predicate->lhs, row_group) && MayMatchRowGroup(predicate->rhs, row_group); - case PredicateKind::Comparison: - if (!predicate->metadata_typed_literal_comparison_bound || - predicate->metadata_typed_column_index >= row_group.columns.size()) { - return true; - } - return MatchesRowGroupComparison(row_group.columns[predicate->metadata_typed_column_index], - predicate->metadata_typed_literal_value, - predicate->metadata_typed_comparison); - case PredicateKind::In: - if (!predicate->metadata_typed_in_set_bound || - predicate->metadata_typed_in_column_index >= row_group.columns.size()) { - return true; - } - - return std::ranges::any_of(predicate->metadata_typed_in_values, [&](const Int128 value) { - return MatchesRowGroupComparison(row_group.columns[predicate->metadata_typed_in_column_index], value, - ComparisonKind::Equal); - }); - case PredicateKind::Like: - case PredicateKind::NotLike: - return true; - } - - return true; -} - -class ScanOperator final : public Operator { - public: - ScanOperator(std::filesystem::path path, std::vector projection_indexes, PredicatePtr filter) - : path_(std::move(path)), - input_(path_), - metadata_(ColumnarBatchReader(path_).GetMetadata()), - filter_(std::move(filter)) { - if (metadata_.schema.columns.empty()) { - throw Error::MalformedData("executor", "columnar schema is empty", path_.string()); - } - - projection_indexes_ = std::move(projection_indexes); - projected_schema_.columns.reserve(projection_indexes_.size()); - - for (const size_t source_index : projection_indexes_) { - if (source_index >= metadata_.schema.columns.size()) { - throw Error::OutOfRange("executor", "projection index out of range", path_.string()); - } - - projected_schema_.columns.push_back(metadata_.schema.columns[source_index]); - } - } - - std::optional Next() override { - if (next_group_ >= metadata_.row_groups.size()) { - return std::nullopt; - } - - const RowGroupMetadata* row_group = nullptr; - while (next_group_ < metadata_.row_groups.size()) { - const RowGroupMetadata& candidate = metadata_.row_groups[next_group_++]; - if (MayMatchRowGroup(filter_, candidate)) { - row_group = &candidate; - break; - } - } - - if (row_group == nullptr) { - return std::nullopt; - } - - Batch batch(projected_schema_, row_group->row_count); - - for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { - const size_t source_index = projection_indexes_[projected_index]; - const auto& chunk = row_group->columns[source_index]; - - batch.ReadColumnFrom(projected_index, input_.StreamAt(chunk.offset), row_group->row_count, chunk.size); - } - - return batch; - } - - private: - std::filesystem::path path_; - InputFile input_; - - ColumnarMetadata metadata_; - std::vector projection_indexes_; - Schema projected_schema_; - PredicatePtr filter_; - size_t next_group_ = 0; -}; - -static std::string NormalizeLiteralForEval(const QueryLiteral& literal) { - if (literal.kind != LiteralKind::String) { - return literal.text; - } - - std::string result; - const std::string_view text = literal.text; - - for (size_t i = 1; i + 1 < text.size(); ++i) { - if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { - result.push_back('\''); - ++i; - continue; - } - result.push_back(text[i]); - } - - return result; -} - -static std::optional TryFindBatchColumn(const Schema& schema, const std::string_view name) { - for (size_t i = 0; i < schema.columns.size(); ++i) { - if (SameColumnName(schema.columns[i].name, name)) { - return i; - } - } - return std::nullopt; -} - -static Int128 EvalInt(const std::string& value) { return ParseInt128(value); } - -static std::string FormatAggregateExprName(const ExprSpec& expr) { - const std::string name = ToUpperAscii(expr.function_name); - if (name == "COUNT" && !expr.arguments.empty() && expr.arguments.front()->kind == ExprKind::Star) { - return std::string(CountStarName); - } - - std::string out = name + "("; - - for (size_t i = 0; i < expr.arguments.size(); ++i) { - if (i > 0) { - out += ", "; - } - out += expr.arguments[i]->output_name; - } - - out += ")"; - - return out; -} - -static std::string EvalExpr(const ExprPtr& expr, const Batch& batch, size_t row); -static bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, size_t row); - -static std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t row) { - const std::string name = ToUpperAscii(expr.function_name); - - if (name == "COUNT" || name == "SUM" || name == "AVG" || name == "MIN" || name == "MAX") { - if (const auto column = TryFindBatchColumn(batch.GetSchema(), FormatAggregateExprName(expr)); - column.has_value()) { - return batch.ColumnAt(*column).ValueAsString(row); - } - if (const auto column = TryFindBatchColumn(batch.GetSchema(), FallbackAggregateAlias); column.has_value()) { - return batch.ColumnAt(*column).ValueAsString(row); - } - if (const auto column = TryFindBatchColumn(batch.GetSchema(), PageViewsAlias); column.has_value()) { - return batch.ColumnAt(*column).ValueAsString(row); - } - } - - if (name == "STRLEN" || name == "LENGTH") { - return std::to_string(EvalExpr(expr.arguments.at(0), batch, row).size()); - } - - if (name == "EXTRACT") { - const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); - const int64_t ts = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); - - const std::chrono::sys_time time{std::chrono::microseconds{ts}}; - const auto day = std::chrono::floor(time); - - const std::chrono::hh_mm_ss tod{time - day}; - - if (part == ExtractMinutePart) { - return std::to_string(tod.minutes().count()); - } - if (part == ExtractHourPart) { - return std::to_string(tod.hours().count()); - } - - throw Error::Unsupported("executor", "unsupported EXTRACT part"); - } - - if (name == "DATE_TRUNC") { - const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); - int64_t micros = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); - - if (part == ExtractMinutePart) { - micros = micros / MinuteMicros * MinuteMicros; - return TimestampToString(micros); - } - - throw Error::Unsupported("executor", "unsupported DATE_TRUNC part"); - } - - if (name == "REGEXP_REPLACE") { - const std::string source = EvalExpr(expr.arguments.at(0), batch, row); - const std::string pattern_text = EvalExpr(expr.arguments.at(1), batch, row); - std::string replacement = EvalExpr(expr.arguments.at(2), batch, row); - - if (pattern_text == R"(^https?://(?:www\.)?([^/]+)/.*$)" && replacement == R"(\1)") { - std::string_view rest = source; - if (rest.starts_with(HttpScheme)) { - rest.remove_prefix(HttpScheme.size()); - } else if (rest.starts_with(HttpsScheme)) { - rest.remove_prefix(HttpsScheme.size()); - } else { - return source; - } - - if (rest.starts_with(WwwPrefix)) { - rest.remove_prefix(WwwPrefix.size()); - } - - const size_t slash = rest.find('/'); - if (slash == std::string_view::npos || slash == 0) { - return source; - } - - const std::string_view suffix = rest.substr(slash + 1); - if (suffix.find('\n') != std::string_view::npos || suffix.find('\r') != std::string_view::npos) { - return source; - } - - return std::string(rest.substr(0, slash)); - } - - if (expr.regex_replace_bound) { - return std::regex_replace(source, expr.regex_pattern, expr.regex_replacement); - } - - for (size_t pos = 0; (pos = replacement.find(RegexBackrefPlaceholder, pos)) != std::string::npos;) { - replacement.replace(pos, RegexBackrefPlaceholder.size(), RegexBackrefReplacement); - pos += RegexBackrefReplacement.size(); - } - - const std::regex pattern(pattern_text); - - return std::regex_replace(source, pattern, replacement); - } - - throw Error::Unsupported("executor", "unsupported function '" + expr.function_name + "'"); -} - -static std::string EvalExpr(const ExprPtr& expr, const Batch& batch, const size_t row) { - switch (expr->kind) { - case ExprKind::Column: { - if (expr->column_index_bound && expr->column_index < batch.ColumnsCount() && - SameColumnName(batch.GetSchema().columns[expr->column_index].name, expr->column.name)) { - return batch.ColumnAt(expr->column_index).ValueAsString(row); - } - - const auto column = TryFindBatchColumn(batch.GetSchema(), expr->column.name); - if (!column.has_value()) { - throw Error::InvalidArgument("executor", "unknown column '" + expr->column.name + "'"); - } - - return batch.ColumnAt(*column).ValueAsString(row); - } - case ExprKind::Literal: - return NormalizeLiteralForEval(expr->literal); - case ExprKind::Binary: { - const Int128 lhs = EvalInt(EvalExpr(expr->left, batch, row)); - const Int128 rhs = EvalInt(EvalExpr(expr->right, batch, row)); - return Int128ToString(expr->binary_op == BinaryOp::Add ? lhs + rhs : lhs - rhs); - } - case ExprKind::Function: - return EvalFunction(*expr, batch, row); - case ExprKind::Case: - return EvaluatePredicate(expr->case_spec.condition, batch, row) - ? EvalExpr(expr->case_spec.then_expr, batch, row) - : EvalExpr(expr->case_spec.else_expr, batch, row); - case ExprKind::Star: - return "*"; - } - return {}; -} - -static ComparisonKind FlipComparison(const ComparisonKind comparison) { - switch (comparison) { - case ComparisonKind::Equal: - return ComparisonKind::Equal; - case ComparisonKind::NotEqual: - return ComparisonKind::NotEqual; - case ComparisonKind::Less: - return ComparisonKind::Greater; - case ComparisonKind::LessOrEqual: - return ComparisonKind::GreaterOrEqual; - case ComparisonKind::Greater: - return ComparisonKind::Less; - case ComparisonKind::GreaterOrEqual: - return ComparisonKind::LessOrEqual; - } - - return comparison; -} - -static bool MatchesInt128Comparison(const Int128 lhs, const Int128 rhs, const ComparisonKind comparison) { - switch (comparison) { - case ComparisonKind::Equal: - return lhs == rhs; - case ComparisonKind::NotEqual: - return lhs != rhs; - case ComparisonKind::Less: - return lhs < rhs; - case ComparisonKind::LessOrEqual: - return lhs <= rhs; - case ComparisonKind::Greater: - return lhs > rhs; - case ComparisonKind::GreaterOrEqual: - return lhs >= rhs; - } - - return false; -} - -static ValueComparison ToValueComparison(const ComparisonKind comparison) { - switch (comparison) { - case ComparisonKind::Equal: - return ValueComparison::Equal; - case ComparisonKind::NotEqual: - return ValueComparison::NotEqual; - case ComparisonKind::Less: - return ValueComparison::Less; - case ComparisonKind::LessOrEqual: - return ValueComparison::LessOrEqual; - case ComparisonKind::Greater: - return ValueComparison::Greater; - case ComparisonKind::GreaterOrEqual: - return ValueComparison::GreaterOrEqual; - } - - return ValueComparison::Equal; -} - -static std::optional LiteralValueForColumnType(const QueryLiteral& literal, const ColumnType type) { - try { - if (literal.kind == LiteralKind::Numeric) { - return ParseInt128(literal.text); - } - - if (literal.kind == LiteralKind::String) { - const QueryLiteral normalized_literal{NormalizeLiteralForEval(literal), LiteralKind::String}; - if (type == ColumnType::Date) { - return ParseDate(normalized_literal.text); - } - if (type == ColumnType::Timestamp) { - return ParseTimestamp(normalized_literal.text); - } - } - } catch (const Error&) { - return std::nullopt; - } - - return std::nullopt; -} - -static std::optional TryEvaluateTypedComparison(const ExprPtr& left, const ExprPtr& right, const Batch& batch, - const size_t row, const ComparisonKind comparison) { - if (!left || !right) { - return std::nullopt; - } - - if (left->kind != ExprKind::Column || right->kind != ExprKind::Literal) { - if (left->kind == ExprKind::Literal && right->kind == ExprKind::Column) { - return TryEvaluateTypedComparison(right, left, batch, row, FlipComparison(comparison)); - } - return std::nullopt; - } - - const auto column_index = TryFindBatchColumn(batch.GetSchema(), left->column.name); - if (!column_index.has_value()) { - return std::nullopt; - } - - const Column& column = batch.ColumnAt(*column_index); - if (column.Type() == ColumnType::String) { - return std::nullopt; - } - - const std::optional rhs = LiteralValueForColumnType(right->literal, column.Type()); - if (!rhs.has_value()) { - return std::nullopt; - } - - return MatchesInt128Comparison(column.ValueAsInt128(row), *rhs, comparison); -} - -static ColumnType InferComparisonType(const std::string& lhs, const std::string& rhs) { - if (TryParseInt128(lhs).has_value() && TryParseInt128(rhs).has_value()) { - return ColumnType::Int128; - } - - if (TryParseDate(lhs).has_value() && TryParseDate(rhs).has_value()) { - return ColumnType::Date; - } - - if (TryParseTimestamp(lhs).has_value() && TryParseTimestamp(rhs).has_value()) { - return ColumnType::Timestamp; - } - - return ColumnType::String; -} - -static bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, const size_t row) { - if (!predicate) { - return true; - } - - switch (predicate->kind) { - case PredicateKind::And: - return EvaluatePredicate(predicate->lhs, batch, row) && EvaluatePredicate(predicate->rhs, batch, row); - case PredicateKind::Like: { - return LikeMatches(EvalExpr(predicate->left, batch, row), EvalExpr(predicate->right, batch, row)); - } - case PredicateKind::NotLike: - return !LikeMatches(EvalExpr(predicate->left, batch, row), EvalExpr(predicate->right, batch, row)); - case PredicateKind::In: { - if (predicate->literal_in_set_bound && predicate->in_column_index < batch.ColumnsCount()) { - return predicate->literal_in_values.contains( - batch.ColumnAt(predicate->in_column_index).ValueAsString(row)); - } - - const std::string lhs = EvalExpr(predicate->left, batch, row); - - return std::ranges::any_of(predicate->values, - [&](const ExprPtr& value) { return lhs == EvalExpr(value, batch, row); }); - } - case PredicateKind::Comparison: { - if (predicate->typed_literal_comparison_bound && predicate->typed_column_index < batch.ColumnsCount()) { - return MatchesInt128Comparison(batch.ColumnAt(predicate->typed_column_index).ValueAsInt128(row), - predicate->typed_literal_value, predicate->typed_comparison); - } - - if (const std::optional result = - TryEvaluateTypedComparison(predicate->left, predicate->right, batch, row, predicate->comparison); - result.has_value()) { - return *result; - } - - const std::string lhs = EvalExpr(predicate->left, batch, row); - const std::string rhs = EvalExpr(predicate->right, batch, row); - - return MatchesComparison(InferComparisonType(lhs, rhs), lhs, rhs, predicate->comparison); - } - } - - return false; -} - -static void SelectRowsMatchingPredicate(const PredicatePtr& predicate, const Batch& batch, std::vector& rows) { - rows.clear(); - rows.reserve(batch.RowsCount()); - - if (!predicate) { - for (size_t row = 0; row < batch.RowsCount(); ++row) { - rows.push_back(row); - } - return; - } - - if (predicate->kind == PredicateKind::Comparison && predicate->typed_literal_comparison_bound && - predicate->typed_column_index < batch.ColumnsCount()) { - batch.ColumnAt(predicate->typed_column_index) - .SelectRowsByInt128Comparison(predicate->typed_literal_value, - ToValueComparison(predicate->typed_comparison), rows); - return; - } - - if ((predicate->kind == PredicateKind::Like || predicate->kind == PredicateKind::NotLike) && - predicate->literal_like_pattern_bound && predicate->like_column_index < batch.ColumnsCount()) { - batch.ColumnAt(predicate->like_column_index) - .SelectRowsByLikePattern(predicate->like_pattern, predicate->like_negated, rows); - return; - } - - if (predicate->kind == PredicateKind::In && predicate->literal_in_set_bound && - predicate->in_column_index < batch.ColumnsCount()) { - batch.ColumnAt(predicate->in_column_index).SelectRowsByStringSet(predicate->literal_in_values, rows); - return; - } - - for (size_t row = 0; row < batch.RowsCount(); ++row) { - if (EvaluatePredicate(predicate, batch, row)) { - rows.push_back(row); - } - } -} - -class FilterOperator final : public Operator { - public: - FilterOperator(std::unique_ptr child, PredicatePtr filter) - : child_(std::move(child)), filter_(std::move(filter)) {} - - std::optional Next() override { - while (auto batch = child_->Next()) { - std::vector selected_rows; - SelectRowsMatchingPredicate(filter_, *batch, selected_rows); - - if (selected_rows.empty()) { - continue; - } - - Batch filtered(batch->GetSchema(), selected_rows.size()); - filtered.AppendRowsSelectedFromBatch(*batch, selected_rows); - - return filtered; - } - - return std::nullopt; - } - - private: - std::unique_ptr child_; - PredicatePtr filter_; -}; - -class EnsureSchemaOperator final : public Operator { - public: - EnsureSchemaOperator(std::unique_ptr child, Schema schema) - : child_(std::move(child)), schema_(std::move(schema)) {} - - std::optional Next() override { - if (returned_empty_) { - return std::nullopt; - } - - if (auto batch = child_->Next()) { - returned_any_ = true; - return batch; - } - - if (returned_any_) { - return std::nullopt; - } - - returned_any_ = true; - returned_empty_ = true; - - return Batch(schema_); - } - - private: - std::unique_ptr child_; - Schema schema_; - bool returned_any_ = false; - bool returned_empty_ = false; -}; - -static bool UsesRowAggInput(const PlannedAgg& aggregate) { - return !aggregate.direct_numeric_argument && aggregate.argument_kind != AggArgumentKind::Column; -} - -static void ConsumeAggRow(const PlannedAgg& aggregate, const Batch& batch, const size_t row, AggState& state) { - if (aggregate.direct_numeric_argument) { - state.ConsumeInt128(batch.ColumnAt(aggregate.column_index).ValueAsInt128(row) + - aggregate.direct_numeric_offset); - return; - } - - if (aggregate.argument_kind == AggArgumentKind::Column) { - state.ConsumeValue(EvalExpr(aggregate.argument, batch, row)); - return; - } - - state.ConsumeRow(); -} - -static void ConsumeAggRows(const PlannedAgg& aggregate, const Batch& batch, const std::span rows, - AggState& state) { - if (UsesRowAggInput(aggregate)) { - state.ConsumeRows(rows.size()); - return; - } - - for (const size_t row : rows) { - ConsumeAggRow(aggregate, batch, row, state); - } -} - -struct AggBinding { - PlannedAgg aggregate; - std::unique_ptr state; -}; - -struct GroupKeyComponent { - ColumnType type = ColumnType::String; - Int128 int_value = 0; - std::string string_value; -}; - -static Int128 ParseGroupKeyIntValue(const ColumnType type, const std::string_view value) { - switch (type) { - case ColumnType::Boolean: - return ParseBoolean(value) ? 1 : 0; - case ColumnType::Int16: - return ParseInt16(value); - case ColumnType::Int32: - return ParseInt32(value); - case ColumnType::Int64: - return ParseInt64(value); - case ColumnType::Int128: - return ParseInt128(value); - case ColumnType::Date: - return ParseDate(value); - case ColumnType::Timestamp: - return ParseTimestamp(value); - case ColumnType::Character: - return ParseCharacter(value); - case ColumnType::String: - break; - } - - throw Error::Unsupported("executor", "unsupported group key type"); -} - -static std::string FormatGroupKeyValue(const GroupKeyComponent& value) { - switch (value.type) { - case ColumnType::Boolean: - return BooleanToString(value.int_value != 0); - case ColumnType::Int16: - case ColumnType::Int32: - case ColumnType::Int64: - case ColumnType::Int128: - return Int128ToString(value.int_value); - case ColumnType::Date: - return DateToString(static_cast(value.int_value)); - case ColumnType::Timestamp: - return TimestampToString(static_cast(value.int_value)); - case ColumnType::Character: - return std::string(1, static_cast(value.int_value)); - case ColumnType::String: - return value.string_value; - } - - return {}; -} - -static std::strong_ordering CompareGroupKeyValue(const GroupKeyComponent& lhs, const GroupKeyComponent& rhs) { - if (lhs.type != rhs.type) { - throw Error::InconsistentData("executor", "group key type mismatch"); - } - - if (lhs.type == ColumnType::String) { - return lhs.string_value <=> rhs.string_value; - } - - return lhs.int_value <=> rhs.int_value; -} - -static bool GroupKeyValueEquals(const GroupKeyComponent& lhs, const GroupKeyComponent& rhs) { - return CompareGroupKeyValue(lhs, rhs) == 0; -} - -static size_t HashCombine(const size_t seed, const size_t value) { - return seed ^ (value + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2)); -} - -static size_t HashInt128(const Int128 value) { - const auto bits = static_cast(value); - const uint64_t low = static_cast(bits); - const uint64_t high = static_cast(bits >> 64); - - return HashCombine(std::hash{}(high), std::hash{}(low)); -} - -struct TypedGroupKey { - std::vector values; -}; - -struct TypedGroupKeyHash { - size_t operator()(const TypedGroupKey& key) const { - size_t hash = 0; - - for (const GroupKeyComponent& value : key.values) { - hash = HashCombine(hash, std::hash{}(static_cast(value.type))); - hash = HashCombine(hash, value.type == ColumnType::String ? std::hash{}(value.string_value) - : HashInt128(value.int_value)); - } - - return hash; - } -}; - -struct TypedGroupKeyEqual { - bool operator()(const TypedGroupKey& lhs, const TypedGroupKey& rhs) const { - if (lhs.values.size() != rhs.values.size()) { - return false; - } - - for (size_t i = 0; i < lhs.values.size(); ++i) { - if (!GroupKeyValueEquals(lhs.values[i], rhs.values[i])) { - return false; - } - } - - return true; - } -}; - -class GroupKeyMaterializer { - public: - explicit GroupKeyMaterializer(std::vector group_keys) : group_keys_(std::move(group_keys)) {} - - TypedGroupKey Materialize(const Batch& batch, const size_t row) const { - TypedGroupKey key; - key.values.reserve(group_keys_.size()); - - for (const auto& group_key : group_keys_) { - if (group_key.expression && group_key.expression->kind == ExprKind::Column && - group_key.expression->column_index_bound && group_key.expression->column_index < batch.ColumnsCount()) { - const Column& column = batch.ColumnAt(group_key.expression->column_index); - if (group_key.column_type == ColumnType::String) { - key.values.push_back(GroupKeyComponent{ - .type = group_key.column_type, - .int_value = 0, - .string_value = column.ValueAsString(row), - }); - } else { - key.values.push_back(GroupKeyComponent{ - .type = group_key.column_type, - .int_value = column.ValueAsInt128(row), - .string_value = {}, - }); - } - continue; - } - - std::string value = EvalExpr(group_key.expression, batch, row); - if (group_key.column_type == ColumnType::String) { - key.values.push_back(GroupKeyComponent{ - .type = group_key.column_type, - .int_value = 0, - .string_value = std::move(value), - }); - } else { - key.values.push_back(GroupKeyComponent{ - .type = group_key.column_type, - .int_value = ParseGroupKeyIntValue(group_key.column_type, value), - .string_value = {}, - }); - } - } - - return key; - } - - std::vector group_keys_; -}; - -class AggOperator final : public Operator { - public: - AggOperator(std::unique_ptr child, const std::vector& aggregates, PredicatePtr filter) - : child_(std::move(child)), filter_(std::move(filter)) { - bindings_.reserve(aggregates.size()); - - for (const auto& aggregate : aggregates) { - bindings_.push_back(AggBinding{ - .aggregate = aggregate, - .state = CreateAggState(aggregate), - }); - } - } - - std::optional Next() override { - if (returned_) { - return std::nullopt; - } - - returned_ = true; - - while (auto batch = child_->Next()) { - std::vector selected_rows; - SelectRowsMatchingPredicate(filter_, *batch, selected_rows); - - for (const auto& binding : bindings_) { - ConsumeAggRows(binding.aggregate, *batch, selected_rows, *binding.state); - } - } - - Schema schema; - schema.columns.reserve(bindings_.size()); - - for (const auto& binding : bindings_) { - schema.columns.push_back( - ColumnSchema(binding.aggregate.output_name, AggregateOutputType(binding.aggregate))); - } - - Batch result(schema); - - for (size_t i = 0; i < bindings_.size(); ++i) { - result.AppendValueFromString(i, bindings_[i].state->Finalize()); - } - - return result; - } - - private: - std::unique_ptr child_; - PredicatePtr filter_; - std::vector bindings_; - bool returned_ = false; -}; - -class GroupAggOperator final : public Operator { - public: - GroupAggOperator(std::unique_ptr child, std::vector group_keys, - const std::vector& aggregates, std::vector select_items, - PredicatePtr filter, PredicatePtr having, const bool sort_by_group_keys) - : child_(std::move(child)), - group_key_materializer_(std::move(group_keys)), - select_items_(std::move(select_items)), - filter_(std::move(filter)), - having_(std::move(having)), - sort_by_group_keys_(sort_by_group_keys) { - bindings_.reserve(aggregates.size()); - - for (const auto& aggregate : aggregates) { - bindings_.push_back(GroupAggBinding{ - .aggregate = aggregate, - }); - } - } - - std::optional Next() override { - if (returned_) { - return std::nullopt; - } - - returned_ = true; - - while (auto batch = child_->Next()) { - std::vector selected_rows; - SelectRowsMatchingPredicate(filter_, *batch, selected_rows); - - for (const size_t row : selected_rows) { - TypedGroupKey key = group_key_materializer_.Materialize(*batch, row); - - size_t group_index = 0; - const auto it = group_indexes_.find(key); - if (it == group_indexes_.end()) { - group_index = groups_.size(); - group_indexes_.emplace(key, group_index); - - groups_.push_back(GroupState{ - .key_values = std::move(key.values), - .states = CreateStates(), - }); - } else { - group_index = it->second; - } - - GroupState& group = groups_[group_index]; - for (size_t i = 0; i < bindings_.size(); ++i) { - ConsumeAggRow(bindings_[i].aggregate, *batch, row, *group.states[i]); - } - } - } - - Schema schema; - schema.columns.reserve(select_items_.size()); - - for (const auto& item : select_items_) { - schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); - } - - std::vector output_order = BuildOutputOrder(); - Batch result(schema, output_order.size()); - - for (const size_t group_index : output_order) { - AppendGroupToBatch(groups_[group_index], result); - } - - if (having_) { - std::vector selected_rows; - selected_rows.reserve(result.RowsCount()); - - for (size_t row = 0; row < result.RowsCount(); ++row) { - if (EvaluatePredicate(having_, result, row)) { - selected_rows.push_back(row); - } - } - - Batch filtered(schema, selected_rows.size()); - filtered.AppendRowsSelectedFromBatch(result, selected_rows); - - return filtered; - } - - return result; - } - - private: - struct GroupState { - std::vector key_values; - std::vector> states; - }; - - struct GroupAggBinding { - PlannedAgg aggregate; - }; - - std::vector> CreateStates() const { - std::vector> states; - states.reserve(bindings_.size()); - - for (const auto& binding : bindings_) { - states.push_back(CreateAggState(binding.aggregate)); - } - - return states; - } - - void AppendGroupToBatch(const GroupState& group, const Batch& batch) const { - for (size_t column = 0; column < select_items_.size(); ++column) { - const PlannedSelectItem& item = select_items_[column]; - if (item.kind == SelectItemKind::GroupKey) { - batch.AppendValueFromString(column, FormatGroupKeyValue(group.key_values[item.index])); - } else { - batch.AppendValueFromString(column, group.states[item.index]->Finalize()); - } - } - } - - std::vector BuildOutputOrder() const { - std::vector order; - order.reserve(groups_.size()); - - for (size_t i = 0; i < groups_.size(); ++i) { - order.push_back(i); - } - - if (!sort_by_group_keys_) { - return order; - } - - std::vector group_key_columns; - - for (size_t i = 0; i < select_items_.size(); ++i) { - if (select_items_[i].kind == SelectItemKind::GroupKey) { - group_key_columns.push_back(i); - } - } - - if (group_key_columns.empty()) { - return order; - } - - std::ranges::sort(order, [&](const size_t lhs_index, const size_t rhs_index) { - const GroupState& lhs = groups_[lhs_index]; - const GroupState& rhs = groups_[rhs_index]; - for (const size_t column : group_key_columns) { - const PlannedSelectItem& item = select_items_[column]; - const std::strong_ordering comparison = - CompareGroupKeyValue(lhs.key_values[item.index], rhs.key_values[item.index]); - if (comparison != 0) { - return comparison < 0; - } - } - - return false; - }); - - return order; - } - - std::unique_ptr child_; - GroupKeyMaterializer group_key_materializer_; - std::vector bindings_; - std::vector select_items_; - PredicatePtr filter_; - PredicatePtr having_; - std::unordered_map group_indexes_; - std::vector groups_; - bool sort_by_group_keys_ = false; - bool returned_ = false; -}; - -class GroupAggTopKOperator final : public Operator { - public: - GroupAggTopKOperator(std::unique_ptr child, std::vector group_keys, - const std::vector& aggregates, std::vector select_items, - PredicatePtr filter, std::vector order_by, const size_t limit) - : child_(std::move(child)), - group_key_materializer_(std::move(group_keys)), - select_items_(std::move(select_items)), - filter_(std::move(filter)), - order_by_(std::move(order_by)), - limit_(limit) { - bindings_.reserve(aggregates.size()); - - for (const auto& aggregate : aggregates) { - bindings_.push_back(GroupAggBinding{ - .aggregate = aggregate, - }); - } - } - - std::optional Next() override { - if (returned_ || limit_ == 0) { - return std::nullopt; - } - - returned_ = true; - - while (auto batch = child_->Next()) { - std::vector selected_rows; - SelectRowsMatchingPredicate(filter_, *batch, selected_rows); - - for (const size_t row : selected_rows) { - TypedGroupKey key = group_key_materializer_.Materialize(*batch, row); - - size_t group_index = 0; - const auto it = group_indexes_.find(key); - if (it == group_indexes_.end()) { - group_index = groups_.size(); - group_indexes_.emplace(key, group_index); - - groups_.push_back(GroupState{ - .key_values = std::move(key.values), - .states = CreateStates(), - .aggregate_values = {}, - .aggregate_values_finalized = false, - }); - } else { - group_index = it->second; - } - - GroupState& group = groups_[group_index]; - for (size_t i = 0; i < bindings_.size(); ++i) { - ConsumeAggRow(bindings_[i].aggregate, *batch, row, *group.states[i]); - } - } - } - - Schema schema; - schema.columns.reserve(select_items_.size()); - for (const auto& item : select_items_) { - schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); - } - - const std::vector top_groups = BuildTopGroupIndexes(); - Batch result(schema, top_groups.size()); - - for (const size_t group_index : top_groups) { - AppendGroupToBatch(groups_[group_index], result); - } - - return result; - } - - private: - struct GroupState { - std::vector key_values; - std::vector> states; - mutable std::vector aggregate_values; - mutable bool aggregate_values_finalized = false; - }; - - struct GroupAggBinding { - PlannedAgg aggregate; - }; - - std::vector> CreateStates() const { - std::vector> states; - states.reserve(bindings_.size()); - - for (const auto& binding : bindings_) { - states.push_back(CreateAggState(binding.aggregate)); - } - - return states; - } - - static void FinalizeAggregateValues(const GroupState& group) { - if (group.aggregate_values_finalized) { - return; - } - - group.aggregate_values.reserve(group.states.size()); - for (const auto& state : group.states) { - group.aggregate_values.push_back(state->Finalize()); - } - group.aggregate_values_finalized = true; - } - - bool OrdersBefore(const size_t lhs_index, const size_t rhs_index) const { - const GroupState& lhs = groups_[lhs_index]; - const GroupState& rhs = groups_[rhs_index]; - - for (const PlannedOrderBy& order : order_by_) { - const PlannedSelectItem& item = select_items_[order.result_column_index]; - std::strong_ordering comparison = std::strong_ordering::equal; - - if (item.kind == SelectItemKind::GroupKey) { - comparison = CompareGroupKeyValue(lhs.key_values[item.index], rhs.key_values[item.index]); - } else { - FinalizeAggregateValues(lhs); - FinalizeAggregateValues(rhs); - comparison = - CompareValues(item.output_type, lhs.aggregate_values[item.index], rhs.aggregate_values[item.index]); - } - - if (comparison != 0) { - return order.descending ? comparison > 0 : comparison < 0; - } - } - - return lhs_index < rhs_index; - } - - void AppendGroupToBatch(const GroupState& group, const Batch& batch) const { - FinalizeAggregateValues(group); - - for (size_t column = 0; column < select_items_.size(); ++column) { - const PlannedSelectItem& item = select_items_[column]; - if (item.kind == SelectItemKind::GroupKey) { - batch.AppendValueFromString(column, FormatGroupKeyValue(group.key_values[item.index])); - } else { - batch.AppendValueFromString(column, group.aggregate_values[item.index]); - } - } - } - - std::vector BuildTopGroupIndexes() const { - std::vector top_groups; - top_groups.reserve(limit_); - - for (size_t i = 0; i < groups_.size(); ++i) { - if (top_groups.size() < limit_) { - top_groups.push_back(i); - std::ranges::push_heap(top_groups, - [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); - continue; - } - - if (OrdersBefore(i, top_groups.front())) { - std::ranges::pop_heap(top_groups, - [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); - top_groups.back() = i; - std::ranges::push_heap(top_groups, - [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); - } - } - - std::ranges::sort(top_groups, [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); - - return top_groups; - } - - std::unique_ptr child_; - GroupKeyMaterializer group_key_materializer_; - std::vector bindings_; - std::vector select_items_; - PredicatePtr filter_; - std::vector order_by_; - size_t limit_ = 0; - std::unordered_map group_indexes_; - mutable std::vector groups_; - bool returned_ = false; -}; - -struct SortedRow { - std::vector values; - size_t ordinal = 0; -}; - -class RowOrdering { - public: - explicit RowOrdering(std::vector order_by) : order_by_(std::move(order_by)) {} - - bool operator()(const SortedRow& lhs, const SortedRow& rhs) const { - for (const auto& order : order_by_) { - const std::strong_ordering comparison = CompareValues( - order.value_type, lhs.values[order.result_column_index], rhs.values[order.result_column_index]); - if (comparison != 0) { - return order.descending ? comparison > 0 : comparison < 0; - } - } - - return lhs.ordinal < rhs.ordinal; - } - - private: - std::vector order_by_; -}; - -static std::vector MaterializeBatchRow(const Batch& batch, const size_t row) { - std::vector values; - values.reserve(batch.ColumnsCount()); - - for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { - values.push_back(batch.ColumnAt(column_index).ValueAsString(row)); - } - - return values; -} - -static Batch BuildBatchFromSortedRows(const Schema& schema, const std::vector& rows) { - Batch result(schema, rows.size()); - - for (const auto& row : rows) { - for (size_t i = 0; i < row.values.size(); ++i) { - result.AppendValueFromString(i, row.values[i]); - } - } - - return result; -} - -class ProjectionOperator final : public Operator { - public: - ProjectionOperator(std::unique_ptr child, std::vector items, Schema source_schema) - : child_(std::move(child)), items_(std::move(items)), source_schema_(std::move(source_schema)) { - for (const auto& item : items_) { - if (item.expression && item.expression->kind == ExprKind::Star) { - for (const size_t source_index : StarColumnIndexes(source_schema_)) { - schema_.columns.push_back(source_schema_.columns[source_index]); - } - } else { - schema_.columns.emplace_back(item.output_name, ColumnType::String); - } - } - } - - std::optional Next() override { - while (auto batch = child_->Next()) { - if (!star_column_indexes_initialized_) { - star_column_indexes_ = StarColumnIndexes(batch->GetSchema()); - star_column_indexes_initialized_ = true; - } - - Batch projected(schema_, batch->RowsCount()); - - size_t output_column = 0; - for (const auto& item : items_) { - if (item.expression && item.expression->kind == ExprKind::Star) { - for (const size_t source_column : star_column_indexes_) { - projected.AppendColumnRange(output_column++, batch->ColumnAt(source_column), 0, - batch->RowsCount()); - } - continue; - } - - const size_t expression_column = output_column++; - for (size_t row = 0; row < batch->RowsCount(); ++row) { - projected.AppendValueFromString(expression_column, EvalExpr(item.expression, *batch, row)); - } - } - - if (projected.RowsCount() > 0) { - return projected; - } - } - - return std::nullopt; - } - - const Schema& GetSchema() const { return schema_; } - - private: - std::unique_ptr child_; - std::vector items_; - Schema source_schema_; - Schema schema_; - std::vector star_column_indexes_; - bool star_column_indexes_initialized_ = false; -}; - -class OrderByOperator final : public Operator { - public: - OrderByOperator(std::unique_ptr child, std::vector order_by, - std::vector /*select_items*/) - : child_(std::move(child)), ordering_(std::move(order_by)) {} - - std::optional Next() override { - if (returned_) { - return std::nullopt; - } - - returned_ = true; - - std::optional schema; - std::vector rows; - size_t ordinal = 0; - - while (auto batch = child_->Next()) { - if (!schema.has_value()) { - schema = batch->GetSchema(); - } - - for (size_t row = 0; row < batch->RowsCount(); ++row) { - rows.push_back(SortedRow{ - .values = MaterializeBatchRow(*batch, row), - .ordinal = ordinal++, - }); - } - } - - if (!schema.has_value()) { - return std::nullopt; - } - - SortRows(rows); - - return BuildBatchFromSortedRows(*schema, rows); - } - - private: - void SortRows(std::vector& rows) const { std::ranges::sort(rows, ordering_); } - - std::unique_ptr child_; - RowOrdering ordering_; - bool returned_ = false; -}; - -class TopKOperator final : public Operator { - public: - TopKOperator(std::unique_ptr child, std::vector order_by, - std::vector /*select_items*/, const size_t limit) - : child_(std::move(child)), ordering_(std::move(order_by)), limit_(limit) {} - - std::optional Next() override { - if (returned_ || limit_ == 0) { - return std::nullopt; - } - - returned_ = true; - - std::optional schema; - std::vector rows; - rows.reserve(limit_); - - size_t ordinal = 0; - - while (auto batch = child_->Next()) { - if (!schema.has_value()) { - schema = batch->GetSchema(); - } - - for (size_t row = 0; row < batch->RowsCount(); ++row) { - SortedRow candidate{ - .values = MaterializeBatchRow(*batch, row), - .ordinal = ordinal++, - }; - - if (rows.size() < limit_) { - rows.push_back(std::move(candidate)); - std::ranges::push_heap(rows, ordering_); - continue; - } - - if (ordering_(candidate, rows.front())) { - std::ranges::pop_heap(rows, ordering_); - rows.back() = std::move(candidate); - std::ranges::push_heap(rows, ordering_); - } - } - } - - if (!schema.has_value()) { - return std::nullopt; - } - - std::ranges::sort(rows, ordering_); - - return BuildBatchFromSortedRows(*schema, rows); - } - - private: - std::unique_ptr child_; - RowOrdering ordering_; - size_t limit_; - bool returned_ = false; -}; - -class LimitOperator final : public Operator { - public: - LimitOperator(std::unique_ptr child, const size_t limit) : child_(std::move(child)), remaining_(limit) {} - - std::optional Next() override { - if (remaining_ == 0) { - return std::nullopt; - } - - while (auto batch = child_->Next()) { - if (batch->RowsCount() <= remaining_) { - remaining_ -= batch->RowsCount(); - - return batch; - } - - Batch limited(batch->GetSchema(), remaining_); - limited.AppendRowsRangeFromBatch(*batch, 0, remaining_); - - remaining_ = 0; - - return limited; - } - - remaining_ = 0; - - return std::nullopt; - } - - private: - std::unique_ptr child_; - size_t remaining_; -}; - -class OffsetOperator final : public Operator { - public: - OffsetOperator(std::unique_ptr child, const size_t offset) - : child_(std::move(child)), remaining_(offset) {} - - std::optional Next() override { - while (const auto batch = child_->Next()) { - if (remaining_ >= batch->RowsCount()) { - remaining_ -= batch->RowsCount(); - continue; - } - - Batch shifted(batch->GetSchema(), batch->RowsCount() - remaining_); - shifted.AppendRowsRangeFromBatch(*batch, remaining_, batch->RowsCount() - remaining_); - remaining_ = 0; - - return shifted; - } - - return std::nullopt; - } - - private: - std::unique_ptr child_; - size_t remaining_; -}; +#include "executor/operators_internal.h" std::unique_ptr BuildPlan(const PlannedQuery& planned) { if (planned.metadata_count_only) { - return std::make_unique(planned.table_path, planned.aggregates.front().output_name); + return CreateMetadataCountOperator(planned.table_path, planned.aggregates.front().output_name); } if (planned.metadata_extrema_only) { - return std::make_unique(planned.table_path, planned.aggregates); + return CreateMetadataExtremaOperator(planned.table_path, planned.aggregates); } std::unique_ptr root = - std::make_unique(planned.table_path, planned.projection_indexes, planned.filter); + CreateScanOperator(planned.table_path, planned.projection_indexes, planned.filter); if (planned.filter && planned.plain_select) { - root = std::make_unique(std::move(root), planned.filter); + root = CreateFilterOperator(std::move(root), planned.filter); } if (planned.plain_select) { bool limit_applied_by_top_k = false; - if (!planned.order_by.empty()) { - if (planned.limit.has_value() && planned.offset == 0) { - root = std::make_unique(std::move(root), planned.order_by, planned.select_items, - *planned.limit); - limit_applied_by_top_k = true; - } else { - root = std::make_unique(std::move(root), planned.order_by, planned.select_items); - } - } - if (planned.offset > 0) { - root = std::make_unique(std::move(root), planned.offset); - } - if (planned.limit.has_value() && !limit_applied_by_top_k) { - root = std::make_unique(std::move(root), *planned.limit); - } + ApplyOrderOffsetLimit(root, planned, limit_applied_by_top_k); - auto projection = - std::make_unique(std::move(root), planned.plain_select_items, planned.table_schema); - const Schema output_schema = projection->GetSchema(); - root = std::move(projection); - root = std::make_unique(std::move(root), output_schema); + const Schema output_schema = ProjectionOutputSchema(planned.plain_select_items, planned.table_schema); + root = CreateProjectionOperator(std::move(root), planned.plain_select_items, planned.table_schema); + root = CreateEnsureSchemaOperator(std::move(root), output_schema); return root; } @@ -1634,36 +32,21 @@ std::unique_ptr BuildPlan(const PlannedQuery& planned) { if (!planned.group_keys.empty()) { if (!planned.order_by.empty() && planned.limit.has_value() && planned.offset == 0 && !planned.having) { - root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, - planned.select_items, planned.filter, planned.order_by, - *planned.limit); + root = CreateGroupAggTopKOperator(std::move(root), planned.group_keys, planned.aggregates, + planned.select_items, planned.filter, planned.order_by, + *planned.limit); limit_applied_by_top_k = true; } else { - root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, - planned.select_items, planned.filter, planned.having, - planned.order_by.empty()); + root = CreateGroupAggOperator(std::move(root), planned.group_keys, planned.aggregates, + planned.select_items, planned.filter, planned.having, + planned.order_by.empty()); } } else { - root = std::make_unique(std::move(root), planned.aggregates, planned.filter); + root = CreateAggOperator(std::move(root), planned.aggregates, planned.filter); } - if (!planned.order_by.empty()) { - if (planned.limit.has_value() && planned.offset == 0) { - root = - std::make_unique(std::move(root), planned.order_by, planned.select_items, *planned.limit); - limit_applied_by_top_k = true; - } else { - root = std::make_unique(std::move(root), planned.order_by, planned.select_items); - } - } - - if (planned.offset > 0) { - root = std::make_unique(std::move(root), planned.offset); - } - - if (planned.limit.has_value() && !limit_applied_by_top_k) { - root = std::make_unique(std::move(root), *planned.limit); - } + ApplyOrderOffsetLimit(root, planned, limit_applied_by_top_k); + root = CreateEnsureSchemaOperator(std::move(root), BuildSelectOutputSchema(planned.select_items)); return root; } diff --git a/src/executor/operators_aggregate.cpp b/src/executor/operators_aggregate.cpp new file mode 100644 index 0000000..56c6620 --- /dev/null +++ b/src/executor/operators_aggregate.cpp @@ -0,0 +1,767 @@ +#include +#include +#include +#include +#include +#include + +#include "common/ascii.h" +#include "common/error.h" +#include "common/parsing.h" +#include "common/string_arena.h" +#include "executor/aggregate_function.h" +#include "executor/aggregate_state.h" +#include "executor/comparison_utils.h" +#include "executor/operators_internal.h" +#include "executor/typed_value_utils.h" + +constexpr std::string_view ExtractMinutePart = "MINUTE"; +constexpr std::string_view ExtractHourPart = "HOUR"; +constexpr int64_t MinuteMicros = 60'000'000; + +template +Schema BuildAggregateOutputSchema(const std::vector& bindings) { + Schema schema; + schema.columns.reserve(bindings.size()); + + for (const auto& binding : bindings) { + schema.columns.push_back(ColumnSchema(binding.aggregate.output_name, AggregateOutputType(binding.aggregate))); + } + + return schema; +} + +std::optional TryEvalTypedGroupKeyInt(const ExprPtr& expr, const Batch& batch, size_t row); + +bool UsesRowAggInput(const PlannedAgg& aggregate) { + return !aggregate.direct_numeric_argument && aggregate.argument_kind != AggArgumentKind::Column; +} + +void ConsumeAggRow(const PlannedAgg& aggregate, const Batch& batch, const size_t row, AggState& state) { + if (aggregate.direct_numeric_argument) { + state.ConsumeInt128(batch.ColumnAt(aggregate.column_index).ValueAsInt128(row) + + aggregate.direct_numeric_offset); + return; + } + + if (aggregate.argument_kind == AggArgumentKind::Column) { + if (aggregate.input_type != ColumnType::String) { + if (const auto typed_value = TryEvalTypedGroupKeyInt(aggregate.argument, batch, row); + typed_value.has_value()) { + state.ConsumeInt128(*typed_value); + return; + } + } + state.ConsumeValue(EvalExpr(aggregate.argument, batch, row)); + return; + } + + state.ConsumeRow(); +} + +void ConsumeAggRows(const PlannedAgg& aggregate, const Batch& batch, const std::span rows, + AggState& state) { + if (UsesRowAggInput(aggregate)) { + state.ConsumeRows(rows.size()); + return; + } + + for (const size_t row : rows) { + ConsumeAggRow(aggregate, batch, row, state); + } +} + +struct AggBinding { + PlannedAgg aggregate; + + std::unique_ptr state; +}; + +struct GroupKeyComponent { + ColumnType type = ColumnType::String; + + Int128 int_value = 0; + std::string_view string_value; +}; + +std::optional TryEvalTypedGroupKeyInt(const ExprPtr& expr, const Batch& batch, const size_t row) { + if (!expr) { + return std::nullopt; + } + + switch (expr->kind) { + case ExprKind::Column: + if (!expr->column_index_bound || expr->column_index >= batch.ColumnsCount()) { + return std::nullopt; + } + if (batch.ColumnAt(expr->column_index).Type() == ColumnType::String) { + return std::nullopt; + } + return batch.ColumnAt(expr->column_index).ValueAsInt128(row); + case ExprKind::Literal: + if (expr->literal.kind != LiteralKind::Numeric) { + return std::nullopt; + } + return ParseInt128(expr->literal.text); + case ExprKind::Binary: { + const auto lhs = TryEvalTypedGroupKeyInt(expr->left, batch, row); + const auto rhs = TryEvalTypedGroupKeyInt(expr->right, batch, row); + if (!lhs.has_value() || !rhs.has_value()) { + return std::nullopt; + } + return expr->binary_op == BinaryOp::Add ? *lhs + *rhs : *lhs - *rhs; + } + case ExprKind::Function: { + const std::string name = ToUpperAscii(expr->function_name); + + if (name == "EXTRACT" && expr->arguments.size() == 2 && expr->arguments[0] && + expr->arguments[0]->kind == ExprKind::Literal) { + const auto ts = TryEvalTypedGroupKeyInt(expr->arguments[1], batch, row); + if (!ts.has_value()) { + return std::nullopt; + } + + const std::string part = ToUpperAscii(expr->arguments[0]->literal.text); + const std::chrono::sys_time time{ + std::chrono::microseconds{static_cast(*ts)}}; + const auto day = std::chrono::floor(time); + const std::chrono::hh_mm_ss tod{time - day}; + + if (part == ExtractMinutePart) { + return tod.minutes().count(); + } + if (part == ExtractHourPart) { + return tod.hours().count(); + } + } + + if (name == "DATE_TRUNC" && expr->arguments.size() == 2 && expr->arguments[0] && + expr->arguments[0]->kind == ExprKind::Literal) { + const auto ts = TryEvalTypedGroupKeyInt(expr->arguments[1], batch, row); + if (!ts.has_value()) { + return std::nullopt; + } + + const std::string part = ToUpperAscii(NormalizeLiteralForEval(expr->arguments[0]->literal)); + if (part == ExtractMinutePart) { + const int64_t micros = static_cast(*ts); + return micros / MinuteMicros * MinuteMicros; + } + } + + return std::nullopt; + } + case ExprKind::Case: + case ExprKind::Star: + return std::nullopt; + } + + return std::nullopt; +} + +std::string FormatGroupKeyValue(const GroupKeyComponent& value) { + switch (value.type) { + case ColumnType::Boolean: + case ColumnType::Int16: + case ColumnType::Int32: + case ColumnType::Int64: + case ColumnType::Int128: + case ColumnType::Date: + case ColumnType::Timestamp: + case ColumnType::Character: + return FormatInt128Value(value.type, value.int_value); + case ColumnType::String: + return std::string(value.string_value); + } + + return {}; +} + +std::strong_ordering CompareGroupKeyValue(const GroupKeyComponent& lhs, const GroupKeyComponent& rhs) { + if (lhs.type != rhs.type) { + throw Error::InconsistentData("executor", "group key type mismatch"); + } + + if (lhs.type == ColumnType::String) { + return lhs.string_value <=> rhs.string_value; + } + + return lhs.int_value <=> rhs.int_value; +} + +bool GroupKeyValueEquals(const GroupKeyComponent& lhs, const GroupKeyComponent& rhs) { + return CompareGroupKeyValue(lhs, rhs) == 0; +} + +size_t HashCombine(const size_t seed, const size_t value) { + return seed ^ (value + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2)); +} + +size_t HashInt128(const Int128 value) { + const auto bits = static_cast(value); + const uint64_t low = static_cast(bits); + const uint64_t high = static_cast(bits >> 64); + + return HashCombine(std::hash{}(high), std::hash{}(low)); +} + +struct TypedGroupKey { + std::vector values; +}; + +struct TypedGroupKeyHash { + size_t operator()(const TypedGroupKey& key) const { + size_t hash = 0; + + for (const GroupKeyComponent& value : key.values) { + hash = HashCombine(hash, std::hash{}(static_cast(value.type))); + hash = HashCombine(hash, value.type == ColumnType::String ? StringViewHash{}(value.string_value) + : HashInt128(value.int_value)); + } + + return hash; + } +}; + +struct TypedGroupKeyEqual { + bool operator()(const TypedGroupKey& lhs, const TypedGroupKey& rhs) const { + if (lhs.values.size() != rhs.values.size()) { + return false; + } + + for (size_t i = 0; i < lhs.values.size(); ++i) { + if (!GroupKeyValueEquals(lhs.values[i], rhs.values[i])) { + return false; + } + } + + return true; + } +}; + +class GroupKeyMaterializer { + public: + explicit GroupKeyMaterializer(std::vector group_keys) : group_keys_(std::move(group_keys)) {} + + TypedGroupKey Materialize(const Batch& batch, const size_t row, StringArena& arena) const { + TypedGroupKey key; + key.values.reserve(group_keys_.size()); + + for (const auto& group_key : group_keys_) { + if (group_key.column_type != ColumnType::String) { + if (const auto typed_value = TryEvalTypedGroupKeyInt(group_key.expression, batch, row); + typed_value.has_value()) { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = *typed_value, + .string_value = {}, + }); + continue; + } + } else if (group_key.expression && group_key.expression->kind == ExprKind::Column && + group_key.expression->column_index_bound && + group_key.expression->column_index < batch.ColumnsCount()) { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = 0, + .string_value = arena.Store(batch.ColumnAt(group_key.expression->column_index).ValueAsString(row)), + }); + continue; + } + + std::string value = EvalExpr(group_key.expression, batch, row); + if (group_key.column_type == ColumnType::String) { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = 0, + .string_value = arena.Store(value), + }); + } else { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = ParseColumnValueAsInt128(group_key.column_type, value), + .string_value = {}, + }); + } + } + + return key; + } + + private: + std::vector group_keys_; +}; + +class AggOperator final : public Operator { + public: + AggOperator(std::unique_ptr child, std::vector aggregates, PredicatePtr filter) + : child_(std::move(child)), filter_(std::move(filter)) { + bindings_.reserve(aggregates.size()); + + for (auto& aggregate : aggregates) { + std::unique_ptr state = CreateAggState(aggregate); + bindings_.push_back(AggBinding{ + .aggregate = std::move(aggregate), + .state = std::move(state), + }); + } + } + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + while (auto batch = child_->Next()) { + for (size_t row = 0; row < batch->RowsCount(); ++row) { + if (!EvaluatePredicate(filter_, *batch, row)) { + continue; + } + for (auto& binding : bindings_) { + ConsumeAggRow(binding.aggregate, *batch, row, *binding.state); + } + } + } + + Batch result(BuildAggregateOutputSchema(bindings_)); + + for (size_t i = 0; i < bindings_.size(); ++i) { + result.AppendValueFromString(i, bindings_[i].state->Finalize()); + } + + return result; + } + + private: + std::unique_ptr child_; + PredicatePtr filter_; + std::vector bindings_; + bool returned_ = false; +}; + +class GroupAggOperator final : public Operator { + public: + GroupAggOperator(std::unique_ptr child, std::vector group_keys, + const std::vector& aggregates, std::vector select_items, + PredicatePtr filter, PredicatePtr having, const bool sort_by_group_keys) + : child_(std::move(child)), + group_key_materializer_(std::move(group_keys)), + select_items_(std::move(select_items)), + filter_(std::move(filter)), + having_(std::move(having)), + sort_by_group_keys_(sort_by_group_keys) { + bindings_.reserve(aggregates.size()); + + for (const auto& aggregate : aggregates) { + bindings_.push_back(GroupAggBinding{ + .aggregate = aggregate, + }); + } + } + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + while (auto batch = child_->Next()) { + for (size_t row = 0; row < batch->RowsCount(); ++row) { + if (!EvaluatePredicate(filter_, *batch, row)) { + continue; + } + + TypedGroupKey key = group_key_materializer_.Materialize(*batch, row, string_arena_); + + size_t group_index = 0; + const auto it = group_indexes_.find(key); + if (it == group_indexes_.end()) { + group_index = groups_.size(); + group_indexes_.emplace(key, group_index); + + groups_.push_back(GroupState{ + .key_values = std::move(key.values), + .states = CreateStates(), + .aggregate_values = {}, + .aggregate_int_values = {}, + .aggregate_values_finalized = false, + }); + } else { + group_index = it->second; + } + + GroupState& group = groups_[group_index]; + for (size_t i = 0; i < bindings_.size(); ++i) { + ConsumeAggRow(bindings_[i].aggregate, *batch, row, *group.states[i]); + } + } + } + + const Schema schema = BuildSelectOutputSchema(select_items_); + const std::vector output_order = BuildOutputOrder(); + Batch result(schema, groups_.size()); + + for (const size_t group_index : output_order) { + if (GroupMatchesHaving(groups_[group_index], schema)) { + AppendGroupToBatch(groups_[group_index], result); + } + } + + return result; + } + + private: + struct GroupState { + std::vector key_values; + + std::vector> states; + + mutable std::vector aggregate_values; + mutable std::vector aggregate_int_values; + + mutable bool aggregate_values_finalized = false; + }; + + struct GroupAggBinding { + PlannedAgg aggregate; + }; + + std::vector> CreateStates() const { + std::vector> states; + states.reserve(bindings_.size()); + + for (const auto& binding : bindings_) { + states.push_back(CreateAggState(binding.aggregate)); + } + + return states; + } + + void FinalizeAggregateValues(const GroupState& group) const { + if (group.aggregate_values_finalized) { + return; + } + + group.aggregate_values.reserve(group.states.size()); + group.aggregate_int_values.reserve(group.states.size()); + + for (size_t i = 0; i < group.states.size(); ++i) { + std::string value = group.states[i]->Finalize(); + const ColumnType type = AggregateOutputType(bindings_[i].aggregate); + group.aggregate_int_values.push_back(type == ColumnType::String ? 0 + : ParseColumnValueAsInt128(type, value)); + group.aggregate_values.push_back(std::move(value)); + } + + group.aggregate_values_finalized = true; + } + + void AppendGroupToBatch(const GroupState& group, const Batch& batch) const { + FinalizeAggregateValues(group); + + for (size_t column = 0; column < select_items_.size(); ++column) { + const PlannedSelectItem& item = select_items_[column]; + if (item.kind == SelectItemKind::GroupKey) { + batch.AppendValueFromString(column, FormatGroupKeyValue(group.key_values[item.index])); + } else { + batch.AppendValueFromString(column, group.aggregate_values[item.index]); + } + } + } + + bool GroupMatchesHaving(const GroupState& group, const Schema& schema) const { + if (!having_) { + return true; + } + + const Batch row(schema, 1); + AppendGroupToBatch(group, row); + + return EvaluatePredicate(having_, row, 0); + } + + std::vector BuildOutputOrder() const { + std::vector order; + order.reserve(groups_.size()); + + for (size_t i = 0; i < groups_.size(); ++i) { + order.push_back(i); + } + + if (!sort_by_group_keys_) { + return order; + } + + std::vector group_key_columns; + + for (size_t i = 0; i < select_items_.size(); ++i) { + if (select_items_[i].kind == SelectItemKind::GroupKey) { + group_key_columns.push_back(i); + } + } + + if (group_key_columns.empty()) { + return order; + } + + std::ranges::sort(order, [&](const size_t lhs_index, const size_t rhs_index) { + const GroupState& lhs = groups_[lhs_index]; + const GroupState& rhs = groups_[rhs_index]; + for (const size_t column : group_key_columns) { + const PlannedSelectItem& item = select_items_[column]; + const std::strong_ordering comparison = + CompareGroupKeyValue(lhs.key_values[item.index], rhs.key_values[item.index]); + if (comparison != 0) { + return comparison < 0; + } + } + + return false; + }); + + return order; + } + + std::unique_ptr child_; + GroupKeyMaterializer group_key_materializer_; + + std::vector bindings_; + std::vector select_items_; + + PredicatePtr filter_; + PredicatePtr having_; + + StringArena string_arena_; + std::unordered_map group_indexes_; + std::vector groups_; + + bool sort_by_group_keys_ = false; + bool returned_ = false; +}; + +class GroupAggTopKOperator final : public Operator { + public: + GroupAggTopKOperator(std::unique_ptr child, std::vector group_keys, + const std::vector& aggregates, std::vector select_items, + PredicatePtr filter, std::vector order_by, const size_t limit) + : child_(std::move(child)), + group_key_materializer_(std::move(group_keys)), + select_items_(std::move(select_items)), + filter_(std::move(filter)), + order_by_(std::move(order_by)), + limit_(limit) { + bindings_.reserve(aggregates.size()); + + for (const auto& aggregate : aggregates) { + bindings_.push_back(GroupAggBinding{ + .aggregate = aggregate, + }); + } + } + + std::optional Next() override { + if (returned_ || limit_ == 0) { + return std::nullopt; + } + + returned_ = true; + + while (auto batch = child_->Next()) { + for (size_t row = 0; row < batch->RowsCount(); ++row) { + if (!EvaluatePredicate(filter_, *batch, row)) { + continue; + } + + TypedGroupKey key = group_key_materializer_.Materialize(*batch, row, string_arena_); + + size_t group_index = 0; + const auto it = group_indexes_.find(key); + if (it == group_indexes_.end()) { + group_index = groups_.size(); + group_indexes_.emplace(key, group_index); + + groups_.push_back(GroupState{ + .key_values = std::move(key.values), + .states = CreateStates(), + .aggregate_values = {}, + .aggregate_int_values = {}, + .aggregate_values_finalized = false, + }); + } else { + group_index = it->second; + } + + GroupState& group = groups_[group_index]; + for (size_t i = 0; i < bindings_.size(); ++i) { + ConsumeAggRow(bindings_[i].aggregate, *batch, row, *group.states[i]); + } + } + } + + const Schema schema = BuildSelectOutputSchema(select_items_); + const std::vector top_groups = BuildTopGroupIndexes(); + Batch result(schema, top_groups.size()); + + for (const size_t group_index : top_groups) { + AppendGroupToBatch(groups_[group_index], result); + } + + return result; + } + + private: + struct GroupState { + std::vector key_values; + + std::vector> states; + + mutable std::vector aggregate_values; + mutable std::vector aggregate_int_values; + + mutable bool aggregate_values_finalized = false; + }; + + struct GroupAggBinding { + PlannedAgg aggregate; + }; + + std::vector> CreateStates() const { + std::vector> states; + states.reserve(bindings_.size()); + + for (const auto& binding : bindings_) { + states.push_back(CreateAggState(binding.aggregate)); + } + + return states; + } + + void FinalizeAggregateValues(const GroupState& group) const { + if (group.aggregate_values_finalized) { + return; + } + + group.aggregate_values.reserve(group.states.size()); + group.aggregate_int_values.reserve(group.states.size()); + + for (size_t i = 0; i < group.states.size(); ++i) { + std::string value = group.states[i]->Finalize(); + const ColumnType type = AggregateOutputType(bindings_[i].aggregate); + group.aggregate_int_values.push_back(type == ColumnType::String ? 0 + : ParseColumnValueAsInt128(type, value)); + group.aggregate_values.push_back(std::move(value)); + } + + group.aggregate_values_finalized = true; + } + + bool OrdersBefore(const size_t lhs_index, const size_t rhs_index) const { + const GroupState& lhs = groups_[lhs_index]; + const GroupState& rhs = groups_[rhs_index]; + + for (const PlannedOrderBy& order : order_by_) { + const PlannedSelectItem& item = select_items_[order.result_column_index]; + std::strong_ordering comparison = std::strong_ordering::equal; + + if (item.kind == SelectItemKind::GroupKey) { + comparison = CompareGroupKeyValue(lhs.key_values[item.index], rhs.key_values[item.index]); + } else { + FinalizeAggregateValues(lhs); + FinalizeAggregateValues(rhs); + comparison = item.output_type == ColumnType::String + ? lhs.aggregate_values[item.index] <=> rhs.aggregate_values[item.index] + : lhs.aggregate_int_values[item.index] <=> rhs.aggregate_int_values[item.index]; + } + + if (comparison != 0) { + return order.descending ? comparison > 0 : comparison < 0; + } + } + + return lhs_index < rhs_index; + } + + void AppendGroupToBatch(const GroupState& group, const Batch& batch) const { + FinalizeAggregateValues(group); + + for (size_t column = 0; column < select_items_.size(); ++column) { + const PlannedSelectItem& item = select_items_[column]; + if (item.kind == SelectItemKind::GroupKey) { + batch.AppendValueFromString(column, FormatGroupKeyValue(group.key_values[item.index])); + } else { + batch.AppendValueFromString(column, group.aggregate_values[item.index]); + } + } + } + + std::vector BuildTopGroupIndexes() const { + std::vector top_groups; + top_groups.reserve(limit_); + + for (size_t i = 0; i < groups_.size(); ++i) { + if (top_groups.size() < limit_) { + top_groups.push_back(i); + std::ranges::push_heap(top_groups, + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + continue; + } + + if (OrdersBefore(i, top_groups.front())) { + std::ranges::pop_heap(top_groups, + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + top_groups.back() = i; + std::ranges::push_heap(top_groups, + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + } + } + + std::ranges::sort(top_groups, [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + + return top_groups; + } + + std::unique_ptr child_; + GroupKeyMaterializer group_key_materializer_; + + std::vector bindings_; + std::vector select_items_; + + PredicatePtr filter_; + + std::vector order_by_; + size_t limit_ = 0; + + StringArena string_arena_; + std::unordered_map group_indexes_; + mutable std::vector groups_; + + bool returned_ = false; +}; + +std::unique_ptr CreateAggOperator(std::unique_ptr child, std::vector aggregates, + PredicatePtr filter) { + return std::make_unique(std::move(child), std::move(aggregates), std::move(filter)); +} + +std::unique_ptr CreateGroupAggOperator(std::unique_ptr child, + std::vector group_keys, + const std::vector& aggregates, + std::vector select_items, PredicatePtr filter, + PredicatePtr having, const bool sort_by_group_keys) { + return std::make_unique(std::move(child), std::move(group_keys), aggregates, + std::move(select_items), std::move(filter), std::move(having), + sort_by_group_keys); +} + +std::unique_ptr CreateGroupAggTopKOperator(std::unique_ptr child, + std::vector group_keys, + const std::vector& aggregates, + std::vector select_items, PredicatePtr filter, + std::vector order_by, const size_t limit) { + return std::make_unique(std::move(child), std::move(group_keys), aggregates, + std::move(select_items), std::move(filter), std::move(order_by), + limit); +} diff --git a/src/executor/operators_runtime.cpp b/src/executor/operators_runtime.cpp new file mode 100644 index 0000000..2e35cbc --- /dev/null +++ b/src/executor/operators_runtime.cpp @@ -0,0 +1,778 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "common/ascii.h" +#include "common/error.h" +#include "common/parsing.h" +#include "common/string_pattern_utils.h" +#include "executor/comparison_utils.h" +#include "executor/operators_internal.h" +#include "executor/query_utils.h" +#include "executor/typed_value_utils.h" + +constexpr std::string_view CountStarName = "COUNT(*)"; +constexpr std::string_view FallbackAggregateAlias = "c"; +constexpr std::string_view PageViewsAlias = "PageViews"; +constexpr std::string_view ExtractMinutePart = "MINUTE"; +constexpr std::string_view ExtractHourPart = "HOUR"; + +constexpr int64_t MinuteMicros = 60'000'000; + +constexpr std::string_view HttpScheme = "http://"; +constexpr std::string_view HttpsScheme = "https://"; +constexpr std::string_view WwwPrefix = "www."; +constexpr std::string_view RegexBackrefPlaceholder = "\\1"; +constexpr std::string_view RegexBackrefReplacement = "$1"; + +Int128 EvalInt(const std::string& value) { return ParseInt128(value); } + +bool IsAggregateLookupFunction(const std::string_view name) { + return name == "COUNT" || name == "SUM" || name == "AVG" || name == "MIN" || name == "MAX"; +} + +std::string FormatAggregateExprName(const ExprSpec& expr) { + const std::string name = ToUpperAscii(expr.function_name); + if (name == "COUNT" && !expr.arguments.empty() && expr.arguments.front()->kind == ExprKind::Star) { + return std::string(CountStarName); + } + + std::string out = name + "("; + + for (size_t i = 0; i < expr.arguments.size(); ++i) { + if (i > 0) { + out += ", "; + } + out += expr.arguments[i]->output_name; + } + + out += ")"; + + return out; +} + +std::optional TryResolveAggregateValue(const ExprSpec& expr, const Batch& batch, const size_t row) { + const std::string name = ToUpperAscii(expr.function_name); + if (!IsAggregateLookupFunction(name)) { + return std::nullopt; + } + + const std::string formatted_name = FormatAggregateExprName(expr); + for (const std::string_view candidate_name : + {std::string_view{formatted_name}, FallbackAggregateAlias, PageViewsAlias}) { + if (const auto column = TryFindBatchColumn(batch.GetSchema(), candidate_name); column.has_value()) { + return batch.ColumnAt(*column).ValueAsString(row); + } + } + + return std::nullopt; +} + +std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t row) { + if (const auto aggregate_value = TryResolveAggregateValue(expr, batch, row); aggregate_value.has_value()) { + return *aggregate_value; + } + + const std::string name = ToUpperAscii(expr.function_name); + + if (name == "STRLEN" || name == "LENGTH") { + return std::to_string(EvalExpr(expr.arguments.at(0), batch, row).size()); + } + + if (name == "EXTRACT") { + const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); + const int64_t ts = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); + + const std::chrono::sys_time time{std::chrono::microseconds{ts}}; + const auto day = std::chrono::floor(time); + + const std::chrono::hh_mm_ss tod{time - day}; + + if (part == ExtractMinutePart) { + return std::to_string(tod.minutes().count()); + } + if (part == ExtractHourPart) { + return std::to_string(tod.hours().count()); + } + + throw Error::Unsupported("executor", "unsupported EXTRACT part"); + } + + if (name == "DATE_TRUNC") { + const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); + int64_t micros = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); + + if (part == ExtractMinutePart) { + micros = micros / MinuteMicros * MinuteMicros; + return TimestampToString(micros); + } + + throw Error::Unsupported("executor", "unsupported DATE_TRUNC part"); + } + + if (name == "REGEXP_REPLACE") { + const std::string source = EvalExpr(expr.arguments.at(0), batch, row); + const std::string pattern_text = EvalExpr(expr.arguments.at(1), batch, row); + std::string replacement = EvalExpr(expr.arguments.at(2), batch, row); + + if (pattern_text == R"(^https?://(?:www\.)?([^/]+)/.*$)" && replacement == R"(\1)") { + std::string_view rest = source; + if (rest.starts_with(HttpScheme)) { + rest.remove_prefix(HttpScheme.size()); + } else if (rest.starts_with(HttpsScheme)) { + rest.remove_prefix(HttpsScheme.size()); + } else { + return source; + } + + if (rest.starts_with(WwwPrefix)) { + rest.remove_prefix(WwwPrefix.size()); + } + + const size_t slash = rest.find('/'); + if (slash == std::string_view::npos || slash == 0) { + return source; + } + + const std::string_view suffix = rest.substr(slash + 1); + if (suffix.find('\n') != std::string_view::npos || suffix.find('\r') != std::string_view::npos) { + return source; + } + + return std::string(rest.substr(0, slash)); + } + + if (expr.regex_replace_bound) { + return std::regex_replace(source, expr.regex_pattern, expr.regex_replacement); + } + + for (size_t pos = 0; (pos = replacement.find(RegexBackrefPlaceholder, pos)) != std::string::npos;) { + replacement.replace(pos, RegexBackrefPlaceholder.size(), RegexBackrefReplacement); + pos += RegexBackrefReplacement.size(); + } + + const std::regex pattern(pattern_text); + + return std::regex_replace(source, pattern, replacement); + } + + throw Error::Unsupported("executor", "unsupported function '" + expr.function_name + "'"); +} + +bool MatchesInt128Comparison(const Int128 lhs, const Int128 rhs, const ComparisonKind comparison) { + switch (comparison) { + case ComparisonKind::Equal: + return lhs == rhs; + case ComparisonKind::NotEqual: + return lhs != rhs; + case ComparisonKind::Less: + return lhs < rhs; + case ComparisonKind::LessOrEqual: + return lhs <= rhs; + case ComparisonKind::Greater: + return lhs > rhs; + case ComparisonKind::GreaterOrEqual: + return lhs >= rhs; + } + + return false; +} + +ValueComparison ToValueComparison(const ComparisonKind comparison) { + switch (comparison) { + case ComparisonKind::Equal: + return ValueComparison::Equal; + case ComparisonKind::NotEqual: + return ValueComparison::NotEqual; + case ComparisonKind::Less: + return ValueComparison::Less; + case ComparisonKind::LessOrEqual: + return ValueComparison::LessOrEqual; + case ComparisonKind::Greater: + return ValueComparison::Greater; + case ComparisonKind::GreaterOrEqual: + return ValueComparison::GreaterOrEqual; + } + + return ValueComparison::Equal; +} + +std::optional TryEvaluateTypedComparison(const ExprPtr& left, const ExprPtr& right, const Batch& batch, + const size_t row, const ComparisonKind comparison) { + if (!left || !right) { + return std::nullopt; + } + + if (left->kind != ExprKind::Column || right->kind != ExprKind::Literal) { + if (left->kind == ExprKind::Literal && right->kind == ExprKind::Column) { + return TryEvaluateTypedComparison(right, left, batch, row, FlipComparison(comparison)); + } + return std::nullopt; + } + + const auto column_index = TryFindBatchColumn(batch.GetSchema(), left->column.name); + if (!column_index.has_value()) { + return std::nullopt; + } + + const Column& column = batch.ColumnAt(*column_index); + if (column.Type() == ColumnType::String) { + return std::nullopt; + } + + const std::optional rhs = TryParseLiteralValueAsInt128(right->literal, column.Type()); + if (!rhs.has_value()) { + return std::nullopt; + } + + return MatchesInt128Comparison(column.ValueAsInt128(row), *rhs, comparison); +} + +ColumnType InferComparisonType(const std::string& lhs, const std::string& rhs) { + if (TryParseInt128(lhs).has_value() && TryParseInt128(rhs).has_value()) { + return ColumnType::Int128; + } + + if (TryParseDate(lhs).has_value() && TryParseDate(rhs).has_value()) { + return ColumnType::Date; + } + + if (TryParseTimestamp(lhs).has_value() && TryParseTimestamp(rhs).has_value()) { + return ColumnType::Timestamp; + } + + return ColumnType::String; +} + +struct RowRef { + size_t batch_index = 0; + size_t row = 0; + + size_t ordinal = 0; +}; + +std::strong_ordering CompareColumnRows(const Column& lhs_column, const size_t lhs_row, const Column& rhs_column, + const size_t rhs_row, const ColumnType type) { + if (type != ColumnType::String) { + return lhs_column.ValueAsInt128(lhs_row) <=> rhs_column.ValueAsInt128(rhs_row); + } + + return lhs_column.ValueAsString(lhs_row) <=> rhs_column.ValueAsString(rhs_row); +} + +class RowOrdering { + public: + RowOrdering(std::vector order_by, const std::vector* batches) + : order_by_(std::move(order_by)), batches_(batches) {} + + bool operator()(const RowRef& lhs, const RowRef& rhs) const { + const Batch& lhs_batch = batches_->at(lhs.batch_index); + const Batch& rhs_batch = batches_->at(rhs.batch_index); + + for (const auto& order : order_by_) { + const Column& lhs_column = lhs_batch.ColumnAt(order.result_column_index); + const Column& rhs_column = rhs_batch.ColumnAt(order.result_column_index); + const std::strong_ordering comparison = + CompareColumnRows(lhs_column, lhs.row, rhs_column, rhs.row, order.value_type); + if (comparison != 0) { + return order.descending ? comparison > 0 : comparison < 0; + } + } + + return lhs.ordinal < rhs.ordinal; + } + + private: + std::vector order_by_; + const std::vector* batches_ = nullptr; +}; + +Batch BuildBatchFromRowRefs(const Schema& schema, const std::vector& batches, const std::vector& rows) { + Batch result(schema, rows.size()); + + for (const auto& row : rows) { + const Batch& source = batches[row.batch_index]; + for (size_t column = 0; column < source.ColumnsCount(); ++column) { + result.AppendValueFromColumn(column, source.ColumnAt(column), row.row); + } + } + + return result; +} + +class FilterOperator final : public Operator { + public: + FilterOperator(std::unique_ptr child, PredicatePtr filter) + : child_(std::move(child)), filter_(std::move(filter)) {} + + std::optional Next() override { + while (auto batch = child_->Next()) { + std::vector selected_rows; + SelectRowsMatchingPredicate(filter_, *batch, selected_rows); + + if (selected_rows.empty()) { + continue; + } + + Batch filtered(batch->GetSchema(), selected_rows.size()); + filtered.AppendRowsSelectedFromBatch(*batch, selected_rows); + + return filtered; + } + + return std::nullopt; + } + + private: + std::unique_ptr child_; + + PredicatePtr filter_; +}; + +class EnsureSchemaOperator final : public Operator { + public: + EnsureSchemaOperator(std::unique_ptr child, Schema schema) + : child_(std::move(child)), schema_(std::move(schema)) {} + + std::optional Next() override { + if (returned_empty_) { + return std::nullopt; + } + + if (auto batch = child_->Next()) { + returned_any_ = true; + return batch; + } + + if (returned_any_) { + return std::nullopt; + } + + returned_any_ = true; + returned_empty_ = true; + + return Batch(schema_); + } + + private: + std::unique_ptr child_; + + Schema schema_; + + bool returned_any_ = false; + bool returned_empty_ = false; +}; + +class ProjectionOperator final : public Operator { + public: + ProjectionOperator(std::unique_ptr child, std::vector items, Schema source_schema) + : child_(std::move(child)), + items_(std::move(items)), + source_schema_(std::move(source_schema)), + schema_(ProjectionOutputSchema(items_, source_schema_)) {} + + std::optional Next() override { + while (auto batch = child_->Next()) { + if (!star_column_indexes_initialized_) { + star_column_indexes_ = StarColumnIndexes(batch->GetSchema()); + star_column_indexes_initialized_ = true; + } + + Batch projected(schema_, batch->RowsCount()); + + size_t output_column = 0; + for (const auto& item : items_) { + if (item.expression && item.expression->kind == ExprKind::Star) { + for (const size_t source_column : star_column_indexes_) { + projected.AppendColumnRange(output_column++, batch->ColumnAt(source_column), 0, + batch->RowsCount()); + } + continue; + } + + const size_t expression_column = output_column++; + for (size_t row = 0; row < batch->RowsCount(); ++row) { + projected.AppendValueFromString(expression_column, EvalExpr(item.expression, *batch, row)); + } + } + + if (projected.RowsCount() > 0) { + return projected; + } + } + + return std::nullopt; + } + + private: + std::unique_ptr child_; + + std::vector items_; + + Schema source_schema_; + Schema schema_; + + std::vector star_column_indexes_; + + bool star_column_indexes_initialized_ = false; +}; + +class OrderByOperator final : public Operator { + public: + OrderByOperator(std::unique_ptr child, std::vector order_by) + : child_(std::move(child)), ordering_(std::move(order_by), &batches_) {} + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + std::optional schema; + std::vector rows; + size_t ordinal = 0; + + while (auto batch = child_->Next()) { + if (!schema.has_value()) { + schema = batch->GetSchema(); + } + + const size_t batch_index = batches_.size(); + const size_t rows_count = batch->RowsCount(); + batches_.push_back(std::move(*batch)); + + for (size_t row = 0; row < rows_count; ++row) { + rows.push_back(RowRef{ + .batch_index = batch_index, + .row = row, + .ordinal = ordinal++, + }); + } + } + + if (!schema.has_value()) { + return std::nullopt; + } + + std::ranges::sort(rows, ordering_); + + return BuildBatchFromRowRefs(*schema, batches_, rows); + } + + private: + std::unique_ptr child_; + + std::vector batches_; + RowOrdering ordering_; + + bool returned_ = false; +}; + +class TopKOperator final : public Operator { + public: + TopKOperator(std::unique_ptr child, std::vector order_by, const size_t limit) + : child_(std::move(child)), ordering_(std::move(order_by), &batches_), limit_(limit) {} + + std::optional Next() override { + if (returned_ || limit_ == 0) { + return std::nullopt; + } + + returned_ = true; + + std::optional schema; + std::vector rows; + rows.reserve(limit_); + + size_t ordinal = 0; + + while (auto batch = child_->Next()) { + if (!schema.has_value()) { + schema = batch->GetSchema(); + } + + const size_t batch_index = batches_.size(); + const size_t rows_count = batch->RowsCount(); + batches_.push_back(std::move(*batch)); + + for (size_t row = 0; row < rows_count; ++row) { + RowRef candidate{ + .batch_index = batch_index, + .row = row, + .ordinal = ordinal++, + }; + + if (rows.size() < limit_) { + rows.push_back(std::move(candidate)); + std::ranges::push_heap(rows, ordering_); + continue; + } + + if (ordering_(candidate, rows.front())) { + std::ranges::pop_heap(rows, ordering_); + rows.back() = std::move(candidate); + std::ranges::push_heap(rows, ordering_); + } + } + } + + if (!schema.has_value()) { + return std::nullopt; + } + + std::ranges::sort(rows, ordering_); + + return BuildBatchFromRowRefs(*schema, batches_, rows); + } + + private: + std::unique_ptr child_; + + std::vector batches_; + RowOrdering ordering_; + + size_t limit_; + + bool returned_ = false; +}; + +class LimitOperator final : public Operator { + public: + LimitOperator(std::unique_ptr child, const size_t limit) : child_(std::move(child)), remaining_(limit) {} + + std::optional Next() override { + if (remaining_ == 0) { + return std::nullopt; + } + + while (auto batch = child_->Next()) { + if (batch->RowsCount() <= remaining_) { + remaining_ -= batch->RowsCount(); + return batch; + } + + Batch limited(batch->GetSchema(), remaining_); + limited.AppendRowsRangeFromBatch(*batch, 0, remaining_); + remaining_ = 0; + + return limited; + } + + remaining_ = 0; + + return std::nullopt; + } + + private: + std::unique_ptr child_; + size_t remaining_; +}; + +class OffsetOperator final : public Operator { + public: + OffsetOperator(std::unique_ptr child, const size_t offset) + : child_(std::move(child)), remaining_(offset) {} + + std::optional Next() override { + while (const auto batch = child_->Next()) { + if (remaining_ >= batch->RowsCount()) { + remaining_ -= batch->RowsCount(); + continue; + } + + Batch shifted(batch->GetSchema(), batch->RowsCount() - remaining_); + shifted.AppendRowsRangeFromBatch(*batch, remaining_, batch->RowsCount() - remaining_); + remaining_ = 0; + + return shifted; + } + + return std::nullopt; + } + + private: + std::unique_ptr child_; + size_t remaining_; +}; + +std::string EvalExpr(const ExprPtr& expr, const Batch& batch, const size_t row) { + switch (expr->kind) { + case ExprKind::Column: { + if (expr->column_index_bound && expr->column_index < batch.ColumnsCount() && + SameColumnName(batch.GetSchema().columns[expr->column_index].name, expr->column.name)) { + return batch.ColumnAt(expr->column_index).ValueAsString(row); + } + + const auto column = TryFindBatchColumn(batch.GetSchema(), expr->column.name); + if (!column.has_value()) { + throw Error::InvalidArgument("executor", "unknown column '" + expr->column.name + "'"); + } + + return batch.ColumnAt(*column).ValueAsString(row); + } + case ExprKind::Literal: + return NormalizeLiteralForEval(expr->literal); + case ExprKind::Binary: { + const Int128 lhs = EvalInt(EvalExpr(expr->left, batch, row)); + const Int128 rhs = EvalInt(EvalExpr(expr->right, batch, row)); + return Int128ToString(expr->binary_op == BinaryOp::Add ? lhs + rhs : lhs - rhs); + } + case ExprKind::Function: + return EvalFunction(*expr, batch, row); + case ExprKind::Case: + return EvaluatePredicate(expr->case_spec.condition, batch, row) + ? EvalExpr(expr->case_spec.then_expr, batch, row) + : EvalExpr(expr->case_spec.else_expr, batch, row); + case ExprKind::Star: + return "*"; + } + return {}; +} + +bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, const size_t row) { + if (!predicate) { + return true; + } + + switch (predicate->kind) { + case PredicateKind::And: + return EvaluatePredicate(predicate->lhs, batch, row) && EvaluatePredicate(predicate->rhs, batch, row); + case PredicateKind::Like: + return LikeMatches(EvalExpr(predicate->left, batch, row), EvalExpr(predicate->right, batch, row)); + case PredicateKind::NotLike: + return !LikeMatches(EvalExpr(predicate->left, batch, row), EvalExpr(predicate->right, batch, row)); + case PredicateKind::In: { + if (predicate->literal_in_set_bound && predicate->in_column_index < batch.ColumnsCount()) { + return predicate->literal_in_values.contains( + batch.ColumnAt(predicate->in_column_index).ValueAsString(row)); + } + + const std::string lhs = EvalExpr(predicate->left, batch, row); + + return std::ranges::any_of(predicate->values, + [&](const ExprPtr& value) { return lhs == EvalExpr(value, batch, row); }); + } + case PredicateKind::Comparison: { + if (predicate->typed_literal_comparison_bound && predicate->typed_column_index < batch.ColumnsCount()) { + return MatchesInt128Comparison(batch.ColumnAt(predicate->typed_column_index).ValueAsInt128(row), + predicate->typed_literal_value, predicate->typed_comparison); + } + + if (const std::optional result = + TryEvaluateTypedComparison(predicate->left, predicate->right, batch, row, predicate->comparison); + result.has_value()) { + return *result; + } + + const std::string lhs = EvalExpr(predicate->left, batch, row); + const std::string rhs = EvalExpr(predicate->right, batch, row); + + return MatchesComparison(InferComparisonType(lhs, rhs), lhs, rhs, predicate->comparison); + } + } + + return false; +} + +void SelectRowsMatchingPredicate(const PredicatePtr& predicate, const Batch& batch, std::vector& rows) { + rows.clear(); + rows.reserve(batch.RowsCount()); + + if (!predicate) { + for (size_t row = 0; row < batch.RowsCount(); ++row) { + rows.push_back(row); + } + return; + } + + if (predicate->kind == PredicateKind::Comparison && predicate->typed_literal_comparison_bound && + predicate->typed_column_index < batch.ColumnsCount()) { + batch.ColumnAt(predicate->typed_column_index) + .SelectRowsByInt128Comparison(predicate->typed_literal_value, + ToValueComparison(predicate->typed_comparison), rows); + return; + } + + if ((predicate->kind == PredicateKind::Like || predicate->kind == PredicateKind::NotLike) && + predicate->literal_like_pattern_bound && predicate->like_column_index < batch.ColumnsCount()) { + batch.ColumnAt(predicate->like_column_index) + .SelectRowsByLikePattern(predicate->like_pattern, predicate->like_negated, rows); + return; + } + + if (predicate->kind == PredicateKind::In && predicate->literal_in_set_bound && + predicate->in_column_index < batch.ColumnsCount()) { + batch.ColumnAt(predicate->in_column_index).SelectRowsByStringSet(predicate->literal_in_values, rows); + return; + } + + for (size_t row = 0; row < batch.RowsCount(); ++row) { + if (EvaluatePredicate(predicate, batch, row)) { + rows.push_back(row); + } + } +} + +Schema BuildSelectOutputSchema(const std::vector& select_items) { + Schema schema; + schema.columns.reserve(select_items.size()); + + for (const auto& item : select_items) { + schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); + } + + return schema; +} + +Schema ProjectionOutputSchema(const std::vector& items, const Schema& source_schema) { + Schema schema; + + for (const auto& item : items) { + if (item.expression && item.expression->kind == ExprKind::Star) { + for (const size_t source_index : StarColumnIndexes(source_schema)) { + schema.columns.push_back(source_schema.columns[source_index]); + } + } else { + schema.columns.emplace_back(item.output_name, ColumnType::String); + } + } + + return schema; +} + +std::unique_ptr CreateFilterOperator(std::unique_ptr child, PredicatePtr filter) { + return std::make_unique(std::move(child), std::move(filter)); +} + +std::unique_ptr CreateEnsureSchemaOperator(std::unique_ptr child, Schema schema) { + return std::make_unique(std::move(child), std::move(schema)); +} + +std::unique_ptr CreateProjectionOperator(std::unique_ptr child, std::vector items, + Schema source_schema) { + return std::make_unique(std::move(child), std::move(items), std::move(source_schema)); +} + +void ApplyOrderOffsetLimit(std::unique_ptr& root, const PlannedQuery& planned, bool& limit_applied_by_top_k) { + if (!planned.order_by.empty()) { + if (planned.limit.has_value() && planned.offset == 0) { + root = std::make_unique(std::move(root), planned.order_by, *planned.limit); + limit_applied_by_top_k = true; + } else { + root = std::make_unique(std::move(root), planned.order_by); + } + } + + if (planned.offset > 0) { + root = std::make_unique(std::move(root), planned.offset); + } + + if (planned.limit.has_value() && !limit_applied_by_top_k) { + root = std::make_unique(std::move(root), *planned.limit); + } +} diff --git a/src/executor/operators_scan.cpp b/src/executor/operators_scan.cpp new file mode 100644 index 0000000..0e89bc3 --- /dev/null +++ b/src/executor/operators_scan.cpp @@ -0,0 +1,234 @@ +#include +#include + +#include "common/ascii.h" +#include "common/error.h" +#include "executor/aggregate_state.h" +#include "executor/comparison_utils.h" +#include "executor/operators_internal.h" +#include "io/columnar_batch.h" +#include "io/file.h" + +class MetadataCountOperator final : public Operator { + public: + MetadataCountOperator(std::filesystem::path path, std::string output_name) + : path_(std::move(path)), output_name_(std::move(output_name)) {} + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + uint64_t rows = 0; + + const ColumnarBatchReader reader(path_); + for (const auto& row_group : reader.GetMetadata().row_groups) { + rows += row_group.row_count; + } + + Batch result(Schema{{ColumnSchema(output_name_, ColumnType::Int64)}}, 1); + result.AppendValueFromString(0, std::to_string(rows)); + + return result; + } + + private: + std::filesystem::path path_; + std::string output_name_; + + bool returned_ = false; +}; + +class MetadataExtremaOperator final : public Operator { + public: + MetadataExtremaOperator(std::filesystem::path path, std::vector aggregates) + : path_(std::move(path)), aggregates_(std::move(aggregates)) {} + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + const ColumnarBatchReader reader(path_); + const ColumnarMetadata& metadata = reader.GetMetadata(); + + Schema schema; + schema.columns.reserve(aggregates_.size()); + + std::vector> states; + states.reserve(aggregates_.size()); + + for (const PlannedAgg& aggregate : aggregates_) { + schema.columns.push_back(ColumnSchema(aggregate.output_name, AggregateOutputType(aggregate))); + states.push_back(CreateAggState(aggregate)); + } + + for (const auto& row_group : metadata.row_groups) { + for (size_t i = 0; i < aggregates_.size(); ++i) { + const PlannedAgg& aggregate = aggregates_[i]; + const auto& chunk = row_group.columns[aggregate.column_index]; + + if (!chunk.has_min_max) { + throw Error::InvalidState("executor", "metadata min/max statistics are missing", path_.string()); + } + + const std::string name = ToUpperAscii(aggregate.function->canonical_name); + states[i]->ConsumeInt128(name == "MIN" ? chunk.min_value : chunk.max_value); + } + } + + Batch result(schema); + for (size_t i = 0; i < states.size(); ++i) { + result.AppendValueFromString(i, states[i]->Finalize()); + } + + return result; + } + + private: + std::filesystem::path path_; + std::vector aggregates_; + + bool returned_ = false; +}; + +bool MatchesRowGroupComparison(const ColumnChunkMetadata& chunk, const Int128 value, const ComparisonKind comparison) { + if (!chunk.has_min_max) { + return true; + } + + switch (comparison) { + case ComparisonKind::Equal: + return chunk.min_value <= value && value <= chunk.max_value; + case ComparisonKind::NotEqual: + return chunk.min_value != value || chunk.max_value != value; + case ComparisonKind::Less: + return chunk.min_value < value; + case ComparisonKind::LessOrEqual: + return chunk.min_value <= value; + case ComparisonKind::Greater: + return chunk.max_value > value; + case ComparisonKind::GreaterOrEqual: + return chunk.max_value >= value; + } + + return true; +} + +bool MayMatchRowGroup(const PredicatePtr& predicate, const RowGroupMetadata& row_group) { + if (!predicate) { + return true; + } + + switch (predicate->kind) { + case PredicateKind::And: + return MayMatchRowGroup(predicate->lhs, row_group) && MayMatchRowGroup(predicate->rhs, row_group); + case PredicateKind::Comparison: + if (!predicate->metadata_typed_literal_comparison_bound || + predicate->metadata_typed_column_index >= row_group.columns.size()) { + return true; + } + return MatchesRowGroupComparison(row_group.columns[predicate->metadata_typed_column_index], + predicate->metadata_typed_literal_value, + predicate->metadata_typed_comparison); + case PredicateKind::In: + if (!predicate->metadata_typed_in_set_bound || + predicate->metadata_typed_in_column_index >= row_group.columns.size()) { + return true; + } + + return std::ranges::any_of(predicate->metadata_typed_in_values, [&](const Int128 value) { + return MatchesRowGroupComparison(row_group.columns[predicate->metadata_typed_in_column_index], value, + ComparisonKind::Equal); + }); + case PredicateKind::Like: + case PredicateKind::NotLike: + return true; + } + + return true; +} + +class ScanOperator final : public Operator { + public: + ScanOperator(std::filesystem::path path, std::vector projection_indexes, PredicatePtr filter) + : path_(std::move(path)), + input_(path_), + metadata_(ColumnarBatchReader(path_).GetMetadata()), + filter_(std::move(filter)) { + if (metadata_.schema.columns.empty()) { + throw Error::MalformedData("executor", "columnar schema is empty", path_.string()); + } + + projection_indexes_ = std::move(projection_indexes); + projected_schema_.columns.reserve(projection_indexes_.size()); + + for (const size_t source_index : projection_indexes_) { + if (source_index >= metadata_.schema.columns.size()) { + throw Error::OutOfRange("executor", "projection index out of range", path_.string()); + } + + projected_schema_.columns.push_back(metadata_.schema.columns[source_index]); + } + } + + std::optional Next() override { + if (next_group_ >= metadata_.row_groups.size()) { + return std::nullopt; + } + + const RowGroupMetadata* row_group = nullptr; + while (next_group_ < metadata_.row_groups.size()) { + const RowGroupMetadata& candidate = metadata_.row_groups[next_group_++]; + if (MayMatchRowGroup(filter_, candidate)) { + row_group = &candidate; + break; + } + } + + if (row_group == nullptr) { + return std::nullopt; + } + + Batch batch(projected_schema_, row_group->row_count); + + for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { + const size_t source_index = projection_indexes_[projected_index]; + const auto& chunk = row_group->columns[source_index]; + + ReadBatchColumnChunk(path_, input_, chunk, row_group->row_count, batch, projected_index); + } + + return batch; + } + + private: + std::filesystem::path path_; + InputFile input_; + + ColumnarMetadata metadata_; + std::vector projection_indexes_; + Schema projected_schema_; + + PredicatePtr filter_; + + size_t next_group_ = 0; +}; + +std::unique_ptr CreateMetadataCountOperator(std::filesystem::path path, std::string output_name) { + return std::make_unique(std::move(path), std::move(output_name)); +} + +std::unique_ptr CreateMetadataExtremaOperator(std::filesystem::path path, + std::vector aggregates) { + return std::make_unique(std::move(path), std::move(aggregates)); +} + +std::unique_ptr CreateScanOperator(std::filesystem::path path, std::vector projection_indexes, + PredicatePtr filter) { + return std::make_unique(std::move(path), std::move(projection_indexes), std::move(filter)); +} diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index a96ea07..33d6261 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -4,46 +4,14 @@ #include "common/ascii.h" #include "common/error.h" -#include "common/operator_utils.h" #include "common/parsing.h" -#include "common/planner_utils.h" +#include "executor/comparison_utils.h" +#include "executor/query_utils.h" +#include "executor/typed_value_utils.h" #include "io/columnar_batch.h" constexpr size_t SqlOrdinalBase = 1; -static bool SameExpr(const ExprPtr& lhs, const ExprPtr& rhs) { - if (!lhs || !rhs || lhs->kind != rhs->kind) { - return false; - } - - switch (lhs->kind) { - case ExprKind::Column: - return SameColumnRef(lhs->column, rhs->column); - case ExprKind::Literal: - return lhs->literal.text == rhs->literal.text && lhs->literal.kind == rhs->literal.kind; - case ExprKind::Star: - return true; - case ExprKind::Binary: - return lhs->binary_op == rhs->binary_op && SameExpr(lhs->left, rhs->left) && - SameExpr(lhs->right, rhs->right); - case ExprKind::Function: - if (!SameColumnName(lhs->function_name, rhs->function_name) || - lhs->arguments.size() != rhs->arguments.size()) { - return false; - } - for (size_t i = 0; i < lhs->arguments.size(); ++i) { - if (!SameExpr(lhs->arguments[i], rhs->arguments[i])) { - return false; - } - } - return true; - case ExprKind::Case: - return lhs->output_name == rhs->output_name; - } - - return false; -} - static ColumnType ColumnTypeOf(const Query& query, const Schema& schema, const ExprPtr& expr); static ColumnType ColumnTypeOfColumn(const Query& query, const Schema& schema, const ColumnRef& column) { @@ -334,35 +302,6 @@ static std::optional TryNumericLiteralValue(const ExprPtr& expr) { } } -static std::string NormalizeStringLiteral(const std::string_view text) { - std::string result; - - for (size_t i = 1; i + 1 < text.size(); ++i) { - if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { - result.push_back('\''); - ++i; - continue; - } - result.push_back(text[i]); - } - - return result; -} - -static std::string LiteralTextForEval(const QueryLiteral& literal) { - if (literal.kind == LiteralKind::String) { - return NormalizeStringLiteral(literal.text); - } - return literal.text; -} - -static void NormalizeRegexReplacement(std::string& replacement) { - for (size_t pos = 0; (pos = replacement.find("\\1", pos)) != std::string::npos;) { - replacement.replace(pos, 2, "$1"); - pos += 2; - } -} - static void BindRegexReplace(const ExprPtr& expr) { if (!expr || expr->kind != ExprKind::Function || !SameColumnName(expr->function_name, "REGEXP_REPLACE") || expr->arguments.size() != 3 || !expr->arguments[1] || !expr->arguments[2] || @@ -371,9 +310,9 @@ static void BindRegexReplace(const ExprPtr& expr) { } try { - std::string replacement = LiteralTextForEval(expr->arguments[2]->literal); + std::string replacement = NormalizeLiteralForEval(expr->arguments[2]->literal); NormalizeRegexReplacement(replacement); - expr->regex_pattern = std::regex(LiteralTextForEval(expr->arguments[1]->literal)); + expr->regex_pattern = std::regex(NormalizeLiteralForEval(expr->arguments[1]->literal)); expr->regex_replacement = std::move(replacement); expr->regex_replace_bound = true; } catch (const std::regex_error&) { @@ -381,44 +320,6 @@ static void BindRegexReplace(const ExprPtr& expr) { } } -static std::optional TryLiteralValueForColumnType(const QueryLiteral& literal, const ColumnType type) { - try { - if (literal.kind == LiteralKind::Numeric) { - return ParseInt128(literal.text); - } - - if (literal.kind != LiteralKind::String) { - return std::nullopt; - } - - const std::string normalized = NormalizeStringLiteral(literal.text); - switch (type) { - case ColumnType::Boolean: - return ParseBoolean(normalized) ? 1 : 0; - case ColumnType::Int16: - return ParseInt16(normalized); - case ColumnType::Int32: - return ParseInt32(normalized); - case ColumnType::Int64: - return ParseInt64(normalized); - case ColumnType::Int128: - return ParseInt128(normalized); - case ColumnType::Date: - return ParseDate(normalized); - case ColumnType::Timestamp: - return ParseTimestamp(normalized); - case ColumnType::Character: - return ParseCharacter(normalized); - case ColumnType::String: - return std::nullopt; - } - } catch (const Error&) { - return std::nullopt; - } - - return std::nullopt; -} - static void BindLiteralInSet(const Query& query_ast, const Schema& schema, const std::vector& projection_indexes, const PredicatePtr& predicate) { if (!predicate || predicate->kind != PredicateKind::In || !predicate->left || @@ -440,7 +341,7 @@ static void BindLiteralInSet(const Query& query_ast, const Schema& schema, if (!value || value->kind != ExprKind::Literal) { return; } - values.insert(LiteralTextForEval(value->literal)); + values.insert(NormalizeLiteralForEval(value->literal)); } predicate->literal_in_set_bound = true; @@ -455,7 +356,8 @@ static void BindLiteralInSet(const Query& query_ast, const Schema& schema, typed_values.reserve(predicate->values.size()); for (const ExprPtr& value : predicate->values) { - const std::optional typed_value = TryLiteralValueForColumnType(value->literal, schema.columns[source_index].type); + const std::optional typed_value = + TryParseLiteralValueAsInt128(value->literal, schema.columns[source_index].type); if (!typed_value.has_value()) { return; } @@ -488,29 +390,10 @@ static void BindLiteralLikePattern(const Query& query_ast, const Schema& schema, predicate->literal_like_pattern_bound = true; predicate->like_column_index = *projected_index; - predicate->like_pattern = LiteralTextForEval(predicate->right->literal); + predicate->like_pattern = NormalizeLiteralForEval(predicate->right->literal); predicate->like_negated = predicate->kind == PredicateKind::NotLike; } -static ComparisonKind FlipComparison(const ComparisonKind comparison) { - switch (comparison) { - case ComparisonKind::Equal: - return ComparisonKind::Equal; - case ComparisonKind::NotEqual: - return ComparisonKind::NotEqual; - case ComparisonKind::Less: - return ComparisonKind::Greater; - case ComparisonKind::LessOrEqual: - return ComparisonKind::GreaterOrEqual; - case ComparisonKind::Greater: - return ComparisonKind::Less; - case ComparisonKind::GreaterOrEqual: - return ComparisonKind::LessOrEqual; - } - - return comparison; -} - static void BindTypedLiteralComparison(const Query& query_ast, const Schema& schema, const std::vector& projection_indexes, const PredicatePtr& predicate) { if (!predicate || predicate->kind != PredicateKind::Comparison || !predicate->left || !predicate->right) { @@ -540,7 +423,7 @@ static void BindTypedLiteralComparison(const Query& query_ast, const Schema& sch } const ColumnType type = schema.columns[source_index].type; - const std::optional literal = TryLiteralValueForColumnType((*literal_expr)->literal, type); + const std::optional literal = TryParseLiteralValueAsInt128((*literal_expr)->literal, type); if (!literal.has_value()) { return; } @@ -579,26 +462,8 @@ static std::optional> TryLinearColumnExpr(const Exp return std::pair{expr->left->column, offset}; } -static bool IsNumericType(const ColumnType type) { - switch (type) { - case ColumnType::Boolean: - case ColumnType::Int16: - case ColumnType::Int32: - case ColumnType::Int64: - case ColumnType::Int128: - case ColumnType::Character: - return true; - case ColumnType::String: - case ColumnType::Date: - case ColumnType::Timestamp: - return false; - } - - return false; -} - static bool SupportsDirectTypedAggInput(const AggFuncDefinition& function, const ColumnType type) { - if (IsNumericType(type)) { + if (IsNumericColumnType(type)) { return true; } const std::string name = ToUpperAscii(function.canonical_name); @@ -766,23 +631,8 @@ PlannedQuery PlanQuery(const Query& query, const std::unordered_map effective_order_by = query.order_by; - - for (const auto& item : query.select_items) { - if (item.expression && item.expression->kind != ExprKind::Star) { - const bool already_ordered = std::ranges::any_of(effective_order_by, [&](const OrderBySpec& order) { - return SameExpr(order.item.expression, item.expression); - }); - if (!already_ordered) { - effective_order_by.push_back(OrderBySpec{ - .item = item, - .descending = false, - }); - } - } - } - for (const auto& order_item : effective_order_by) { + for (const auto& order_item : query.order_by) { size_t column_index = 0; const auto selected = FindSelectItem(query, order_item.item); const ExprPtr order_expression = diff --git a/src/common/planner_utils.cpp b/src/executor/query_utils.cpp similarity index 66% rename from src/common/planner_utils.cpp rename to src/executor/query_utils.cpp index d85657c..710db9f 100644 --- a/src/common/planner_utils.cpp +++ b/src/executor/query_utils.cpp @@ -1,4 +1,4 @@ -#include "common/planner_utils.h" +#include "executor/query_utils.h" #include @@ -41,7 +41,7 @@ void ValidateColumnQualifier(const Query& query, const ColumnRef& column) { throw Error::InvalidArgument("executor", "unknown table qualifier '" + column.qualifier + "'"); } -static std::string UnescapeSqlString(const std::string_view text) { +std::string UnescapeSqlString(const std::string_view text) { if (text.size() < SqlStringQuoteCount || text.front() != SqlStringQuote || text.back() != SqlStringQuote) { throw Error::InvalidArgument("executor", "invalid string literal"); } @@ -77,6 +77,21 @@ std::string NormalizeLiteral(const QueryLiteral& literal, const ColumnType type) } } +std::string NormalizeLiteralForEval(const QueryLiteral& literal) { + if (literal.kind != LiteralKind::String) { + return literal.text; + } + + return UnescapeSqlString(literal.text); +} + +void NormalizeRegexReplacement(std::string& replacement) { + for (size_t pos = 0; (pos = replacement.find("\\1", pos)) != std::string::npos;) { + replacement.replace(pos, 2, "$1"); + pos += 2; + } +} + bool SameColumnName(const std::string_view lhs, const std::string_view rhs) { return ToLowerAscii(lhs) == ToLowerAscii(rhs); } @@ -109,6 +124,39 @@ bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { return SameColumnName(lhs.output_name, rhs.output_name); } +bool SameExpr(const ExprPtr& lhs, const ExprPtr& rhs) { + if (!lhs || !rhs || lhs->kind != rhs->kind) { + return false; + } + + switch (lhs->kind) { + case ExprKind::Column: + return SameColumnRef(lhs->column, rhs->column); + case ExprKind::Literal: + return lhs->literal.text == rhs->literal.text && lhs->literal.kind == rhs->literal.kind; + case ExprKind::Star: + return true; + case ExprKind::Binary: + return lhs->binary_op == rhs->binary_op && SameExpr(lhs->left, rhs->left) && + SameExpr(lhs->right, rhs->right); + case ExprKind::Function: + if (!SameColumnName(lhs->function_name, rhs->function_name) || + lhs->arguments.size() != rhs->arguments.size()) { + return false; + } + for (size_t i = 0; i < lhs->arguments.size(); ++i) { + if (!SameExpr(lhs->arguments[i], rhs->arguments[i])) { + return false; + } + } + return true; + case ExprKind::Case: + return lhs->output_name == rhs->output_name; + } + + return false; +} + std::vector StarColumnIndexes(const Schema& schema) { std::vector clickbench_subset; @@ -134,3 +182,12 @@ std::vector StarColumnIndexes(const Schema& schema) { return all; } + +std::optional TryFindBatchColumn(const Schema& schema, const std::string_view name) { + for (size_t i = 0; i < schema.columns.size(); ++i) { + if (SameColumnName(schema.columns[i].name, name)) { + return i; + } + } + return std::nullopt; +} diff --git a/src/executor/typed_value_utils.cpp b/src/executor/typed_value_utils.cpp new file mode 100644 index 0000000..6e6d243 --- /dev/null +++ b/src/executor/typed_value_utils.cpp @@ -0,0 +1,107 @@ +#include "executor/typed_value_utils.h" + +#include + +#include "common/error.h" +#include "common/parsing.h" +#include "executor/query_utils.h" + +bool IsNumericColumnType(const ColumnType type) { + switch (type) { + case ColumnType::Boolean: + case ColumnType::Int16: + case ColumnType::Int32: + case ColumnType::Int64: + case ColumnType::Int128: + case ColumnType::Character: + return true; + case ColumnType::String: + case ColumnType::Date: + case ColumnType::Timestamp: + return false; + } + + return false; +} + +Int128 ParseColumnValueAsInt128(const ColumnType type, const std::string_view value) { + switch (type) { + case ColumnType::Boolean: + return ParseBoolean(value) ? 1 : 0; + case ColumnType::Int16: + return ParseInt16(value); + case ColumnType::Int32: + return ParseInt32(value); + case ColumnType::Int64: + return ParseInt64(value); + case ColumnType::Int128: + return ParseInt128(value); + case ColumnType::Date: + return ParseDate(value); + case ColumnType::Timestamp: + return ParseTimestamp(value); + case ColumnType::Character: + return ParseCharacter(value); + case ColumnType::String: + break; + } + + throw Error::Unsupported("executor", "unsupported typed value conversion"); +} + +std::optional TryParseLiteralValueAsInt128(const QueryLiteral& literal, const ColumnType type) { + try { + if (literal.kind == LiteralKind::Numeric) { + return ParseInt128(literal.text); + } + + if (literal.kind != LiteralKind::String) { + return std::nullopt; + } + + return ParseColumnValueAsInt128(type, NormalizeLiteralForEval(literal)); + } catch (const Error&) { + return std::nullopt; + } +} + +std::string FormatInt128Value(const ColumnType type, const Int128 value) { + switch (type) { + case ColumnType::Boolean: + return BooleanToString(value != 0); + case ColumnType::Int16: + case ColumnType::Int32: + case ColumnType::Int64: + case ColumnType::Int128: + return Int128ToString(value); + case ColumnType::Date: + return DateToString(static_cast(value)); + case ColumnType::Timestamp: + return TimestampToString(static_cast(value)); + case ColumnType::Character: + return std::string(1, static_cast(value)); + case ColumnType::String: + break; + } + + throw Error::Unsupported("executor", "unsupported typed value formatting"); +} + +ComparisonKind FlipComparison(const ComparisonKind comparison) { + switch (comparison) { + case ComparisonKind::Equal: + return ComparisonKind::Equal; + case ComparisonKind::NotEqual: + return ComparisonKind::NotEqual; + case ComparisonKind::Less: + return ComparisonKind::Greater; + case ComparisonKind::LessOrEqual: + return ComparisonKind::GreaterOrEqual; + case ComparisonKind::Greater: + return ComparisonKind::Less; + case ComparisonKind::GreaterOrEqual: + return ComparisonKind::LessOrEqual; + } + + return comparison; +} diff --git a/src/io/columnar_batch.cpp b/src/io/columnar_batch.cpp index e38a389..25c7f6b 100644 --- a/src/io/columnar_batch.cpp +++ b/src/io/columnar_batch.cpp @@ -3,12 +3,15 @@ #include #include #include +#include #include #include #include #include +#include #include "common/error.h" +#include "io/compression.h" #include "io/stream.h" static constexpr std::string_view ColumnarMagic = "CLMN"; @@ -53,6 +56,42 @@ static void PopulateChunkMinMax(const Column& column, ColumnChunkMetadata& chunk chunk.max_value = max_value; } +static std::vector SerializeColumn(const Column& column) { + std::ostringstream buffer(std::ios::binary); + column.WriteTo(buffer); + const std::string bytes = std::move(buffer).str(); + return std::vector(bytes.begin(), bytes.end()); +} + +static ColumnChunkMetadata WriteColumnChunk(const std::filesystem::path& path, std::ofstream& out, const Column& column, + const Compression compression) { + const std::vector uncompressed = SerializeColumn(column); + const std::vector compressed = Compress(uncompressed, compression); + + const std::span payload = compressed.size() < uncompressed.size() + ? std::span(compressed) + : std::span(uncompressed); + const Compression stored_compression = compressed.size() < uncompressed.size() ? compression : Compression::None; + const uint64_t offset = TellWrite(path, out); + + if (!payload.empty()) { + out.write(reinterpret_cast(payload.data()), static_cast(payload.size())); + if (!out) { + throw Error::PathIo("io", path, "write file"); + } + } + + ColumnChunkMetadata chunk; + chunk.offset = offset; + chunk.compressed_size = payload.size(); + chunk.uncompressed_size = uncompressed.size(); + chunk.compression = stored_compression; + + PopulateChunkMinMax(column, chunk); + + return chunk; +} + ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::path& path) { const auto file_metadata = GetFileMetadata(path); @@ -106,14 +145,15 @@ std::optional ColumnarBatchReader::ReadNext() { for (size_t col = 0; col < batch.ColumnsCount(); ++col) { const auto& chunk = row_group_columns[col]; - batch.ReadColumnFrom(col, input_.StreamAt(chunk.offset), row_count, chunk.size); + ReadBatchColumnChunk(path_, input_, chunk, row_count, batch, col); } return batch; } -ColumnarBatchWriter::ColumnarBatchWriter(const std::filesystem::path& path, Schema schema) - : path_(path), out_(OpenOutputFile(path)) { +ColumnarBatchWriter::ColumnarBatchWriter(const std::filesystem::path& path, Schema schema, + const Compression compression) + : path_(path), out_(OpenOutputFile(path)), compression_(compression) { if (schema.columns.empty()) { throw Error::InvalidArgument("io", "schema has no columns", path.string()); } @@ -145,15 +185,8 @@ void ColumnarBatchWriter::Write(const Batch& batch) { group.columns.reserve(batch.ColumnsCount()); for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { - const uint64_t offset = TellWrite(path_, out_); const Column& column = batch.ColumnAt(column_index); - column.WriteTo(out_); - - const uint64_t end = TellWrite(path_, out_); - - ColumnChunkMetadata chunk{offset, end - offset}; - PopulateChunkMinMax(column, chunk); - group.columns.push_back(chunk); + group.columns.push_back(WriteColumnChunk(path_, out_, column, compression_)); } metadata_.row_groups.push_back(std::move(group)); @@ -161,6 +194,8 @@ void ColumnarBatchWriter::Write(const Batch& batch) { void ColumnarBatchWriter::Flush() { FlushWrite(path_, out_); } +void ColumnarBatchWriter::Finalize() & { std::move(*this).Finalize(); } + void ColumnarBatchWriter::Finalize() && { if (finalized_) { throw Error::InvalidState("io", "writer already finalized", path_.string()); @@ -178,3 +213,26 @@ void ColumnarBatchWriter::Finalize() && { finalized_ = true; } + +std::vector ReadColumnChunk(const std::filesystem::path& path, InputFile& input, + const ColumnChunkMetadata& chunk) { + const std::string raw = input.ReadStringAt(chunk.offset, chunk.compressed_size); + const std::span raw_bytes(reinterpret_cast(raw.data()), raw.size()); + + if (chunk.compression == Compression::None) { + if (chunk.compressed_size != chunk.uncompressed_size) { + throw Error::MalformedData("io", "uncompressed chunk size mismatch", path.string()); + } + return std::vector(raw_bytes.begin(), raw_bytes.end()); + } + + return Decompress(raw_bytes, chunk.compression, chunk.uncompressed_size); +} + +void ReadBatchColumnChunk(const std::filesystem::path& path, InputFile& input, const ColumnChunkMetadata& chunk, + const uint32_t row_count, const Batch& batch, const size_t column_index) { + const std::vector payload = ReadColumnChunk(path, input, chunk); + const std::string bytes(payload.begin(), payload.end()); + std::istringstream stream(bytes, std::ios::binary); + batch.ReadColumnFrom(column_index, stream, row_count, chunk.uncompressed_size); +} diff --git a/src/io/compression.cpp b/src/io/compression.cpp new file mode 100644 index 0000000..ec39b62 --- /dev/null +++ b/src/io/compression.cpp @@ -0,0 +1,97 @@ +#include "io/compression.h" + +#include + +#include +#include +#include + +#include "common/error.h" + +const char* CompressionName(const Compression compression) { + switch (compression) { + case Compression::None: + return "none"; + case Compression::Lz4: + return "lz4"; + } + + throw Error::InvalidArgument("compression", "unknown compression codec"); +} + +Compression CompressionFromName(const std::string_view name) { + if (name == "none") { + return Compression::None; + } + if (name == "lz4") { + return Compression::Lz4; + } + + throw Error::InvalidArgument("compression", "unsupported compression codec: " + std::string(name)); +} + +std::vector Compress(const std::span input, const Compression compression) { + switch (compression) { + case Compression::None: + return std::vector(input.begin(), input.end()); + case Compression::Lz4: + break; + } + + if (input.size() > static_cast(std::numeric_limits::max())) { + throw Error::Overflow("compression", "input too large for lz4"); + } + + const int upper_bound = LZ4_compressBound(static_cast(input.size())); + if (upper_bound <= 0) { + throw Error::InvalidState("compression", "lz4 upper bound failed"); + } + + std::vector output(static_cast(upper_bound)); + const int compressed_size = + LZ4_compress_default(reinterpret_cast(input.data()), reinterpret_cast(output.data()), + static_cast(input.size()), upper_bound); + + if (compressed_size <= 0) { + throw Error::Io("compression", "lz4 compression failed"); + } + + output.resize(static_cast(compressed_size)); + + return output; +} + +std::vector Decompress(const std::span input, const Compression compression, + const uint64_t uncompressed_size) { + switch (compression) { + case Compression::None: + if (input.size() != uncompressed_size) { + throw Error::MalformedData("compression", "uncompressed chunk size mismatch"); + } + return std::vector(input.begin(), input.end()); + case Compression::Lz4: + break; + } + + if (input.size() > static_cast(std::numeric_limits::max())) { + throw Error::Overflow("compression", "compressed input too large for lz4"); + } + if (uncompressed_size > static_cast(std::numeric_limits::max())) { + throw Error::Overflow("compression", "uncompressed output too large for lz4"); + } + + std::vector output(uncompressed_size); + const int decompressed_size = + LZ4_decompress_safe(reinterpret_cast(input.data()), reinterpret_cast(output.data()), + static_cast(input.size()), static_cast(uncompressed_size)); + + if (decompressed_size < 0) { + throw Error::MalformedData("compression", "lz4 decompression failed"); + } + + if (static_cast(decompressed_size) != uncompressed_size) { + throw Error::MalformedData("compression", "lz4 decompressed size mismatch"); + } + + return output; +} diff --git a/src/model/column_string.cpp b/src/model/column_string.cpp index e2b0e55..82ce4b1 100644 --- a/src/model/column_string.cpp +++ b/src/model/column_string.cpp @@ -5,7 +5,7 @@ #include #include "common/error.h" -#include "common/operator_utils.h" +#include "common/string_pattern_utils.h" #include "io/stream.h" constexpr char EncodedValueSeparator = ':'; diff --git a/src/model/metadata.cpp b/src/model/metadata.cpp index 9b1dd59..ff9d561 100644 --- a/src/model/metadata.cpp +++ b/src/model/metadata.cpp @@ -1,11 +1,15 @@ #include "model/metadata.h" +#include #include +#include #include #include "common/error.h" #include "io/stream.h" +static constexpr std::array MetadataMagic = {'C', 'M', 'D', '2'}; + static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { switch (static_cast(type_byte)) { case ColumnType::Int64: @@ -23,7 +27,17 @@ static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { throw Error::MalformedData("model", "unknown column type in metadata"); } -ColumnarMetadata ReadMetadata(std::istream& in) { +static Compression CompressionFromByte(const uint8_t compression_byte) { + switch (static_cast(compression_byte)) { + case Compression::None: + case Compression::Lz4: + return static_cast(compression_byte); + } + + throw Error::MalformedData("model", "unknown compression codec in metadata"); +} + +static ColumnarMetadata ReadLegacyMetadata(std::istream& in) { ColumnarMetadata metadata; const uint32_t column_count = ReadStream(in); @@ -58,7 +72,9 @@ ColumnarMetadata ReadMetadata(std::istream& in) { for (uint32_t col = 0; col < columns_in_group; ++col) { ColumnChunkMetadata chunk; chunk.offset = ReadStream(in); - chunk.size = ReadStream(in); + chunk.compressed_size = ReadStream(in); + chunk.uncompressed_size = chunk.compressed_size; + chunk.compression = Compression::None; group.columns.push_back(chunk); } @@ -85,11 +101,99 @@ ColumnarMetadata ReadMetadata(std::istream& in) { return metadata; } +static ColumnarMetadata ReadV2Metadata(std::istream& in) { + ColumnarMetadata metadata; + + const uint32_t version = ReadStream(in); + if (version != 2) { + throw Error::MalformedData("model", "unsupported metadata version"); + } + + const uint32_t column_count = ReadStream(in); + metadata.schema.columns.reserve(column_count); + + for (uint32_t i = 0; i < column_count; ++i) { + const uint32_t name_size = ReadStream(in); + std::string name(name_size, '\0'); + + ReadBytes(in, name.data(), name.size()); + + const uint8_t type_byte = ReadStream(in); + const ColumnType type = ColumnTypeFromByte(type_byte); + + metadata.schema.columns.push_back(ColumnSchema{std::move(name), type}); + } + + const uint32_t row_group_count = ReadStream(in); + metadata.row_groups.reserve(row_group_count); + + for (uint32_t i = 0; i < row_group_count; ++i) { + RowGroupMetadata group; + group.row_count = ReadStream(in); + + const uint32_t columns_in_group = ReadStream(in); + if (columns_in_group != column_count) { + throw Error::InconsistentData("model", "row group column count mismatch"); + } + + group.columns.reserve(columns_in_group); + + for (uint32_t col = 0; col < columns_in_group; ++col) { + ColumnChunkMetadata chunk; + chunk.offset = ReadStream(in); + chunk.compressed_size = ReadStream(in); + chunk.uncompressed_size = ReadStream(in); + chunk.compression = CompressionFromByte(ReadStream(in)); + + group.columns.push_back(chunk); + } + + metadata.row_groups.push_back(std::move(group)); + } + + for (auto& row_group : metadata.row_groups) { + for (auto& column : row_group.columns) { + column.has_min_max = ReadStream(in) != 0; + if (!column.has_min_max) { + continue; + } + + column.min_value = ReadStream(in); + column.max_value = ReadStream(in); + } + } + + if (in.peek() != std::istream::traits_type::eof()) { + throw Error::MalformedData("model", "unexpected trailing bytes in metadata"); + } + + return metadata; +} + +ColumnarMetadata ReadMetadata(std::istream& in) { + const std::streampos start = in.tellg(); + if (start != std::streampos(-1)) { + char magic[MetadataMagic.size()]; + in.read(magic, sizeof(magic)); + if (in.gcount() == static_cast(MetadataMagic.size()) && + std::equal(std::begin(magic), std::end(magic), MetadataMagic.begin())) { + return ReadV2Metadata(in); + } + + in.clear(); + in.seekg(start); + } + + return ReadLegacyMetadata(in); +} + void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { if (metadata.schema.columns.size() > std::numeric_limits::max()) { throw Error::Overflow("model", "too many columns for metadata"); } + WriteBytes(out, {MetadataMagic.data(), MetadataMagic.size()}); + WriteStream(out, 2); WriteStream(out, metadata.schema.columns.size()); for (const auto& [name, type] : metadata.schema.columns) { @@ -114,7 +218,9 @@ void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { for (const auto& column : columns) { WriteStream(out, column.offset); - WriteStream(out, column.size); + WriteStream(out, column.compressed_size); + WriteStream(out, column.uncompressed_size); + WriteStream(out, static_cast(column.compression)); } } diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp index 3bb0a9e..9902a56 100644 --- a/tests/test_columnar.cpp +++ b/tests/test_columnar.cpp @@ -1,16 +1,20 @@ #include +#include #include #include +#include "common/error.h" +#include "common/parsing.h" #include "convert/csv_columnar.h" #include "gtest/gtest.h" #include "io/columnar_batch.h" +#include "io/compression.h" #include "io/csv.h" #include "io/file.h" +#include "io/stream.h" +#include "model/metadata.h" #include "model/schema.h" #include "model/schema_csv.h" -#include "common/error.h" -#include "common/parsing.h" #include "testing/temp_file.h" TEST(columnar, csv_to_columnar_and_back) { @@ -96,10 +100,18 @@ TEST(columnar, metadata_offsets_and_sizes) { return total; }; - EXPECT_EQ(row_groups[0].columns[0].size, sizeof(int64_t) * 2u); - EXPECT_EQ(row_groups[0].columns[1].size, string_chunk_size(0, 2)); - EXPECT_EQ(row_groups[1].columns[0].size, sizeof(int64_t) * 1u); - EXPECT_EQ(row_groups[1].columns[1].size, string_chunk_size(2, 1)); + EXPECT_EQ(row_groups[0].columns[0].compressed_size, sizeof(int64_t) * 2u); + EXPECT_EQ(row_groups[0].columns[0].uncompressed_size, sizeof(int64_t) * 2u); + EXPECT_EQ(row_groups[0].columns[0].compression, Compression::None); + EXPECT_EQ(row_groups[0].columns[1].compressed_size, string_chunk_size(0, 2)); + EXPECT_EQ(row_groups[0].columns[1].uncompressed_size, string_chunk_size(0, 2)); + EXPECT_EQ(row_groups[0].columns[1].compression, Compression::None); + EXPECT_EQ(row_groups[1].columns[0].compressed_size, sizeof(int64_t) * 1u); + EXPECT_EQ(row_groups[1].columns[0].uncompressed_size, sizeof(int64_t) * 1u); + EXPECT_EQ(row_groups[1].columns[0].compression, Compression::None); + EXPECT_EQ(row_groups[1].columns[1].compressed_size, string_chunk_size(2, 1)); + EXPECT_EQ(row_groups[1].columns[1].uncompressed_size, string_chunk_size(2, 1)); + EXPECT_EQ(row_groups[1].columns[1].compression, Compression::None); EXPECT_TRUE(row_groups[0].columns[0].has_min_max); EXPECT_EQ(row_groups[0].columns[0].min_value, 1); EXPECT_EQ(row_groups[0].columns[0].max_value, 2); @@ -113,7 +125,7 @@ TEST(columnar, metadata_offsets_and_sizes) { for (const auto& row_group : row_groups) { for (const auto& column : row_group.columns) { EXPECT_EQ(column.offset, expected_offset); - expected_offset += column.size; + expected_offset += column.compressed_size; } } @@ -142,15 +154,8 @@ TEST(columnar, mixed_supported_types_roundtrip) { const TempFile data_out("data_mixed_out"); const std::vector> schema_rows = { - {"flag", "bool"}, - {"small", "int16"}, - {"medium", "int32"}, - {"big", "int64"}, - {"huge", "int128"}, - {"name", "string"}, - {"birth_date", "date"}, - {"created_at", "timestamp"}, - {"grade", "char"}, + {"flag", "bool"}, {"small", "int16"}, {"medium", "int32"}, {"big", "int64"}, {"huge", "int128"}, + {"name", "string"}, {"birth_date", "date"}, {"created_at", "timestamp"}, {"grade", "char"}, }; const std::vector> data_rows = { @@ -186,18 +191,17 @@ TEST(columnar, infer_schema_from_csv_is_convert_compatible) { WriteRows(data_in.Path(), data_rows); WriteSchemaCsv(schema_out.Path(), InferSchemaCsv(data_in.Path())); - EXPECT_EQ(ReadRows(schema_out.Path()), - (std::vector>{ - {"column_1", "bool"}, - {"column_2", "int16"}, - {"column_3", "int32"}, - {"column_4", "int64"}, - {"column_5", "int128"}, - {"column_6", "string"}, - {"column_7", "date"}, - {"column_8", "timestamp"}, - {"column_9", "char"}, - })); + EXPECT_EQ(ReadRows(schema_out.Path()), (std::vector>{ + {"column_1", "bool"}, + {"column_2", "int16"}, + {"column_3", "int32"}, + {"column_4", "int64"}, + {"column_5", "int128"}, + {"column_6", "string"}, + {"column_7", "date"}, + {"column_8", "timestamp"}, + {"column_9", "char"}, + })); ConvertCsvToColumnar(schema_out.Path(), data_in.Path(), columnar_file.Path(), 1); ConvertColumnarToCsv(columnar_file.Path(), schema_out.Path(), data_out.Path()); @@ -240,3 +244,79 @@ TEST(columnar, metadata_records_minmax_stats_for_date_columns) { EXPECT_EQ(DateToString(static_cast(row_groups[1].columns[0].min_value)), "2024-01-01"); EXPECT_EQ(DateToString(static_cast(row_groups[1].columns[0].max_value)), "2024-01-09"); } + +TEST(columnar, lz4_compressed_roundtrip) { + const TempFile schema_in("schema_lz4_in"); + const TempFile data_in("data_lz4_in"); + const TempFile columnar_file("columnar_lz4"); + const TempFile data_out("data_lz4_out"); + const TempFile schema_out("schema_lz4_out"); + + WriteRows(schema_in.Path(), {{"id", "int64"}, {"payload", "string"}}); + + std::vector> data_rows; + for (int i = 0; i < 256; ++i) { + data_rows.push_back({std::to_string(i), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + } + WriteRows(data_in.Path(), data_rows); + + ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 256, Compression::Lz4); + ConvertColumnarToCsv(columnar_file.Path(), schema_out.Path(), data_out.Path()); + + ColumnarBatchReader reader(columnar_file.Path()); + const auto& row_group = reader.GetMetadata().row_groups.at(0); + ASSERT_EQ(row_group.columns.size(), 2u); + EXPECT_EQ(ReadRows(data_out.Path()), data_rows); + EXPECT_EQ(ReadRows(schema_out.Path()), + (std::vector>{{"id", "int64"}, {"payload", "string"}})); + EXPECT_EQ(row_group.columns[1].compression, Compression::Lz4); + EXPECT_LT(row_group.columns[1].compressed_size, row_group.columns[1].uncompressed_size); +} + +TEST(columnar, lz4_falls_back_to_none_when_chunk_does_not_shrink) { + const TempFile schema_in("schema_lz4_fallback_in"); + const TempFile data_in("data_lz4_fallback_in"); + const TempFile columnar_file("columnar_lz4_fallback"); + + WriteRows(schema_in.Path(), {{"value", "int64"}}); + WriteRows(data_in.Path(), {{"42"}}); + + ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 1, Compression::Lz4); + + const ColumnarBatchReader reader(columnar_file.Path()); + const auto& chunk = reader.GetMetadata().row_groups.at(0).columns.at(0); + EXPECT_EQ(chunk.compression, Compression::None); + EXPECT_EQ(chunk.compressed_size, sizeof(int64_t)); + EXPECT_EQ(chunk.uncompressed_size, sizeof(int64_t)); +} + +TEST(columnar, read_legacy_metadata_without_compression_fields) { + std::ostringstream out(std::ios::binary); + + WriteStream(out, 1); + WriteStream(out, 2); + WriteBytes(out, "id"); + WriteStream(out, static_cast(ColumnType::Int64)); + WriteStream(out, 1); + WriteStream(out, 3); + WriteStream(out, 1); + WriteStream(out, 10); + WriteStream(out, 24); + WriteStream(out, 1); + WriteStream(out, 7); + WriteStream(out, 9); + + std::istringstream in(out.str(), std::ios::binary); + const ColumnarMetadata metadata = ReadMetadata(in); + + ASSERT_EQ(metadata.schema.columns.size(), 1u); + ASSERT_EQ(metadata.row_groups.size(), 1u); + const auto& chunk = metadata.row_groups[0].columns[0]; + EXPECT_EQ(chunk.offset, 10u); + EXPECT_EQ(chunk.compressed_size, 24u); + EXPECT_EQ(chunk.uncompressed_size, 24u); + EXPECT_EQ(chunk.compression, Compression::None); + EXPECT_TRUE(chunk.has_min_max); + EXPECT_EQ(chunk.min_value, 7); + EXPECT_EQ(chunk.max_value, 9); +} diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index f48594c..3dd9b80 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -261,6 +261,57 @@ TEST(executor, rejects_unknown_aggregate_functions) { EXPECT_EQ(result.error().GetCode(), Error::Code::Unsupported); } +TEST(executor, supports_group_by_computed_numeric_and_timestamp_keys) { + const TempFile schema_file("executor_computed_schema"); + const TempFile data_file("executor_computed_data"); + const TempFile columnar_file("executor_computed_columnar"); + + WriteRows(schema_file.Path(), { + {"ClientIP", "int64"}, + {"EventTime", "timestamp"}, + }); + WriteRows(data_file.Path(), { + {"10", "2024-01-01 12:03:11"}, + {"11", "2024-01-01 12:03:49"}, + {"10", "2024-01-01 12:04:00"}, + }); + + ConvertCsvToColumnar(schema_file.Path(), data_file.Path(), columnar_file.Path(), 2); + + Executor executor; + executor.RegisterTable("hits", columnar_file.Path()); + + { + auto result = executor.Execute( + "SELECT ClientIP - 1, COUNT(*) AS c FROM hits GROUP BY ClientIP - 1 ORDER BY c DESC, ClientIP - 1 ASC;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); + EXPECT_EQ(BatchRows(result.value()), (std::vector>{ + {"9", "2"}, + {"10", "1"}, + })); + } + + { + auto result = executor.Execute( + "SELECT extract(minute FROM EventTime) AS m, COUNT(*) AS c FROM hits GROUP BY m ORDER BY m ASC;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); + EXPECT_EQ(BatchRows(result.value()), (std::vector>{ + {"3", "2"}, + {"4", "1"}, + })); + } + + { + auto result = executor.Execute( + "SELECT DATE_TRUNC('minute', EventTime) AS m, COUNT(*) AS c FROM hits GROUP BY m ORDER BY m ASC;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); + EXPECT_EQ(BatchRows(result.value()), (std::vector>{ + {"2024-01-01 12:03:00", "2"}, + {"2024-01-01 12:04:00", "1"}, + })); + } +} + TEST(executor, prunes_row_groups_for_typed_where_filters_before_reading_chunks) { const TempFile schema_file("executor_prune_schema"); const TempFile data_file("executor_prune_data"); @@ -301,11 +352,14 @@ TEST(executor, prunes_row_groups_for_typed_where_filters_before_reading_chunks) std::memcpy(&metadata_size, bytes.data() + footer_offset, sizeof(metadata_size)); const size_t metadata_offset = footer_offset - metadata_size; - bytes.erase(bytes.begin() + static_cast(metadata_offset), bytes.begin() + static_cast(footer_offset)); - bytes.insert(bytes.begin() + static_cast(metadata_offset), metadata_blob.begin(), metadata_blob.end()); + bytes.erase(bytes.begin() + static_cast(metadata_offset), + bytes.begin() + static_cast(footer_offset)); + bytes.insert(bytes.begin() + static_cast(metadata_offset), metadata_blob.begin(), + metadata_blob.end()); const uint64_t patched_metadata_size = metadata_blob.size(); - std::memcpy(bytes.data() + metadata_offset + metadata_blob.size(), &patched_metadata_size, sizeof(patched_metadata_size)); + std::memcpy(bytes.data() + metadata_offset + metadata_blob.size(), &patched_metadata_size, + sizeof(patched_metadata_size)); WriteFileBytes(columnar_file.Path(), bytes);