From b64754a004351c7ff45c97949b10f8fe418ad99a Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 03:27:22 +0300 Subject: [PATCH 01/13] fix: small fixes --- include/common/planner_utils.h | 1 + include/io/columnar_batch.h | 1 + src/common/planner_utils.cpp | 2 +- src/convert/csv_columnar.cpp | 2 +- src/executor/executor.cpp | 2 +- src/executor/operators.cpp | 149 ++++++++++++++++----------------- src/executor/query_planner.cpp | 19 +---- src/io/columnar_batch.cpp | 2 + 8 files changed, 81 insertions(+), 97 deletions(-) diff --git a/include/common/planner_utils.h b/include/common/planner_utils.h index 1e5bc12..a4a59e9 100644 --- a/include/common/planner_utils.h +++ b/include/common/planner_utils.h @@ -9,6 +9,7 @@ size_t FindColumnIndex(const Schema& schema, std::string_view column_name); void ValidateColumnQualifier(const Query& query, const ColumnRef& column); +std::string UnescapeSqlString(std::string_view text); std::string NormalizeLiteral(const QueryLiteral& literal, ColumnType type); bool SameColumnName(std::string_view lhs, std::string_view rhs); diff --git a/include/io/columnar_batch.h b/include/io/columnar_batch.h index 9dfebb0..788f7c2 100644 --- a/include/io/columnar_batch.h +++ b/include/io/columnar_batch.h @@ -43,6 +43,7 @@ class ColumnarBatchWriter final : public BatchWriter { void Write(const Batch& batch) override; void Flush() override; + void Finalize() &; void Finalize() &&; const ColumnarMetadata& GetMetadata() const { return metadata_; } diff --git a/src/common/planner_utils.cpp b/src/common/planner_utils.cpp index d85657c..5a30b3e 100644 --- a/src/common/planner_utils.cpp +++ b/src/common/planner_utils.cpp @@ -41,7 +41,7 @@ void ValidateColumnQualifier(const Query& query, const ColumnRef& column) { throw Error::InvalidArgument("executor", "unknown table qualifier '" + column.qualifier + "'"); } -static std::string UnescapeSqlString(const std::string_view text) { +std::string UnescapeSqlString(const std::string_view text) { if (text.size() < SqlStringQuoteCount || text.front() != SqlStringQuote || text.back() != SqlStringQuote) { throw Error::InvalidArgument("executor", "invalid string literal"); } diff --git a/src/convert/csv_columnar.cpp b/src/convert/csv_columnar.cpp index eff1ace..b49e36d 100644 --- a/src/convert/csv_columnar.cpp +++ b/src/convert/csv_columnar.cpp @@ -25,7 +25,7 @@ void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::f batch_writer.Write(*batch); } - std::move(batch_writer).Finalize(); + batch_writer.Finalize(); } void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, diff --git a/src/executor/executor.cpp b/src/executor/executor.cpp index 79045a2..1b6e087 100644 --- a/src/executor/executor.cpp +++ b/src/executor/executor.cpp @@ -26,7 +26,7 @@ ExecuteExpected Executor::Execute(const std::string_view query) const { return Batch{}; } - return std::move(*batch); + return std::move(batch).value(); } catch (const Error& error) { if (unsupported_fallback_enabled_) { return Batch{}; diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index 72d39b2..39649a0 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -245,19 +245,7 @@ static std::string NormalizeLiteralForEval(const QueryLiteral& literal) { return literal.text; } - std::string result; - const std::string_view text = literal.text; - - for (size_t i = 1; i + 1 < text.size(); ++i) { - if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { - result.push_back('\''); - ++i; - continue; - } - result.push_back(text[i]); - } - - return result; + return UnescapeSqlString(literal.text); } static std::optional TryFindBatchColumn(const Schema& schema, const std::string_view name) { @@ -271,6 +259,10 @@ static std::optional TryFindBatchColumn(const Schema& schema, const std: static Int128 EvalInt(const std::string& value) { return ParseInt128(value); } +static bool IsAggregateLookupFunction(const std::string_view name) { + return name == "COUNT" || name == "SUM" || name == "AVG" || name == "MIN" || name == "MAX"; +} + static std::string FormatAggregateExprName(const ExprSpec& expr) { const std::string name = ToUpperAscii(expr.function_name); if (name == "COUNT" && !expr.arguments.empty() && expr.arguments.front()->kind == ExprKind::Star) { @@ -294,22 +286,30 @@ static std::string FormatAggregateExprName(const ExprSpec& expr) { static std::string EvalExpr(const ExprPtr& expr, const Batch& batch, size_t row); static bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, size_t row); -static std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t row) { +static std::optional TryResolveAggregateValue(const ExprSpec& expr, const Batch& batch, const size_t row) { const std::string name = ToUpperAscii(expr.function_name); + if (!IsAggregateLookupFunction(name)) { + return std::nullopt; + } - if (name == "COUNT" || name == "SUM" || name == "AVG" || name == "MIN" || name == "MAX") { - if (const auto column = TryFindBatchColumn(batch.GetSchema(), FormatAggregateExprName(expr)); - column.has_value()) { - return batch.ColumnAt(*column).ValueAsString(row); - } - if (const auto column = TryFindBatchColumn(batch.GetSchema(), FallbackAggregateAlias); column.has_value()) { - return batch.ColumnAt(*column).ValueAsString(row); - } - if (const auto column = TryFindBatchColumn(batch.GetSchema(), PageViewsAlias); column.has_value()) { + const std::string formatted_name = FormatAggregateExprName(expr); + for (const std::string_view candidate_name : + {std::string_view{formatted_name}, FallbackAggregateAlias, PageViewsAlias}) { + if (const auto column = TryFindBatchColumn(batch.GetSchema(), candidate_name); column.has_value()) { return batch.ColumnAt(*column).ValueAsString(row); } } + return std::nullopt; +} + +static std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t row) { + if (const auto aggregate_value = TryResolveAggregateValue(expr, batch, row); aggregate_value.has_value()) { + return std::move(*aggregate_value); + } + + const std::string name = ToUpperAscii(expr.function_name); + if (name == "STRLEN" || name == "LENGTH") { return std::to_string(EvalExpr(expr.arguments.at(0), batch, row).size()); } @@ -639,6 +639,29 @@ static void SelectRowsMatchingPredicate(const PredicatePtr& predicate, const Bat } } +template +static Schema BuildAggregateOutputSchema(const std::vector& bindings) { + Schema schema; + schema.columns.reserve(bindings.size()); + + for (const auto& binding : bindings) { + schema.columns.push_back(ColumnSchema(binding.aggregate.output_name, AggregateOutputType(binding.aggregate))); + } + + return schema; +} + +static Schema BuildSelectOutputSchema(const std::vector& select_items) { + Schema schema; + schema.columns.reserve(select_items.size()); + + for (const auto& item : select_items) { + schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); + } + + return schema; +} + class FilterOperator final : public Operator { public: FilterOperator(std::unique_ptr child, PredicatePtr filter) @@ -930,15 +953,7 @@ class AggOperator final : public Operator { } } - Schema schema; - schema.columns.reserve(bindings_.size()); - - for (const auto& binding : bindings_) { - schema.columns.push_back( - ColumnSchema(binding.aggregate.output_name, AggregateOutputType(binding.aggregate))); - } - - Batch result(schema); + Batch result(BuildAggregateOutputSchema(bindings_)); for (size_t i = 0; i < bindings_.size(); ++i) { result.AppendValueFromString(i, bindings_[i].state->Finalize()); @@ -1009,13 +1024,7 @@ class GroupAggOperator final : public Operator { } } - Schema schema; - schema.columns.reserve(select_items_.size()); - - for (const auto& item : select_items_) { - schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); - } - + const Schema schema = BuildSelectOutputSchema(select_items_); std::vector output_order = BuildOutputOrder(); Batch result(schema, output_order.size()); @@ -1185,12 +1194,7 @@ class GroupAggTopKOperator final : public Operator { } } - Schema schema; - schema.columns.reserve(select_items_.size()); - for (const auto& item : select_items_) { - schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); - } - + const Schema schema = BuildSelectOutputSchema(select_items_); const std::vector top_groups = BuildTopGroupIndexes(); Batch result(schema, top_groups.size()); @@ -1587,6 +1591,27 @@ class OffsetOperator final : public Operator { size_t remaining_; }; +static void ApplyOrderOffsetLimit(std::unique_ptr& root, const PlannedQuery& planned, + bool& limit_applied_by_top_k) { + if (!planned.order_by.empty()) { + if (planned.limit.has_value() && planned.offset == 0) { + root = + std::make_unique(std::move(root), planned.order_by, planned.select_items, *planned.limit); + limit_applied_by_top_k = true; + } else { + root = std::make_unique(std::move(root), planned.order_by, planned.select_items); + } + } + + if (planned.offset > 0) { + root = std::make_unique(std::move(root), planned.offset); + } + + if (planned.limit.has_value() && !limit_applied_by_top_k) { + root = std::make_unique(std::move(root), *planned.limit); + } +} + std::unique_ptr BuildPlan(const PlannedQuery& planned) { if (planned.metadata_count_only) { return std::make_unique(planned.table_path, planned.aggregates.front().output_name); @@ -1605,21 +1630,7 @@ std::unique_ptr BuildPlan(const PlannedQuery& planned) { if (planned.plain_select) { bool limit_applied_by_top_k = false; - if (!planned.order_by.empty()) { - if (planned.limit.has_value() && planned.offset == 0) { - root = std::make_unique(std::move(root), planned.order_by, planned.select_items, - *planned.limit); - limit_applied_by_top_k = true; - } else { - root = std::make_unique(std::move(root), planned.order_by, planned.select_items); - } - } - if (planned.offset > 0) { - root = std::make_unique(std::move(root), planned.offset); - } - if (planned.limit.has_value() && !limit_applied_by_top_k) { - root = std::make_unique(std::move(root), *planned.limit); - } + ApplyOrderOffsetLimit(root, planned, limit_applied_by_top_k); auto projection = std::make_unique(std::move(root), planned.plain_select_items, planned.table_schema); @@ -1647,23 +1658,7 @@ std::unique_ptr BuildPlan(const PlannedQuery& planned) { root = std::make_unique(std::move(root), planned.aggregates, planned.filter); } - if (!planned.order_by.empty()) { - if (planned.limit.has_value() && planned.offset == 0) { - root = - std::make_unique(std::move(root), planned.order_by, planned.select_items, *planned.limit); - limit_applied_by_top_k = true; - } else { - root = std::make_unique(std::move(root), planned.order_by, planned.select_items); - } - } - - if (planned.offset > 0) { - root = std::make_unique(std::move(root), planned.offset); - } - - if (planned.limit.has_value() && !limit_applied_by_top_k) { - root = std::make_unique(std::move(root), *planned.limit); - } + ApplyOrderOffsetLimit(root, planned, limit_applied_by_top_k); return root; } diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index a96ea07..282f2c7 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -334,24 +334,9 @@ static std::optional TryNumericLiteralValue(const ExprPtr& expr) { } } -static std::string NormalizeStringLiteral(const std::string_view text) { - std::string result; - - for (size_t i = 1; i + 1 < text.size(); ++i) { - if (text[i] == '\'' && i + 2 < text.size() && text[i + 1] == '\'') { - result.push_back('\''); - ++i; - continue; - } - result.push_back(text[i]); - } - - return result; -} - static std::string LiteralTextForEval(const QueryLiteral& literal) { if (literal.kind == LiteralKind::String) { - return NormalizeStringLiteral(literal.text); + return UnescapeSqlString(literal.text); } return literal.text; } @@ -391,7 +376,7 @@ static std::optional TryLiteralValueForColumnType(const QueryLiteral& li return std::nullopt; } - const std::string normalized = NormalizeStringLiteral(literal.text); + const std::string normalized = UnescapeSqlString(literal.text); switch (type) { case ColumnType::Boolean: return ParseBoolean(normalized) ? 1 : 0; diff --git a/src/io/columnar_batch.cpp b/src/io/columnar_batch.cpp index e38a389..392c3f9 100644 --- a/src/io/columnar_batch.cpp +++ b/src/io/columnar_batch.cpp @@ -161,6 +161,8 @@ void ColumnarBatchWriter::Write(const Batch& batch) { void ColumnarBatchWriter::Flush() { FlushWrite(path_, out_); } +void ColumnarBatchWriter::Finalize() & { std::move(*this).Finalize(); } + void ColumnarBatchWriter::Finalize() && { if (finalized_) { throw Error::InvalidState("io", "writer already finalized", path_.string()); From ea579f8f836fa527c675f475f76857112bb92549 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 13:35:16 +0300 Subject: [PATCH 02/13] feat: added lz4 compression - 6.71x ratio --- include/convert/csv_columnar.h | 6 +- include/io/columnar_batch.h | 11 ++- include/io/compression.h | 17 +++++ include/model/metadata.h | 5 +- scripts/dashboard.py | 12 +++ src/CMakeLists.txt | 13 ++++ src/app/main.cpp | 4 +- src/convert/csv_columnar.cpp | 5 +- src/executor/operators.cpp | 2 +- src/io/columnar_batch.cpp | 78 ++++++++++++++++--- src/io/compression.cpp | 97 +++++++++++++++++++++++ src/model/metadata.cpp | 112 ++++++++++++++++++++++++++- tests/test_columnar.cpp | 136 ++++++++++++++++++++++++++------- 13 files changed, 448 insertions(+), 50 deletions(-) create mode 100644 include/io/compression.h create mode 100644 src/io/compression.cpp diff --git a/include/convert/csv_columnar.h b/include/convert/csv_columnar.h index 058af34..a069e1e 100644 --- a/include/convert/csv_columnar.h +++ b/include/convert/csv_columnar.h @@ -3,10 +3,12 @@ #include #include +#include "io/compression.h" + inline constexpr size_t DefaultMaxRowsPerGroup = 1 << 14; void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, - const std::filesystem::path& output_path, - size_t max_rows_per_group = DefaultMaxRowsPerGroup); + const std::filesystem::path& output_path, size_t max_rows_per_group = DefaultMaxRowsPerGroup, + Compression compression = Compression::None); void ConvertColumnarToCsv(const std::filesystem::path& columnar_path, const std::filesystem::path& schema_path, const std::filesystem::path& data_path); diff --git a/include/io/columnar_batch.h b/include/io/columnar_batch.h index 788f7c2..ae8c0b8 100644 --- a/include/io/columnar_batch.h +++ b/include/io/columnar_batch.h @@ -2,8 +2,11 @@ #include #include +#include +#include #include "io/batch.h" +#include "io/compression.h" #include "io/file.h" #include "model/metadata.h" @@ -33,7 +36,7 @@ class ColumnarBatchReader final : public BatchReader { class ColumnarBatchWriter final : public BatchWriter { public: - ColumnarBatchWriter(const std::filesystem::path& path, Schema schema); + ColumnarBatchWriter(const std::filesystem::path& path, Schema schema, Compression compression = Compression::None); ColumnarBatchWriter(const ColumnarBatchWriter&) = delete; ColumnarBatchWriter(ColumnarBatchWriter&&) noexcept = default; ColumnarBatchWriter& operator=(const ColumnarBatchWriter&) = delete; @@ -53,5 +56,11 @@ class ColumnarBatchWriter final : public BatchWriter { std::ofstream out_; ColumnarMetadata metadata_; + Compression compression_ = Compression::None; bool finalized_ = false; }; + +std::vector ReadColumnChunk(const std::filesystem::path& path, InputFile& input, + const ColumnChunkMetadata& chunk); +void ReadBatchColumnChunk(const std::filesystem::path& path, InputFile& input, const ColumnChunkMetadata& chunk, + uint32_t row_count, const Batch& batch, size_t column_index); diff --git a/include/io/compression.h b/include/io/compression.h new file mode 100644 index 0000000..4f9d0f1 --- /dev/null +++ b/include/io/compression.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include +#include + +enum class Compression : uint8_t { + None = 0, + Lz4 = 1, +}; + +const char* CompressionName(Compression compression); +Compression CompressionFromName(std::string_view name); + +std::vector Compress(std::span input, Compression compression); +std::vector Decompress(std::span input, Compression compression, uint64_t uncompressed_size); diff --git a/include/model/metadata.h b/include/model/metadata.h index dbc3844..2152d1c 100644 --- a/include/model/metadata.h +++ b/include/model/metadata.h @@ -5,11 +5,14 @@ #include #include "common/int128.h" +#include "io/compression.h" #include "model/schema.h" struct ColumnChunkMetadata { uint64_t offset = 0; - uint64_t size = 0; + uint64_t compressed_size = 0; + uint64_t uncompressed_size = 0; + Compression compression = Compression::None; bool has_min_max = false; Int128 min_value = 0; Int128 max_value = 0; diff --git a/scripts/dashboard.py b/scripts/dashboard.py index 4f8f3eb..8b8ce3f 100644 --- a/scripts/dashboard.py +++ b/scripts/dashboard.py @@ -76,6 +76,12 @@ def parse_args() -> argparse.Namespace: default=1 << 14, help="Row group size for the CSV -> columnar conversion benchmark.", ) + parser.add_argument( + "--compression", + default="none", + choices=("none", "lz4"), + help="Compression codec for the CSV -> columnar conversion benchmark.", + ) return parser.parse_args() @@ -161,6 +167,7 @@ def collect_conversion_stats( schema_path: pathlib.Path, source_csv_path: pathlib.Path, row_group_size: int, + compression: str, ) -> dict[str, float | int | str] | None: if not schema_path.exists() or not source_csv_path.exists(): return None @@ -182,6 +189,8 @@ def collect_conversion_stats( str(output_columnar), "--row-group-size", str(row_group_size), + "--compression", + compression, ] roundtrip_command = [ str(binary), @@ -215,6 +224,7 @@ def collect_conversion_stats( return { "source_csv_path": str(source_csv_path.relative_to(root)), "schema_path": str(schema_path.relative_to(root)), + "compression": compression, "source_csv_bytes": source_csv_bytes, "columnar_bytes": columnar_bytes, "roundtrip_csv_bytes": roundtrip_csv_bytes, @@ -460,6 +470,7 @@ def render_markdown( "| --- | ---: |", f"| source csv | `{conversion_stats['source_csv_path']}` |", f"| schema | `{conversion_stats['schema_path']}` |", + f"| compression | `{conversion_stats['compression']}` |", f"| source size | {format_size(int(conversion_stats['source_csv_bytes']))} |", f"| columnar size | {format_size(int(conversion_stats['columnar_bytes']))} |", f"| roundtrip csv size | {format_size(int(conversion_stats['roundtrip_csv_bytes']))} |", @@ -612,6 +623,7 @@ def main() -> int: schema_path=schema_path, source_csv_path=source_csv_path, row_group_size=args.row_group_size, + compression=args.compression, ) write_csv(csv_path, rows) markdown_block = render_markdown( diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b81e294..efb6711 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,9 +1,21 @@ +find_path(LZ4_INCLUDE_DIR lz4.h) +find_library(LZ4_LIBRARY lz4) + +if (NOT LZ4_INCLUDE_DIR OR NOT LZ4_LIBRARY) + message(FATAL_ERROR "lz4 headers/library not found") +endif () + +add_library(lz4_external INTERFACE) +target_include_directories(lz4_external INTERFACE ${LZ4_INCLUDE_DIR}) +target_link_libraries(lz4_external INTERFACE ${LZ4_LIBRARY}) + add_library(columnar_engine_core model/batch.cpp model/column.cpp model/column_string.cpp model/metadata.cpp io/batch.cpp + io/compression.cpp io/file.cpp io/stream.cpp common/error.cpp @@ -14,6 +26,7 @@ add_library(columnar_engine_core ) target_include_directories(columnar_engine_core PUBLIC ${COLUMNAR_INCLUDE_DIRS}) +target_link_libraries(columnar_engine_core PUBLIC lz4_external) add_library(columnar_engine_columnar io/columnar_batch.cpp diff --git a/src/app/main.cpp b/src/app/main.cpp index 9277427..75ef337 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -23,6 +23,7 @@ void ConfigureConvertCommand(argparse::ArgumentParser& command) { command.add_argument("--input").required(); command.add_argument("--output").required(); command.add_argument("--row-group-size").scan<'u', size_t>().default_value(size_t{1 << 14}); + command.add_argument("--compression").default_value(std::string("none")); } void ConfigureRoundtripCommand(argparse::ArgumentParser& command) { @@ -57,7 +58,8 @@ int RunConvert(const argparse::ArgumentParser& command) { EnsureParentDirectory(output_path); ConvertCsvToColumnar(std::filesystem::path(command.get("--schema")), std::filesystem::path(command.get("--input")), output_path, - command.get("--row-group-size")); + command.get("--row-group-size"), + CompressionFromName(command.get("--compression"))); return 0; } diff --git a/src/convert/csv_columnar.cpp b/src/convert/csv_columnar.cpp index b49e36d..ef7f599 100644 --- a/src/convert/csv_columnar.cpp +++ b/src/convert/csv_columnar.cpp @@ -8,7 +8,8 @@ #include "model/schema_csv.h" void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::filesystem::path& data_path, - const std::filesystem::path& output_path, size_t max_rows_per_group) { + const std::filesystem::path& output_path, const size_t max_rows_per_group, + const Compression compression) { if (max_rows_per_group == 0) { throw Error::InvalidArgument("convert", "row group size must be > 0"); } @@ -19,7 +20,7 @@ void ConvertCsvToColumnar(const std::filesystem::path& schema_path, const std::f sizing.max_rows = max_rows_per_group; CsvBatchReader batch_reader(data_path, schema, sizing); - ColumnarBatchWriter batch_writer(output_path, schema); + ColumnarBatchWriter batch_writer(output_path, schema, compression); while (auto batch = batch_reader.ReadNext()) { batch_writer.Write(*batch); diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index 39649a0..aac847c 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -223,7 +223,7 @@ class ScanOperator final : public Operator { const size_t source_index = projection_indexes_[projected_index]; const auto& chunk = row_group->columns[source_index]; - batch.ReadColumnFrom(projected_index, input_.StreamAt(chunk.offset), row_group->row_count, chunk.size); + ReadBatchColumnChunk(path_, input_, chunk, row_group->row_count, batch, projected_index); } return batch; diff --git a/src/io/columnar_batch.cpp b/src/io/columnar_batch.cpp index 392c3f9..25c7f6b 100644 --- a/src/io/columnar_batch.cpp +++ b/src/io/columnar_batch.cpp @@ -3,12 +3,15 @@ #include #include #include +#include #include #include #include #include +#include #include "common/error.h" +#include "io/compression.h" #include "io/stream.h" static constexpr std::string_view ColumnarMagic = "CLMN"; @@ -53,6 +56,42 @@ static void PopulateChunkMinMax(const Column& column, ColumnChunkMetadata& chunk chunk.max_value = max_value; } +static std::vector SerializeColumn(const Column& column) { + std::ostringstream buffer(std::ios::binary); + column.WriteTo(buffer); + const std::string bytes = std::move(buffer).str(); + return std::vector(bytes.begin(), bytes.end()); +} + +static ColumnChunkMetadata WriteColumnChunk(const std::filesystem::path& path, std::ofstream& out, const Column& column, + const Compression compression) { + const std::vector uncompressed = SerializeColumn(column); + const std::vector compressed = Compress(uncompressed, compression); + + const std::span payload = compressed.size() < uncompressed.size() + ? std::span(compressed) + : std::span(uncompressed); + const Compression stored_compression = compressed.size() < uncompressed.size() ? compression : Compression::None; + const uint64_t offset = TellWrite(path, out); + + if (!payload.empty()) { + out.write(reinterpret_cast(payload.data()), static_cast(payload.size())); + if (!out) { + throw Error::PathIo("io", path, "write file"); + } + } + + ColumnChunkMetadata chunk; + chunk.offset = offset; + chunk.compressed_size = payload.size(); + chunk.uncompressed_size = uncompressed.size(); + chunk.compression = stored_compression; + + PopulateChunkMinMax(column, chunk); + + return chunk; +} + ColumnarMetadata ColumnarBatchReader::ReadFileMetadata(const std::filesystem::path& path) { const auto file_metadata = GetFileMetadata(path); @@ -106,14 +145,15 @@ std::optional ColumnarBatchReader::ReadNext() { for (size_t col = 0; col < batch.ColumnsCount(); ++col) { const auto& chunk = row_group_columns[col]; - batch.ReadColumnFrom(col, input_.StreamAt(chunk.offset), row_count, chunk.size); + ReadBatchColumnChunk(path_, input_, chunk, row_count, batch, col); } return batch; } -ColumnarBatchWriter::ColumnarBatchWriter(const std::filesystem::path& path, Schema schema) - : path_(path), out_(OpenOutputFile(path)) { +ColumnarBatchWriter::ColumnarBatchWriter(const std::filesystem::path& path, Schema schema, + const Compression compression) + : path_(path), out_(OpenOutputFile(path)), compression_(compression) { if (schema.columns.empty()) { throw Error::InvalidArgument("io", "schema has no columns", path.string()); } @@ -145,15 +185,8 @@ void ColumnarBatchWriter::Write(const Batch& batch) { group.columns.reserve(batch.ColumnsCount()); for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { - const uint64_t offset = TellWrite(path_, out_); const Column& column = batch.ColumnAt(column_index); - column.WriteTo(out_); - - const uint64_t end = TellWrite(path_, out_); - - ColumnChunkMetadata chunk{offset, end - offset}; - PopulateChunkMinMax(column, chunk); - group.columns.push_back(chunk); + group.columns.push_back(WriteColumnChunk(path_, out_, column, compression_)); } metadata_.row_groups.push_back(std::move(group)); @@ -180,3 +213,26 @@ void ColumnarBatchWriter::Finalize() && { finalized_ = true; } + +std::vector ReadColumnChunk(const std::filesystem::path& path, InputFile& input, + const ColumnChunkMetadata& chunk) { + const std::string raw = input.ReadStringAt(chunk.offset, chunk.compressed_size); + const std::span raw_bytes(reinterpret_cast(raw.data()), raw.size()); + + if (chunk.compression == Compression::None) { + if (chunk.compressed_size != chunk.uncompressed_size) { + throw Error::MalformedData("io", "uncompressed chunk size mismatch", path.string()); + } + return std::vector(raw_bytes.begin(), raw_bytes.end()); + } + + return Decompress(raw_bytes, chunk.compression, chunk.uncompressed_size); +} + +void ReadBatchColumnChunk(const std::filesystem::path& path, InputFile& input, const ColumnChunkMetadata& chunk, + const uint32_t row_count, const Batch& batch, const size_t column_index) { + const std::vector payload = ReadColumnChunk(path, input, chunk); + const std::string bytes(payload.begin(), payload.end()); + std::istringstream stream(bytes, std::ios::binary); + batch.ReadColumnFrom(column_index, stream, row_count, chunk.uncompressed_size); +} diff --git a/src/io/compression.cpp b/src/io/compression.cpp new file mode 100644 index 0000000..ec39b62 --- /dev/null +++ b/src/io/compression.cpp @@ -0,0 +1,97 @@ +#include "io/compression.h" + +#include + +#include +#include +#include + +#include "common/error.h" + +const char* CompressionName(const Compression compression) { + switch (compression) { + case Compression::None: + return "none"; + case Compression::Lz4: + return "lz4"; + } + + throw Error::InvalidArgument("compression", "unknown compression codec"); +} + +Compression CompressionFromName(const std::string_view name) { + if (name == "none") { + return Compression::None; + } + if (name == "lz4") { + return Compression::Lz4; + } + + throw Error::InvalidArgument("compression", "unsupported compression codec: " + std::string(name)); +} + +std::vector Compress(const std::span input, const Compression compression) { + switch (compression) { + case Compression::None: + return std::vector(input.begin(), input.end()); + case Compression::Lz4: + break; + } + + if (input.size() > static_cast(std::numeric_limits::max())) { + throw Error::Overflow("compression", "input too large for lz4"); + } + + const int upper_bound = LZ4_compressBound(static_cast(input.size())); + if (upper_bound <= 0) { + throw Error::InvalidState("compression", "lz4 upper bound failed"); + } + + std::vector output(static_cast(upper_bound)); + const int compressed_size = + LZ4_compress_default(reinterpret_cast(input.data()), reinterpret_cast(output.data()), + static_cast(input.size()), upper_bound); + + if (compressed_size <= 0) { + throw Error::Io("compression", "lz4 compression failed"); + } + + output.resize(static_cast(compressed_size)); + + return output; +} + +std::vector Decompress(const std::span input, const Compression compression, + const uint64_t uncompressed_size) { + switch (compression) { + case Compression::None: + if (input.size() != uncompressed_size) { + throw Error::MalformedData("compression", "uncompressed chunk size mismatch"); + } + return std::vector(input.begin(), input.end()); + case Compression::Lz4: + break; + } + + if (input.size() > static_cast(std::numeric_limits::max())) { + throw Error::Overflow("compression", "compressed input too large for lz4"); + } + if (uncompressed_size > static_cast(std::numeric_limits::max())) { + throw Error::Overflow("compression", "uncompressed output too large for lz4"); + } + + std::vector output(uncompressed_size); + const int decompressed_size = + LZ4_decompress_safe(reinterpret_cast(input.data()), reinterpret_cast(output.data()), + static_cast(input.size()), static_cast(uncompressed_size)); + + if (decompressed_size < 0) { + throw Error::MalformedData("compression", "lz4 decompression failed"); + } + + if (static_cast(decompressed_size) != uncompressed_size) { + throw Error::MalformedData("compression", "lz4 decompressed size mismatch"); + } + + return output; +} diff --git a/src/model/metadata.cpp b/src/model/metadata.cpp index 9b1dd59..ff9d561 100644 --- a/src/model/metadata.cpp +++ b/src/model/metadata.cpp @@ -1,11 +1,15 @@ #include "model/metadata.h" +#include #include +#include #include #include "common/error.h" #include "io/stream.h" +static constexpr std::array MetadataMagic = {'C', 'M', 'D', '2'}; + static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { switch (static_cast(type_byte)) { case ColumnType::Int64: @@ -23,7 +27,17 @@ static ColumnType ColumnTypeFromByte(const uint8_t type_byte) { throw Error::MalformedData("model", "unknown column type in metadata"); } -ColumnarMetadata ReadMetadata(std::istream& in) { +static Compression CompressionFromByte(const uint8_t compression_byte) { + switch (static_cast(compression_byte)) { + case Compression::None: + case Compression::Lz4: + return static_cast(compression_byte); + } + + throw Error::MalformedData("model", "unknown compression codec in metadata"); +} + +static ColumnarMetadata ReadLegacyMetadata(std::istream& in) { ColumnarMetadata metadata; const uint32_t column_count = ReadStream(in); @@ -58,7 +72,9 @@ ColumnarMetadata ReadMetadata(std::istream& in) { for (uint32_t col = 0; col < columns_in_group; ++col) { ColumnChunkMetadata chunk; chunk.offset = ReadStream(in); - chunk.size = ReadStream(in); + chunk.compressed_size = ReadStream(in); + chunk.uncompressed_size = chunk.compressed_size; + chunk.compression = Compression::None; group.columns.push_back(chunk); } @@ -85,11 +101,99 @@ ColumnarMetadata ReadMetadata(std::istream& in) { return metadata; } +static ColumnarMetadata ReadV2Metadata(std::istream& in) { + ColumnarMetadata metadata; + + const uint32_t version = ReadStream(in); + if (version != 2) { + throw Error::MalformedData("model", "unsupported metadata version"); + } + + const uint32_t column_count = ReadStream(in); + metadata.schema.columns.reserve(column_count); + + for (uint32_t i = 0; i < column_count; ++i) { + const uint32_t name_size = ReadStream(in); + std::string name(name_size, '\0'); + + ReadBytes(in, name.data(), name.size()); + + const uint8_t type_byte = ReadStream(in); + const ColumnType type = ColumnTypeFromByte(type_byte); + + metadata.schema.columns.push_back(ColumnSchema{std::move(name), type}); + } + + const uint32_t row_group_count = ReadStream(in); + metadata.row_groups.reserve(row_group_count); + + for (uint32_t i = 0; i < row_group_count; ++i) { + RowGroupMetadata group; + group.row_count = ReadStream(in); + + const uint32_t columns_in_group = ReadStream(in); + if (columns_in_group != column_count) { + throw Error::InconsistentData("model", "row group column count mismatch"); + } + + group.columns.reserve(columns_in_group); + + for (uint32_t col = 0; col < columns_in_group; ++col) { + ColumnChunkMetadata chunk; + chunk.offset = ReadStream(in); + chunk.compressed_size = ReadStream(in); + chunk.uncompressed_size = ReadStream(in); + chunk.compression = CompressionFromByte(ReadStream(in)); + + group.columns.push_back(chunk); + } + + metadata.row_groups.push_back(std::move(group)); + } + + for (auto& row_group : metadata.row_groups) { + for (auto& column : row_group.columns) { + column.has_min_max = ReadStream(in) != 0; + if (!column.has_min_max) { + continue; + } + + column.min_value = ReadStream(in); + column.max_value = ReadStream(in); + } + } + + if (in.peek() != std::istream::traits_type::eof()) { + throw Error::MalformedData("model", "unexpected trailing bytes in metadata"); + } + + return metadata; +} + +ColumnarMetadata ReadMetadata(std::istream& in) { + const std::streampos start = in.tellg(); + if (start != std::streampos(-1)) { + char magic[MetadataMagic.size()]; + in.read(magic, sizeof(magic)); + if (in.gcount() == static_cast(MetadataMagic.size()) && + std::equal(std::begin(magic), std::end(magic), MetadataMagic.begin())) { + return ReadV2Metadata(in); + } + + in.clear(); + in.seekg(start); + } + + return ReadLegacyMetadata(in); +} + void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { if (metadata.schema.columns.size() > std::numeric_limits::max()) { throw Error::Overflow("model", "too many columns for metadata"); } + WriteBytes(out, {MetadataMagic.data(), MetadataMagic.size()}); + WriteStream(out, 2); WriteStream(out, metadata.schema.columns.size()); for (const auto& [name, type] : metadata.schema.columns) { @@ -114,7 +218,9 @@ void WriteMetadata(std::ostream& out, const ColumnarMetadata& metadata) { for (const auto& column : columns) { WriteStream(out, column.offset); - WriteStream(out, column.size); + WriteStream(out, column.compressed_size); + WriteStream(out, column.uncompressed_size); + WriteStream(out, static_cast(column.compression)); } } diff --git a/tests/test_columnar.cpp b/tests/test_columnar.cpp index 3bb0a9e..9902a56 100644 --- a/tests/test_columnar.cpp +++ b/tests/test_columnar.cpp @@ -1,16 +1,20 @@ #include +#include #include #include +#include "common/error.h" +#include "common/parsing.h" #include "convert/csv_columnar.h" #include "gtest/gtest.h" #include "io/columnar_batch.h" +#include "io/compression.h" #include "io/csv.h" #include "io/file.h" +#include "io/stream.h" +#include "model/metadata.h" #include "model/schema.h" #include "model/schema_csv.h" -#include "common/error.h" -#include "common/parsing.h" #include "testing/temp_file.h" TEST(columnar, csv_to_columnar_and_back) { @@ -96,10 +100,18 @@ TEST(columnar, metadata_offsets_and_sizes) { return total; }; - EXPECT_EQ(row_groups[0].columns[0].size, sizeof(int64_t) * 2u); - EXPECT_EQ(row_groups[0].columns[1].size, string_chunk_size(0, 2)); - EXPECT_EQ(row_groups[1].columns[0].size, sizeof(int64_t) * 1u); - EXPECT_EQ(row_groups[1].columns[1].size, string_chunk_size(2, 1)); + EXPECT_EQ(row_groups[0].columns[0].compressed_size, sizeof(int64_t) * 2u); + EXPECT_EQ(row_groups[0].columns[0].uncompressed_size, sizeof(int64_t) * 2u); + EXPECT_EQ(row_groups[0].columns[0].compression, Compression::None); + EXPECT_EQ(row_groups[0].columns[1].compressed_size, string_chunk_size(0, 2)); + EXPECT_EQ(row_groups[0].columns[1].uncompressed_size, string_chunk_size(0, 2)); + EXPECT_EQ(row_groups[0].columns[1].compression, Compression::None); + EXPECT_EQ(row_groups[1].columns[0].compressed_size, sizeof(int64_t) * 1u); + EXPECT_EQ(row_groups[1].columns[0].uncompressed_size, sizeof(int64_t) * 1u); + EXPECT_EQ(row_groups[1].columns[0].compression, Compression::None); + EXPECT_EQ(row_groups[1].columns[1].compressed_size, string_chunk_size(2, 1)); + EXPECT_EQ(row_groups[1].columns[1].uncompressed_size, string_chunk_size(2, 1)); + EXPECT_EQ(row_groups[1].columns[1].compression, Compression::None); EXPECT_TRUE(row_groups[0].columns[0].has_min_max); EXPECT_EQ(row_groups[0].columns[0].min_value, 1); EXPECT_EQ(row_groups[0].columns[0].max_value, 2); @@ -113,7 +125,7 @@ TEST(columnar, metadata_offsets_and_sizes) { for (const auto& row_group : row_groups) { for (const auto& column : row_group.columns) { EXPECT_EQ(column.offset, expected_offset); - expected_offset += column.size; + expected_offset += column.compressed_size; } } @@ -142,15 +154,8 @@ TEST(columnar, mixed_supported_types_roundtrip) { const TempFile data_out("data_mixed_out"); const std::vector> schema_rows = { - {"flag", "bool"}, - {"small", "int16"}, - {"medium", "int32"}, - {"big", "int64"}, - {"huge", "int128"}, - {"name", "string"}, - {"birth_date", "date"}, - {"created_at", "timestamp"}, - {"grade", "char"}, + {"flag", "bool"}, {"small", "int16"}, {"medium", "int32"}, {"big", "int64"}, {"huge", "int128"}, + {"name", "string"}, {"birth_date", "date"}, {"created_at", "timestamp"}, {"grade", "char"}, }; const std::vector> data_rows = { @@ -186,18 +191,17 @@ TEST(columnar, infer_schema_from_csv_is_convert_compatible) { WriteRows(data_in.Path(), data_rows); WriteSchemaCsv(schema_out.Path(), InferSchemaCsv(data_in.Path())); - EXPECT_EQ(ReadRows(schema_out.Path()), - (std::vector>{ - {"column_1", "bool"}, - {"column_2", "int16"}, - {"column_3", "int32"}, - {"column_4", "int64"}, - {"column_5", "int128"}, - {"column_6", "string"}, - {"column_7", "date"}, - {"column_8", "timestamp"}, - {"column_9", "char"}, - })); + EXPECT_EQ(ReadRows(schema_out.Path()), (std::vector>{ + {"column_1", "bool"}, + {"column_2", "int16"}, + {"column_3", "int32"}, + {"column_4", "int64"}, + {"column_5", "int128"}, + {"column_6", "string"}, + {"column_7", "date"}, + {"column_8", "timestamp"}, + {"column_9", "char"}, + })); ConvertCsvToColumnar(schema_out.Path(), data_in.Path(), columnar_file.Path(), 1); ConvertColumnarToCsv(columnar_file.Path(), schema_out.Path(), data_out.Path()); @@ -240,3 +244,79 @@ TEST(columnar, metadata_records_minmax_stats_for_date_columns) { EXPECT_EQ(DateToString(static_cast(row_groups[1].columns[0].min_value)), "2024-01-01"); EXPECT_EQ(DateToString(static_cast(row_groups[1].columns[0].max_value)), "2024-01-09"); } + +TEST(columnar, lz4_compressed_roundtrip) { + const TempFile schema_in("schema_lz4_in"); + const TempFile data_in("data_lz4_in"); + const TempFile columnar_file("columnar_lz4"); + const TempFile data_out("data_lz4_out"); + const TempFile schema_out("schema_lz4_out"); + + WriteRows(schema_in.Path(), {{"id", "int64"}, {"payload", "string"}}); + + std::vector> data_rows; + for (int i = 0; i < 256; ++i) { + data_rows.push_back({std::to_string(i), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + } + WriteRows(data_in.Path(), data_rows); + + ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 256, Compression::Lz4); + ConvertColumnarToCsv(columnar_file.Path(), schema_out.Path(), data_out.Path()); + + ColumnarBatchReader reader(columnar_file.Path()); + const auto& row_group = reader.GetMetadata().row_groups.at(0); + ASSERT_EQ(row_group.columns.size(), 2u); + EXPECT_EQ(ReadRows(data_out.Path()), data_rows); + EXPECT_EQ(ReadRows(schema_out.Path()), + (std::vector>{{"id", "int64"}, {"payload", "string"}})); + EXPECT_EQ(row_group.columns[1].compression, Compression::Lz4); + EXPECT_LT(row_group.columns[1].compressed_size, row_group.columns[1].uncompressed_size); +} + +TEST(columnar, lz4_falls_back_to_none_when_chunk_does_not_shrink) { + const TempFile schema_in("schema_lz4_fallback_in"); + const TempFile data_in("data_lz4_fallback_in"); + const TempFile columnar_file("columnar_lz4_fallback"); + + WriteRows(schema_in.Path(), {{"value", "int64"}}); + WriteRows(data_in.Path(), {{"42"}}); + + ConvertCsvToColumnar(schema_in.Path(), data_in.Path(), columnar_file.Path(), 1, Compression::Lz4); + + const ColumnarBatchReader reader(columnar_file.Path()); + const auto& chunk = reader.GetMetadata().row_groups.at(0).columns.at(0); + EXPECT_EQ(chunk.compression, Compression::None); + EXPECT_EQ(chunk.compressed_size, sizeof(int64_t)); + EXPECT_EQ(chunk.uncompressed_size, sizeof(int64_t)); +} + +TEST(columnar, read_legacy_metadata_without_compression_fields) { + std::ostringstream out(std::ios::binary); + + WriteStream(out, 1); + WriteStream(out, 2); + WriteBytes(out, "id"); + WriteStream(out, static_cast(ColumnType::Int64)); + WriteStream(out, 1); + WriteStream(out, 3); + WriteStream(out, 1); + WriteStream(out, 10); + WriteStream(out, 24); + WriteStream(out, 1); + WriteStream(out, 7); + WriteStream(out, 9); + + std::istringstream in(out.str(), std::ios::binary); + const ColumnarMetadata metadata = ReadMetadata(in); + + ASSERT_EQ(metadata.schema.columns.size(), 1u); + ASSERT_EQ(metadata.row_groups.size(), 1u); + const auto& chunk = metadata.row_groups[0].columns[0]; + EXPECT_EQ(chunk.offset, 10u); + EXPECT_EQ(chunk.compressed_size, 24u); + EXPECT_EQ(chunk.uncompressed_size, 24u); + EXPECT_EQ(chunk.compression, Compression::None); + EXPECT_TRUE(chunk.has_min_max); + EXPECT_EQ(chunk.min_value, 7); + EXPECT_EQ(chunk.max_value, 9); +} From aba7ad14d77412507051ede688550f1339a2a2f7 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 13:35:29 +0300 Subject: [PATCH 03/13] upd: update dashboard --- README.md | 161 ++++++++++++----------- benchmarks/results/readme_benchmarks.csv | 86 ++++++------ 2 files changed, 124 insertions(+), 123 deletions(-) diff --git a/README.md b/README.md index 4456393..23ae626 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## Benchmark Dashboard -`generated 2026-05-28 03:03:29` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) +`generated 2026-05-28 13:26:56` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) ### Summary @@ -13,14 +13,14 @@ | --- | ---: | | queries ok | 43 / 43 | | queries failed | 0 | -| median warm time | 237.07 ms | -| average warm time | 268.20 ms | -| p95 warm time | 777.08 ms | -| cold/warm delta | 11.3% | +| median warm time | 239.79 ms | +| average warm time | 274.57 ms | +| p95 warm time | 813.63 ms | +| cold/warm delta | 6.7% | | total output size | 20.3 KB | | max output size | 6.0 KB | -| fastest query | Q06 · 3.68 ms | -| slowest query | Q18 · 1112.19 ms | +| fastest query | Q06 · 4.20 ms | +| slowest query | Q18 · 1146.61 ms | ### Storage @@ -28,15 +28,16 @@ | --- | ---: | | source csv | `benchmarks/hits_sample.csv` | | schema | `benchmarks/scheme.csv` | +| compression | `lz4` | | source size | 802.6 MB | -| columnar size | 694.9 MB | +| columnar size | 119.7 MB | | roundtrip csv size | 743.0 MB | -| compression ratio | 1.16x | -| columnar / csv | 86.6% | -| csv -> columnar | 5.77s | -| columnar -> csv | 9.67s | -| convert throughput | 139.2 MB/s | -| roundtrip throughput | 71.9 MB/s | +| compression ratio | 6.71x | +| columnar / csv | 14.9% | +| csv -> columnar | 5.52s | +| columnar -> csv | 9.65s | +| convert throughput | 145.4 MB/s | +| roundtrip throughput | 12.4 MB/s | ### Heatmap @@ -44,89 +45,89 @@ | slot 1 | slot 2 | slot 3 | slot 4 | slot 5 | slot 6 | | ---: | ---: | ---: | ---: | ---: | ---: | -| Q00 🟩 `3.91` | Q01 🟩 `6.32` | Q02 🟩 `17.25` | Q03 🟩 `13.37` | Q04 🟩 `148.12` | Q05 🟩 `130.96` | -| Q06 🟩 `3.68` | Q07 🟩 `6.69` | Q08 🟩 `193.36` | Q09 🟩 `209.45` | Q10 🟥 `268.08` | Q11 🟥 `269.48` | -| Q12 🟦 `233.32` | Q13 🟦 `245.55` | Q14 🟦 `238.18` | Q15 🟩 `73.32` | Q16 🟩 `122.39` | Q17 🟩 `144.28` | -| Q18 🟥 `1112.19` | Q19 🟩 `4.11` | Q20 🟩 `134.97` | Q21 🟥 `268.11` | Q22 🟥 `357.07` | Q23 🟦 `236.84` | -| Q24 🟥 `273.73` | Q25 🟦 `226.36` | Q26 🟥 `289.97` | Q27 🟥 `398.98` | Q28 🟥 `907.32` | Q29 🟥 `414.52` | -| Q30 🟦 `237.07` | Q31 🟥 `262.05` | Q32 🟥 `787.69` | Q33 🟥 `419.25` | Q34 🟥 `423.99` | Q35 🟥 `681.60` | -| Q36 🟥 `337.69` | Q37 🟦 `244.01` | Q38 🟩 `67.28` | Q39 🟥 `426.31` | Q40 🟩 `92.18` | Q41 🟩 `33.18` | -| Q42 🟥 `568.52` | | | | | | +| Q00 🟩 `5.33` | Q01 🟩 `7.51` | Q02 🟩 `18.46` | Q03 🟩 `14.22` | Q04 🟩 `142.67` | Q05 🟩 `127.92` | +| Q06 🟩 `4.20` | Q07 🟩 `6.58` | Q08 🟩 `184.89` | Q09 🟩 `200.71` | Q10 🟥 `266.04` | Q11 🟥 `308.16` | +| Q12 🟦 `228.27` | Q13 🟦 `244.72` | Q14 🟦 `229.88` | Q15 🟩 `69.88` | Q16 🟩 `139.03` | Q17 🟩 `149.92` | +| Q18 🟥 `1146.61` | Q19 🟩 `4.85` | Q20 🟩 `144.36` | Q21 🟥 `283.42` | Q22 🟥 `371.82` | Q23 🟦 `245.27` | +| Q24 🟥 `275.45` | Q25 🟥 `329.49` | Q26 🟥 `269.75` | Q27 🟥 `439.36` | Q28 🟥 `957.04` | Q29 🟥 `390.44` | +| Q30 🟦 `239.79` | Q31 🟦 `261.03` | Q32 🟥 `828.89` | Q33 🟥 `369.78` | Q34 🟥 `482.41` | Q35 🟥 `676.27` | +| Q36 🟥 `324.34` | Q37 🟦 `235.02` | Q38 🟩 `69.98` | Q39 🟥 `416.78` | Q40 🟩 `90.83` | Q41 🟩 `32.00` | +| Q42 🟥 `573.07` | | | | | | ### Largest Result Sets | Query | Output | Warm median, ms | SQL | | --- | ---: | ---: | --- | -| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 357.07 | SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT… | -| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 236.84 | SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime… | -| [Q29](benchmarks/queries/query_29.sql) | 990 B | 414.52 | SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(Reso… | -| [Q39](benchmarks/queries/query_39.sql) | 835 B | 426.31 | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (… | -| [Q24](benchmarks/queries/query_24.sql) | 658 B | 273.73 | SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY… | +| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 371.82 | SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT… | +| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 245.27 | SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime… | +| [Q29](benchmarks/queries/query_29.sql) | 990 B | 390.44 | SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(Reso… | +| [Q39](benchmarks/queries/query_39.sql) | 835 B | 416.78 | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (… | +| [Q24](benchmarks/queries/query_24.sql) | 658 B | 275.45 | SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY… | ### Fastest Queries | Query | SQL | Warm median, ms | First run, ms | | --- | --- | ---: | ---: | -| [Q06](benchmarks/queries/query_6.sql) | SELECT MIN(EventDate), MAX(EventDate) FROM hits; | 3.68 | 4.43 | -| [Q00](benchmarks/queries/query_0.sql) | SELECT COUNT(*) FROM hits; | 3.91 | 8.14 | -| [Q19](benchmarks/queries/query_19.sql) | SELECT UserID FROM hits WHERE UserID = 435090932899640449; | 4.11 | 4.50 | -| [Q01](benchmarks/queries/query_1.sql) | SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; | 6.32 | 34.17 | -| [Q07](benchmarks/queries/query_7.sql) | SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY … | 6.69 | 6.40 | +| [Q06](benchmarks/queries/query_6.sql) | SELECT MIN(EventDate), MAX(EventDate) FROM hits; | 4.20 | 4.73 | +| [Q19](benchmarks/queries/query_19.sql) | SELECT UserID FROM hits WHERE UserID = 435090932899640449; | 4.85 | 5.58 | +| [Q00](benchmarks/queries/query_0.sql) | SELECT COUNT(*) FROM hits; | 5.33 | 10.91 | +| [Q07](benchmarks/queries/query_7.sql) | SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY … | 6.58 | 6.60 | +| [Q01](benchmarks/queries/query_1.sql) | SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; | 7.51 | 14.12 | ### Slowest Queries | Query | SQL | Warm median, ms | Warm max, ms | | --- | --- | ---: | ---: | -| [Q18](benchmarks/queries/query_18.sql) | SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT… | 1112.19 | 1178.59 | -| [Q28](benchmarks/queries/query_28.sql) | SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1')… | 907.32 | 910.64 | -| [Q32](benchmarks/queries/query_32.sql) | SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(Resolution… | 787.69 | 790.48 | -| [Q35](benchmarks/queries/query_35.sql) | SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS … | 681.60 | 683.99 | -| [Q42](benchmarks/queries/query_42.sql) | SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM… | 568.52 | 569.02 | +| [Q18](benchmarks/queries/query_18.sql) | SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT… | 1146.61 | 1176.15 | +| [Q28](benchmarks/queries/query_28.sql) | SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1')… | 957.04 | 967.25 | +| [Q32](benchmarks/queries/query_32.sql) | SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(Resolution… | 828.89 | 953.89 | +| [Q35](benchmarks/queries/query_35.sql) | SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS … | 676.27 | 678.94 | +| [Q42](benchmarks/queries/query_42.sql) | SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM… | 573.07 | 574.43 | ### Query Table | Query | Output CSV | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | -| [Q00](benchmarks/queries/query_0.sql) | 7 B | 8.14 | 3.95 | 3.91 | 3.78 | 4.22 | ok | -| [Q01](benchmarks/queries/query_1.sql) | 6 B | 34.17 | 6.33 | 6.32 | 6.01 | 6.67 | ok | -| [Q02](benchmarks/queries/query_2.sql) | 18 B | 48.00 | 17.26 | 17.25 | 16.83 | 17.71 | ok | -| [Q03](benchmarks/queries/query_3.sql) | 20 B | 49.77 | 13.43 | 13.37 | 13.21 | 13.76 | ok | -| [Q04](benchmarks/queries/query_4.sql) | 6 B | 148.05 | 149.27 | 148.12 | 147.47 | 153.38 | ok | -| [Q05](benchmarks/queries/query_5.sql) | 6 B | 165.31 | 130.88 | 130.96 | 130.57 | 131.02 | ok | -| [Q06](benchmarks/queries/query_6.sql) | 22 B | 4.43 | 3.69 | 3.68 | 3.60 | 3.82 | ok | -| [Q07](benchmarks/queries/query_7.sql) | 31 B | 6.40 | 6.80 | 6.69 | 6.46 | 7.35 | ok | -| [Q08](benchmarks/queries/query_8.sql) | 81 B | 230.36 | 193.81 | 193.36 | 191.19 | 197.33 | ok | -| [Q09](benchmarks/queries/query_9.sql) | 237 B | 210.07 | 210.02 | 209.45 | 209.09 | 212.08 | ok | -| [Q10](benchmarks/queries/query_10.sql) | 102 B | 299.72 | 268.08 | 268.08 | 267.32 | 268.86 | ok | -| [Q11](benchmarks/queries/query_11.sql) | 116 B | 287.55 | 270.30 | 269.48 | 268.57 | 273.67 | ok | -| [Q12](benchmarks/queries/query_12.sql) | 387 B | 229.11 | 232.94 | 233.32 | 230.58 | 234.54 | ok | -| [Q13](benchmarks/queries/query_13.sql) | 350 B | 247.57 | 246.07 | 245.55 | 244.77 | 248.40 | ok | -| [Q14](benchmarks/queries/query_14.sql) | 400 B | 249.88 | 237.64 | 238.18 | 235.56 | 238.64 | ok | -| [Q15](benchmarks/queries/query_15.sql) | 236 B | 75.58 | 73.07 | 73.32 | 71.09 | 74.55 | ok | -| [Q16](benchmarks/queries/query_16.sql) | 244 B | 130.68 | 122.37 | 122.39 | 121.98 | 122.72 | ok | -| [Q17](benchmarks/queries/query_17.sql) | 414 B | 155.01 | 144.52 | 144.28 | 142.55 | 146.97 | ok | -| [Q18](benchmarks/queries/query_18.sql) | 264 B | 1288.46 | 1127.01 | 1112.19 | 1105.08 | 1178.59 | ok | -| [Q19](benchmarks/queries/query_19.sql) | 0 B | 4.50 | 4.34 | 4.11 | 4.01 | 5.15 | ok | -| [Q20](benchmarks/queries/query_20.sql) | 3 B | 227.21 | 135.07 | 134.97 | 134.86 | 135.50 | ok | -| [Q21](benchmarks/queries/query_21.sql) | 162 B | 266.42 | 268.70 | 268.11 | 264.90 | 273.70 | ok | -| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 471.23 | 356.98 | 357.07 | 354.97 | 358.80 | ok | -| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 272.00 | 236.75 | 236.84 | 235.63 | 237.68 | ok | -| [Q24](benchmarks/queries/query_24.sql) | 658 B | 280.44 | 274.08 | 273.73 | 272.01 | 276.87 | ok | -| [Q25](benchmarks/queries/query_25.sql) | 382 B | 226.93 | 226.56 | 226.36 | 226.06 | 227.44 | ok | -| [Q26](benchmarks/queries/query_26.sql) | 658 B | 272.76 | 300.66 | 289.97 | 271.68 | 351.01 | ok | -| [Q27](benchmarks/queries/query_27.sql) | 26 B | 449.09 | 398.29 | 398.98 | 394.65 | 400.55 | ok | -| [Q28](benchmarks/queries/query_28.sql) | 556 B | 1015.82 | 907.49 | 907.32 | 904.67 | 910.64 | ok | -| [Q29](benchmarks/queries/query_29.sql) | 990 B | 414.40 | 414.81 | 414.52 | 414.18 | 416.03 | ok | -| [Q30](benchmarks/queries/query_30.sql) | 244 B | 290.88 | 236.69 | 237.07 | 234.72 | 237.89 | ok | -| [Q31](benchmarks/queries/query_31.sql) | 398 B | 271.91 | 261.89 | 262.05 | 258.68 | 264.77 | ok | -| [Q32](benchmarks/queries/query_32.sql) | 376 B | 1001.96 | 785.02 | 787.69 | 774.24 | 790.48 | ok | -| [Q33](benchmarks/queries/query_33.sql) | 464 B | 391.70 | 454.24 | 419.25 | 390.48 | 587.97 | ok | -| [Q34](benchmarks/queries/query_34.sql) | 484 B | 436.48 | 424.10 | 423.99 | 413.11 | 435.32 | ok | -| [Q35](benchmarks/queries/query_35.sql) | 493 B | 689.66 | 681.69 | 681.60 | 679.59 | 683.99 | ok | -| [Q36](benchmarks/queries/query_36.sql) | 569 B | 379.10 | 338.62 | 337.69 | 333.24 | 345.84 | ok | -| [Q37](benchmarks/queries/query_37.sql) | 550 B | 256.58 | 244.32 | 244.01 | 243.86 | 245.39 | ok | -| [Q38](benchmarks/queries/query_38.sql) | 480 B | 83.42 | 67.53 | 67.28 | 67.05 | 68.52 | ok | -| [Q39](benchmarks/queries/query_39.sql) | 835 B | 536.54 | 426.07 | 426.31 | 423.11 | 428.55 | ok | -| [Q40](benchmarks/queries/query_40.sql) | 356 B | 113.53 | 92.15 | 92.18 | 91.78 | 92.46 | ok | -| [Q41](benchmarks/queries/query_41.sql) | 132 B | 46.85 | 33.59 | 33.18 | 32.37 | 35.65 | ok | -| [Q42](benchmarks/queries/query_42.sql) | 240 B | 572.27 | 568.26 | 568.52 | 566.96 | 569.02 | ok | +| [Q00](benchmarks/queries/query_0.sql) | 7 B | 10.91 | 5.73 | 5.33 | 5.06 | 7.21 | ok | +| [Q01](benchmarks/queries/query_1.sql) | 6 B | 14.12 | 7.41 | 7.51 | 6.92 | 7.72 | ok | +| [Q02](benchmarks/queries/query_2.sql) | 18 B | 27.96 | 18.37 | 18.46 | 17.43 | 19.11 | ok | +| [Q03](benchmarks/queries/query_3.sql) | 20 B | 24.73 | 14.17 | 14.22 | 13.58 | 14.67 | ok | +| [Q04](benchmarks/queries/query_4.sql) | 6 B | 142.46 | 143.75 | 142.67 | 142.39 | 147.26 | ok | +| [Q05](benchmarks/queries/query_5.sql) | 6 B | 137.41 | 127.97 | 127.92 | 127.70 | 128.33 | ok | +| [Q06](benchmarks/queries/query_6.sql) | 22 B | 4.73 | 4.21 | 4.20 | 4.11 | 4.35 | ok | +| [Q07](benchmarks/queries/query_7.sql) | 31 B | 6.60 | 6.58 | 6.58 | 6.55 | 6.64 | ok | +| [Q08](benchmarks/queries/query_8.sql) | 81 B | 187.86 | 185.13 | 184.89 | 184.61 | 186.12 | ok | +| [Q09](benchmarks/queries/query_9.sql) | 237 B | 199.68 | 200.89 | 200.71 | 200.13 | 202.00 | ok | +| [Q10](benchmarks/queries/query_10.sql) | 102 B | 276.16 | 266.05 | 266.04 | 263.92 | 268.23 | ok | +| [Q11](benchmarks/queries/query_11.sql) | 116 B | 276.72 | 309.33 | 308.16 | 267.94 | 353.04 | ok | +| [Q12](benchmarks/queries/query_12.sql) | 387 B | 410.85 | 229.06 | 228.27 | 225.02 | 234.66 | ok | +| [Q13](benchmarks/queries/query_13.sql) | 350 B | 243.61 | 249.32 | 244.72 | 239.30 | 268.53 | ok | +| [Q14](benchmarks/queries/query_14.sql) | 400 B | 228.50 | 230.86 | 229.88 | 226.83 | 236.84 | ok | +| [Q15](benchmarks/queries/query_15.sql) | 236 B | 74.99 | 69.83 | 69.88 | 69.21 | 70.34 | ok | +| [Q16](benchmarks/queries/query_16.sql) | 244 B | 123.59 | 135.96 | 139.03 | 119.77 | 146.01 | ok | +| [Q17](benchmarks/queries/query_17.sql) | 414 B | 186.73 | 152.43 | 149.92 | 145.57 | 164.29 | ok | +| [Q18](benchmarks/queries/query_18.sql) | 264 B | 1168.73 | 1151.55 | 1146.61 | 1136.85 | 1176.15 | ok | +| [Q19](benchmarks/queries/query_19.sql) | 0 B | 5.58 | 4.83 | 4.85 | 4.65 | 4.96 | ok | +| [Q20](benchmarks/queries/query_20.sql) | 3 B | 188.34 | 144.23 | 144.36 | 142.56 | 145.65 | ok | +| [Q21](benchmarks/queries/query_21.sql) | 162 B | 277.03 | 282.07 | 283.42 | 270.81 | 290.61 | ok | +| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 410.96 | 372.51 | 371.82 | 362.17 | 384.20 | ok | +| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 276.18 | 245.86 | 245.27 | 242.46 | 250.46 | ok | +| [Q24](benchmarks/queries/query_24.sql) | 658 B | 272.21 | 287.04 | 275.45 | 272.69 | 324.55 | ok | +| [Q25](benchmarks/queries/query_25.sql) | 382 B | 287.75 | 410.16 | 329.49 | 263.83 | 717.82 | ok | +| [Q26](benchmarks/queries/query_26.sql) | 658 B | 316.19 | 270.07 | 269.75 | 268.76 | 272.01 | ok | +| [Q27](benchmarks/queries/query_27.sql) | 26 B | 464.09 | 448.40 | 439.36 | 421.28 | 493.62 | ok | +| [Q28](benchmarks/queries/query_28.sql) | 556 B | 1003.27 | 958.87 | 957.04 | 954.14 | 967.25 | ok | +| [Q29](benchmarks/queries/query_29.sql) | 990 B | 395.02 | 390.10 | 390.44 | 387.37 | 392.13 | ok | +| [Q30](benchmarks/queries/query_30.sql) | 244 B | 256.21 | 238.89 | 239.79 | 235.62 | 240.37 | ok | +| [Q31](benchmarks/queries/query_31.sql) | 398 B | 299.84 | 261.53 | 261.03 | 258.93 | 265.14 | ok | +| [Q32](benchmarks/queries/query_32.sql) | 376 B | 986.27 | 854.79 | 828.89 | 807.48 | 953.89 | ok | +| [Q33](benchmarks/queries/query_33.sql) | 464 B | 378.58 | 374.65 | 369.78 | 363.09 | 395.96 | ok | +| [Q34](benchmarks/queries/query_34.sql) | 484 B | 473.26 | 480.15 | 482.41 | 408.94 | 546.83 | ok | +| [Q35](benchmarks/queries/query_35.sql) | 493 B | 676.69 | 675.28 | 676.27 | 669.64 | 678.94 | ok | +| [Q36](benchmarks/queries/query_36.sql) | 569 B | 340.63 | 324.75 | 324.34 | 321.71 | 328.61 | ok | +| [Q37](benchmarks/queries/query_37.sql) | 550 B | 249.18 | 234.70 | 235.02 | 233.13 | 235.62 | ok | +| [Q38](benchmarks/queries/query_38.sql) | 480 B | 75.87 | 70.08 | 69.98 | 69.94 | 70.44 | ok | +| [Q39](benchmarks/queries/query_39.sql) | 835 B | 489.41 | 416.60 | 416.78 | 408.36 | 424.48 | ok | +| [Q40](benchmarks/queries/query_40.sql) | 356 B | 106.21 | 91.04 | 90.83 | 90.28 | 92.23 | ok | +| [Q41](benchmarks/queries/query_41.sql) | 132 B | 37.17 | 32.00 | 32.00 | 31.92 | 32.07 | ok | +| [Q42](benchmarks/queries/query_42.sql) | 240 B | 580.31 | 572.39 | 573.07 | 568.98 | 574.43 | ok | diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index 0a6a1d8..0ad6d04 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,44 +1,44 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,output_csv_bytes,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,8.14,3.95,3.91,3.78,4.22,7,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,34.17,6.33,6.32,6.01,6.67,6,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",48.00,17.26,17.25,16.83,17.71,18,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,49.77,13.43,13.37,13.21,13.76,20,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,148.05,149.27,148.12,147.47,153.38,6,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,165.31,130.88,130.96,130.57,131.02,6,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",4.43,3.69,3.68,3.60,3.82,22,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",6.40,6.80,6.69,6.46,7.35,31,ok, -8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",230.36,193.81,193.36,191.19,197.33,81,ok, -9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",210.07,210.02,209.45,209.09,212.08,237,ok, -10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",299.72,268.08,268.08,267.32,268.86,102,ok, -11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",287.55,270.30,269.48,268.57,273.67,116,ok, -12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",229.11,232.94,233.32,230.58,234.54,387,ok, -13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",247.57,246.07,245.55,244.77,248.40,350,ok, -14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",249.88,237.64,238.18,235.56,238.64,400,ok, -15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",75.58,73.07,73.32,71.09,74.55,236,ok, -16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",130.68,122.37,122.39,121.98,122.72,244,ok, -17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",155.01,144.52,144.28,142.55,146.97,414,ok, -18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",1288.46,1127.01,1112.19,1105.08,1178.59,264,ok, -19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,4.50,4.34,4.11,4.01,5.15,0,ok, -20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,227.21,135.07,134.97,134.86,135.50,3,ok, -21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",266.42,268.70,268.11,264.90,273.70,162,ok, -22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",471.23,356.98,357.07,354.97,358.80,6188,ok, -23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,272.00,236.75,236.84,235.63,237.68,2633,ok, -24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,280.44,274.08,273.73,272.01,276.87,658,ok, -25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,226.93,226.56,226.36,226.06,227.44,382,ok, -26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",272.76,300.66,289.97,271.68,351.01,658,ok, -27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",449.09,398.29,398.98,394.65,400.55,26,ok, -28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",1015.82,907.49,907.32,904.67,910.64,556,ok, -29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",414.40,414.81,414.52,414.18,416.03,990,ok, -30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",290.88,236.69,237.07,234.72,237.89,244,ok, -31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",271.91,261.89,262.05,258.68,264.77,398,ok, -32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",1001.96,785.02,787.69,774.24,790.48,376,ok, -33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",391.70,454.24,419.25,390.48,587.97,464,ok, -34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",436.48,424.10,423.99,413.11,435.32,484,ok, -35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",689.66,681.69,681.60,679.59,683.99,493,ok, -36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",379.10,338.62,337.69,333.24,345.84,569,ok, -37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",256.58,244.32,244.01,243.86,245.39,550,ok, -38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",83.42,67.53,67.28,67.05,68.52,480,ok, -39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",536.54,426.07,426.31,423.11,428.55,835,ok, -40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",113.53,92.15,92.18,91.78,92.46,356,ok, -41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",46.85,33.59,33.18,32.37,35.65,132,ok, -42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",572.27,568.26,568.52,566.96,569.02,240,ok, +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,10.91,5.73,5.33,5.06,7.21,7,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,14.12,7.41,7.51,6.92,7.72,6,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",27.96,18.37,18.46,17.43,19.11,18,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,24.73,14.17,14.22,13.58,14.67,20,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,142.46,143.75,142.67,142.39,147.26,6,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,137.41,127.97,127.92,127.70,128.33,6,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",4.73,4.21,4.20,4.11,4.35,22,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",6.60,6.58,6.58,6.55,6.64,31,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",187.86,185.13,184.89,184.61,186.12,81,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",199.68,200.89,200.71,200.13,202.00,237,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",276.16,266.05,266.04,263.92,268.23,102,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",276.72,309.33,308.16,267.94,353.04,116,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",410.85,229.06,228.27,225.02,234.66,387,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",243.61,249.32,244.72,239.30,268.53,350,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",228.50,230.86,229.88,226.83,236.84,400,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",74.99,69.83,69.88,69.21,70.34,236,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",123.59,135.96,139.03,119.77,146.01,244,ok, +17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",186.73,152.43,149.92,145.57,164.29,414,ok, +18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",1168.73,1151.55,1146.61,1136.85,1176.15,264,ok, +19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,5.58,4.83,4.85,4.65,4.96,0,ok, +20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,188.34,144.23,144.36,142.56,145.65,3,ok, +21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",277.03,282.07,283.42,270.81,290.61,162,ok, +22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",410.96,372.51,371.82,362.17,384.20,6188,ok, +23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,276.18,245.86,245.27,242.46,250.46,2633,ok, +24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,272.21,287.04,275.45,272.69,324.55,658,ok, +25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,287.75,410.16,329.49,263.83,717.82,382,ok, +26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",316.19,270.07,269.75,268.76,272.01,658,ok, +27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",464.09,448.40,439.36,421.28,493.62,26,ok, +28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",1003.27,958.87,957.04,954.14,967.25,556,ok, +29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",395.02,390.10,390.44,387.37,392.13,990,ok, +30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",256.21,238.89,239.79,235.62,240.37,244,ok, +31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",299.84,261.53,261.03,258.93,265.14,398,ok, +32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",986.27,854.79,828.89,807.48,953.89,376,ok, +33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",378.58,374.65,369.78,363.09,395.96,464,ok, +34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",473.26,480.15,482.41,408.94,546.83,484,ok, +35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",676.69,675.28,676.27,669.64,678.94,493,ok, +36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",340.63,324.75,324.34,321.71,328.61,569,ok, +37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",249.18,234.70,235.02,233.13,235.62,550,ok, +38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",75.87,70.08,69.98,69.94,70.44,480,ok, +39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",489.41,416.60,416.78,408.36,424.48,835,ok, +40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",106.21,91.04,90.83,90.28,92.23,356,ok, +41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",37.17,32.00,32.00,31.92,32.07,132,ok, +42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",580.31,572.39,573.07,568.98,574.43,240,ok, From ed0155574efc71a6623709ec832280e0603a86f8 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 14:22:17 +0300 Subject: [PATCH 04/13] fix: split operators + fix issues with offset --- include/executor/operators_internal.h | 41 + src/CMakeLists.txt | 3 + src/executor/operators.cpp | 1644 +------------------------ src/executor/operators_aggregate.cpp | 729 +++++++++++ src/executor/operators_runtime.cpp | 802 ++++++++++++ src/executor/operators_scan.cpp | 232 ++++ tests/test_executor.cpp | 68 +- 7 files changed, 1888 insertions(+), 1631 deletions(-) create mode 100644 include/executor/operators_internal.h create mode 100644 src/executor/operators_aggregate.cpp create mode 100644 src/executor/operators_runtime.cpp create mode 100644 src/executor/operators_scan.cpp diff --git a/include/executor/operators_internal.h b/include/executor/operators_internal.h new file mode 100644 index 0000000..47dee2f --- /dev/null +++ b/include/executor/operators_internal.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "executor/operator.h" + +std::unique_ptr CreateMetadataCountOperator(std::filesystem::path path, std::string output_name); +std::unique_ptr CreateMetadataExtremaOperator(std::filesystem::path path, std::vector aggregates); +std::unique_ptr CreateScanOperator(std::filesystem::path path, std::vector projection_indexes, + PredicatePtr filter); +std::unique_ptr CreateFilterOperator(std::unique_ptr child, PredicatePtr filter); +std::unique_ptr CreateEnsureSchemaOperator(std::unique_ptr child, Schema schema); +std::unique_ptr CreateProjectionOperator(std::unique_ptr child, std::vector items, + Schema source_schema); +std::unique_ptr CreateAggOperator(std::unique_ptr child, std::vector aggregates, + PredicatePtr filter); +std::unique_ptr CreateGroupAggOperator(std::unique_ptr child, + std::vector group_keys, + const std::vector& aggregates, + std::vector select_items, PredicatePtr filter, + PredicatePtr having, bool sort_by_group_keys); +std::unique_ptr CreateGroupAggTopKOperator(std::unique_ptr child, + std::vector group_keys, + const std::vector& aggregates, + std::vector select_items, PredicatePtr filter, + std::vector order_by, size_t limit); + +void ApplyOrderOffsetLimit(std::unique_ptr& root, const PlannedQuery& planned, bool& limit_applied_by_top_k); +Schema ProjectionOutputSchema(const std::vector& items, const Schema& source_schema); + +std::string NormalizeLiteralForEval(const QueryLiteral& literal); +std::optional TryFindBatchColumn(const Schema& schema, std::string_view name); +std::string EvalExpr(const ExprPtr& expr, const Batch& batch, size_t row); +bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, size_t row); +void SelectRowsMatchingPredicate(const PredicatePtr& predicate, const Batch& batch, std::vector& rows); +Schema BuildSelectOutputSchema(const std::vector& select_items); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index efb6711..72b2d7b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,6 +58,9 @@ add_library(columnar_engine_executor executor/query_planner.cpp executor/aggregate_states.cpp executor/operators.cpp + executor/operators_aggregate.cpp + executor/operators_runtime.cpp + executor/operators_scan.cpp sql_parser/tokenizer_factory.cpp sql_parser/tokenizer_tokens.cpp sql_parser/tokenizer.cpp diff --git a/src/executor/operators.cpp b/src/executor/operators.cpp index aac847c..23d2aac 100644 --- a/src/executor/operators.cpp +++ b/src/executor/operators.cpp @@ -1,1642 +1,29 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/ascii.h" -#include "common/error.h" -#include "common/int128.h" -#include "common/operator_utils.h" -#include "common/parsing.h" -#include "common/planner_utils.h" -#include "executor/aggregate_state.h" #include "executor/operator.h" -#include "io/columnar_batch.h" -#include "io/file.h" - -constexpr std::string_view CountStarName = "COUNT(*)"; -constexpr std::string_view FallbackAggregateAlias = "c"; -constexpr std::string_view PageViewsAlias = "PageViews"; -constexpr std::string_view ExtractMinutePart = "MINUTE"; -constexpr std::string_view ExtractHourPart = "HOUR"; - -constexpr int64_t MinuteMicros = 60'000'000; - -constexpr std::string_view HttpScheme = "http://"; -constexpr std::string_view HttpsScheme = "https://"; -constexpr std::string_view WwwPrefix = "www."; -constexpr std::string_view RegexBackrefPlaceholder = "\\1"; -constexpr std::string_view RegexBackrefReplacement = "$1"; - -class MetadataCountOperator final : public Operator { - public: - MetadataCountOperator(std::filesystem::path path, std::string output_name) - : path_(std::move(path)), output_name_(std::move(output_name)) {} - - std::optional Next() override { - if (returned_) { - return std::nullopt; - } - - returned_ = true; - - uint64_t rows = 0; - - const ColumnarBatchReader reader(path_); - for (const auto& row_group : reader.GetMetadata().row_groups) { - rows += row_group.row_count; - } - - Batch result(Schema{{ColumnSchema(output_name_, ColumnType::Int64)}}, 1); - result.AppendValueFromString(0, std::to_string(rows)); - - return result; - } - - private: - std::filesystem::path path_; - std::string output_name_; - bool returned_ = false; -}; - -class MetadataExtremaOperator final : public Operator { - public: - MetadataExtremaOperator(std::filesystem::path path, std::vector aggregates) - : path_(std::move(path)), aggregates_(std::move(aggregates)) {} - - std::optional Next() override { - if (returned_) { - return std::nullopt; - } - - returned_ = true; - - const ColumnarBatchReader reader(path_); - const ColumnarMetadata& metadata = reader.GetMetadata(); - - Schema schema; - schema.columns.reserve(aggregates_.size()); - - std::vector> states; - states.reserve(aggregates_.size()); - - for (const PlannedAgg& aggregate : aggregates_) { - schema.columns.push_back(ColumnSchema(aggregate.output_name, AggregateOutputType(aggregate))); - states.push_back(CreateAggState(aggregate)); - } - - for (const auto& row_group : metadata.row_groups) { - for (size_t i = 0; i < aggregates_.size(); ++i) { - const PlannedAgg& aggregate = aggregates_[i]; - const auto& chunk = row_group.columns[aggregate.column_index]; - - if (!chunk.has_min_max) { - throw Error::InvalidState("executor", "metadata min/max statistics are missing", path_.string()); - } - - const std::string name = ToUpperAscii(aggregate.function->canonical_name); - states[i]->ConsumeInt128(name == "MIN" ? chunk.min_value : chunk.max_value); - } - } - - Batch result(schema); - for (size_t i = 0; i < states.size(); ++i) { - result.AppendValueFromString(i, states[i]->Finalize()); - } - - return result; - } - - private: - std::filesystem::path path_; - std::vector aggregates_; - bool returned_ = false; -}; - -static bool MatchesRowGroupComparison(const ColumnChunkMetadata& chunk, const Int128 value, - const ComparisonKind comparison) { - if (!chunk.has_min_max) { - return true; - } - - switch (comparison) { - case ComparisonKind::Equal: - return chunk.min_value <= value && value <= chunk.max_value; - case ComparisonKind::NotEqual: - return chunk.min_value != value || chunk.max_value != value; - case ComparisonKind::Less: - return chunk.min_value < value; - case ComparisonKind::LessOrEqual: - return chunk.min_value <= value; - case ComparisonKind::Greater: - return chunk.max_value > value; - case ComparisonKind::GreaterOrEqual: - return chunk.max_value >= value; - } - - return true; -} - -static bool MayMatchRowGroup(const PredicatePtr& predicate, const RowGroupMetadata& row_group) { - if (!predicate) { - return true; - } - - switch (predicate->kind) { - case PredicateKind::And: - return MayMatchRowGroup(predicate->lhs, row_group) && MayMatchRowGroup(predicate->rhs, row_group); - case PredicateKind::Comparison: - if (!predicate->metadata_typed_literal_comparison_bound || - predicate->metadata_typed_column_index >= row_group.columns.size()) { - return true; - } - return MatchesRowGroupComparison(row_group.columns[predicate->metadata_typed_column_index], - predicate->metadata_typed_literal_value, - predicate->metadata_typed_comparison); - case PredicateKind::In: - if (!predicate->metadata_typed_in_set_bound || - predicate->metadata_typed_in_column_index >= row_group.columns.size()) { - return true; - } - - return std::ranges::any_of(predicate->metadata_typed_in_values, [&](const Int128 value) { - return MatchesRowGroupComparison(row_group.columns[predicate->metadata_typed_in_column_index], value, - ComparisonKind::Equal); - }); - case PredicateKind::Like: - case PredicateKind::NotLike: - return true; - } - - return true; -} - -class ScanOperator final : public Operator { - public: - ScanOperator(std::filesystem::path path, std::vector projection_indexes, PredicatePtr filter) - : path_(std::move(path)), - input_(path_), - metadata_(ColumnarBatchReader(path_).GetMetadata()), - filter_(std::move(filter)) { - if (metadata_.schema.columns.empty()) { - throw Error::MalformedData("executor", "columnar schema is empty", path_.string()); - } - - projection_indexes_ = std::move(projection_indexes); - projected_schema_.columns.reserve(projection_indexes_.size()); - - for (const size_t source_index : projection_indexes_) { - if (source_index >= metadata_.schema.columns.size()) { - throw Error::OutOfRange("executor", "projection index out of range", path_.string()); - } - - projected_schema_.columns.push_back(metadata_.schema.columns[source_index]); - } - } - - std::optional Next() override { - if (next_group_ >= metadata_.row_groups.size()) { - return std::nullopt; - } - - const RowGroupMetadata* row_group = nullptr; - while (next_group_ < metadata_.row_groups.size()) { - const RowGroupMetadata& candidate = metadata_.row_groups[next_group_++]; - if (MayMatchRowGroup(filter_, candidate)) { - row_group = &candidate; - break; - } - } - - if (row_group == nullptr) { - return std::nullopt; - } - - Batch batch(projected_schema_, row_group->row_count); - - for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { - const size_t source_index = projection_indexes_[projected_index]; - const auto& chunk = row_group->columns[source_index]; - - ReadBatchColumnChunk(path_, input_, chunk, row_group->row_count, batch, projected_index); - } - - return batch; - } - - private: - std::filesystem::path path_; - InputFile input_; - - ColumnarMetadata metadata_; - std::vector projection_indexes_; - Schema projected_schema_; - PredicatePtr filter_; - size_t next_group_ = 0; -}; - -static std::string NormalizeLiteralForEval(const QueryLiteral& literal) { - if (literal.kind != LiteralKind::String) { - return literal.text; - } - - return UnescapeSqlString(literal.text); -} - -static std::optional TryFindBatchColumn(const Schema& schema, const std::string_view name) { - for (size_t i = 0; i < schema.columns.size(); ++i) { - if (SameColumnName(schema.columns[i].name, name)) { - return i; - } - } - return std::nullopt; -} - -static Int128 EvalInt(const std::string& value) { return ParseInt128(value); } - -static bool IsAggregateLookupFunction(const std::string_view name) { - return name == "COUNT" || name == "SUM" || name == "AVG" || name == "MIN" || name == "MAX"; -} - -static std::string FormatAggregateExprName(const ExprSpec& expr) { - const std::string name = ToUpperAscii(expr.function_name); - if (name == "COUNT" && !expr.arguments.empty() && expr.arguments.front()->kind == ExprKind::Star) { - return std::string(CountStarName); - } - - std::string out = name + "("; - - for (size_t i = 0; i < expr.arguments.size(); ++i) { - if (i > 0) { - out += ", "; - } - out += expr.arguments[i]->output_name; - } - - out += ")"; - - return out; -} - -static std::string EvalExpr(const ExprPtr& expr, const Batch& batch, size_t row); -static bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, size_t row); - -static std::optional TryResolveAggregateValue(const ExprSpec& expr, const Batch& batch, const size_t row) { - const std::string name = ToUpperAscii(expr.function_name); - if (!IsAggregateLookupFunction(name)) { - return std::nullopt; - } - - const std::string formatted_name = FormatAggregateExprName(expr); - for (const std::string_view candidate_name : - {std::string_view{formatted_name}, FallbackAggregateAlias, PageViewsAlias}) { - if (const auto column = TryFindBatchColumn(batch.GetSchema(), candidate_name); column.has_value()) { - return batch.ColumnAt(*column).ValueAsString(row); - } - } - - return std::nullopt; -} - -static std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t row) { - if (const auto aggregate_value = TryResolveAggregateValue(expr, batch, row); aggregate_value.has_value()) { - return std::move(*aggregate_value); - } - - const std::string name = ToUpperAscii(expr.function_name); - - if (name == "STRLEN" || name == "LENGTH") { - return std::to_string(EvalExpr(expr.arguments.at(0), batch, row).size()); - } - - if (name == "EXTRACT") { - const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); - const int64_t ts = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); - - const std::chrono::sys_time time{std::chrono::microseconds{ts}}; - const auto day = std::chrono::floor(time); - - const std::chrono::hh_mm_ss tod{time - day}; - - if (part == ExtractMinutePart) { - return std::to_string(tod.minutes().count()); - } - if (part == ExtractHourPart) { - return std::to_string(tod.hours().count()); - } - - throw Error::Unsupported("executor", "unsupported EXTRACT part"); - } - - if (name == "DATE_TRUNC") { - const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); - int64_t micros = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); - - if (part == ExtractMinutePart) { - micros = micros / MinuteMicros * MinuteMicros; - return TimestampToString(micros); - } - - throw Error::Unsupported("executor", "unsupported DATE_TRUNC part"); - } - - if (name == "REGEXP_REPLACE") { - const std::string source = EvalExpr(expr.arguments.at(0), batch, row); - const std::string pattern_text = EvalExpr(expr.arguments.at(1), batch, row); - std::string replacement = EvalExpr(expr.arguments.at(2), batch, row); - - if (pattern_text == R"(^https?://(?:www\.)?([^/]+)/.*$)" && replacement == R"(\1)") { - std::string_view rest = source; - if (rest.starts_with(HttpScheme)) { - rest.remove_prefix(HttpScheme.size()); - } else if (rest.starts_with(HttpsScheme)) { - rest.remove_prefix(HttpsScheme.size()); - } else { - return source; - } - - if (rest.starts_with(WwwPrefix)) { - rest.remove_prefix(WwwPrefix.size()); - } - - const size_t slash = rest.find('/'); - if (slash == std::string_view::npos || slash == 0) { - return source; - } - - const std::string_view suffix = rest.substr(slash + 1); - if (suffix.find('\n') != std::string_view::npos || suffix.find('\r') != std::string_view::npos) { - return source; - } - - return std::string(rest.substr(0, slash)); - } - - if (expr.regex_replace_bound) { - return std::regex_replace(source, expr.regex_pattern, expr.regex_replacement); - } - - for (size_t pos = 0; (pos = replacement.find(RegexBackrefPlaceholder, pos)) != std::string::npos;) { - replacement.replace(pos, RegexBackrefPlaceholder.size(), RegexBackrefReplacement); - pos += RegexBackrefReplacement.size(); - } - - const std::regex pattern(pattern_text); - - return std::regex_replace(source, pattern, replacement); - } - - throw Error::Unsupported("executor", "unsupported function '" + expr.function_name + "'"); -} - -static std::string EvalExpr(const ExprPtr& expr, const Batch& batch, const size_t row) { - switch (expr->kind) { - case ExprKind::Column: { - if (expr->column_index_bound && expr->column_index < batch.ColumnsCount() && - SameColumnName(batch.GetSchema().columns[expr->column_index].name, expr->column.name)) { - return batch.ColumnAt(expr->column_index).ValueAsString(row); - } - - const auto column = TryFindBatchColumn(batch.GetSchema(), expr->column.name); - if (!column.has_value()) { - throw Error::InvalidArgument("executor", "unknown column '" + expr->column.name + "'"); - } - - return batch.ColumnAt(*column).ValueAsString(row); - } - case ExprKind::Literal: - return NormalizeLiteralForEval(expr->literal); - case ExprKind::Binary: { - const Int128 lhs = EvalInt(EvalExpr(expr->left, batch, row)); - const Int128 rhs = EvalInt(EvalExpr(expr->right, batch, row)); - return Int128ToString(expr->binary_op == BinaryOp::Add ? lhs + rhs : lhs - rhs); - } - case ExprKind::Function: - return EvalFunction(*expr, batch, row); - case ExprKind::Case: - return EvaluatePredicate(expr->case_spec.condition, batch, row) - ? EvalExpr(expr->case_spec.then_expr, batch, row) - : EvalExpr(expr->case_spec.else_expr, batch, row); - case ExprKind::Star: - return "*"; - } - return {}; -} - -static ComparisonKind FlipComparison(const ComparisonKind comparison) { - switch (comparison) { - case ComparisonKind::Equal: - return ComparisonKind::Equal; - case ComparisonKind::NotEqual: - return ComparisonKind::NotEqual; - case ComparisonKind::Less: - return ComparisonKind::Greater; - case ComparisonKind::LessOrEqual: - return ComparisonKind::GreaterOrEqual; - case ComparisonKind::Greater: - return ComparisonKind::Less; - case ComparisonKind::GreaterOrEqual: - return ComparisonKind::LessOrEqual; - } - - return comparison; -} - -static bool MatchesInt128Comparison(const Int128 lhs, const Int128 rhs, const ComparisonKind comparison) { - switch (comparison) { - case ComparisonKind::Equal: - return lhs == rhs; - case ComparisonKind::NotEqual: - return lhs != rhs; - case ComparisonKind::Less: - return lhs < rhs; - case ComparisonKind::LessOrEqual: - return lhs <= rhs; - case ComparisonKind::Greater: - return lhs > rhs; - case ComparisonKind::GreaterOrEqual: - return lhs >= rhs; - } - - return false; -} - -static ValueComparison ToValueComparison(const ComparisonKind comparison) { - switch (comparison) { - case ComparisonKind::Equal: - return ValueComparison::Equal; - case ComparisonKind::NotEqual: - return ValueComparison::NotEqual; - case ComparisonKind::Less: - return ValueComparison::Less; - case ComparisonKind::LessOrEqual: - return ValueComparison::LessOrEqual; - case ComparisonKind::Greater: - return ValueComparison::Greater; - case ComparisonKind::GreaterOrEqual: - return ValueComparison::GreaterOrEqual; - } - - return ValueComparison::Equal; -} - -static std::optional LiteralValueForColumnType(const QueryLiteral& literal, const ColumnType type) { - try { - if (literal.kind == LiteralKind::Numeric) { - return ParseInt128(literal.text); - } - - if (literal.kind == LiteralKind::String) { - const QueryLiteral normalized_literal{NormalizeLiteralForEval(literal), LiteralKind::String}; - if (type == ColumnType::Date) { - return ParseDate(normalized_literal.text); - } - if (type == ColumnType::Timestamp) { - return ParseTimestamp(normalized_literal.text); - } - } - } catch (const Error&) { - return std::nullopt; - } - - return std::nullopt; -} - -static std::optional TryEvaluateTypedComparison(const ExprPtr& left, const ExprPtr& right, const Batch& batch, - const size_t row, const ComparisonKind comparison) { - if (!left || !right) { - return std::nullopt; - } - - if (left->kind != ExprKind::Column || right->kind != ExprKind::Literal) { - if (left->kind == ExprKind::Literal && right->kind == ExprKind::Column) { - return TryEvaluateTypedComparison(right, left, batch, row, FlipComparison(comparison)); - } - return std::nullopt; - } - - const auto column_index = TryFindBatchColumn(batch.GetSchema(), left->column.name); - if (!column_index.has_value()) { - return std::nullopt; - } - - const Column& column = batch.ColumnAt(*column_index); - if (column.Type() == ColumnType::String) { - return std::nullopt; - } - - const std::optional rhs = LiteralValueForColumnType(right->literal, column.Type()); - if (!rhs.has_value()) { - return std::nullopt; - } - - return MatchesInt128Comparison(column.ValueAsInt128(row), *rhs, comparison); -} - -static ColumnType InferComparisonType(const std::string& lhs, const std::string& rhs) { - if (TryParseInt128(lhs).has_value() && TryParseInt128(rhs).has_value()) { - return ColumnType::Int128; - } - - if (TryParseDate(lhs).has_value() && TryParseDate(rhs).has_value()) { - return ColumnType::Date; - } - - if (TryParseTimestamp(lhs).has_value() && TryParseTimestamp(rhs).has_value()) { - return ColumnType::Timestamp; - } - - return ColumnType::String; -} - -static bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, const size_t row) { - if (!predicate) { - return true; - } - - switch (predicate->kind) { - case PredicateKind::And: - return EvaluatePredicate(predicate->lhs, batch, row) && EvaluatePredicate(predicate->rhs, batch, row); - case PredicateKind::Like: { - return LikeMatches(EvalExpr(predicate->left, batch, row), EvalExpr(predicate->right, batch, row)); - } - case PredicateKind::NotLike: - return !LikeMatches(EvalExpr(predicate->left, batch, row), EvalExpr(predicate->right, batch, row)); - case PredicateKind::In: { - if (predicate->literal_in_set_bound && predicate->in_column_index < batch.ColumnsCount()) { - return predicate->literal_in_values.contains( - batch.ColumnAt(predicate->in_column_index).ValueAsString(row)); - } - - const std::string lhs = EvalExpr(predicate->left, batch, row); - - return std::ranges::any_of(predicate->values, - [&](const ExprPtr& value) { return lhs == EvalExpr(value, batch, row); }); - } - case PredicateKind::Comparison: { - if (predicate->typed_literal_comparison_bound && predicate->typed_column_index < batch.ColumnsCount()) { - return MatchesInt128Comparison(batch.ColumnAt(predicate->typed_column_index).ValueAsInt128(row), - predicate->typed_literal_value, predicate->typed_comparison); - } - - if (const std::optional result = - TryEvaluateTypedComparison(predicate->left, predicate->right, batch, row, predicate->comparison); - result.has_value()) { - return *result; - } - - const std::string lhs = EvalExpr(predicate->left, batch, row); - const std::string rhs = EvalExpr(predicate->right, batch, row); - - return MatchesComparison(InferComparisonType(lhs, rhs), lhs, rhs, predicate->comparison); - } - } - - return false; -} - -static void SelectRowsMatchingPredicate(const PredicatePtr& predicate, const Batch& batch, std::vector& rows) { - rows.clear(); - rows.reserve(batch.RowsCount()); - - if (!predicate) { - for (size_t row = 0; row < batch.RowsCount(); ++row) { - rows.push_back(row); - } - return; - } - - if (predicate->kind == PredicateKind::Comparison && predicate->typed_literal_comparison_bound && - predicate->typed_column_index < batch.ColumnsCount()) { - batch.ColumnAt(predicate->typed_column_index) - .SelectRowsByInt128Comparison(predicate->typed_literal_value, - ToValueComparison(predicate->typed_comparison), rows); - return; - } - - if ((predicate->kind == PredicateKind::Like || predicate->kind == PredicateKind::NotLike) && - predicate->literal_like_pattern_bound && predicate->like_column_index < batch.ColumnsCount()) { - batch.ColumnAt(predicate->like_column_index) - .SelectRowsByLikePattern(predicate->like_pattern, predicate->like_negated, rows); - return; - } - - if (predicate->kind == PredicateKind::In && predicate->literal_in_set_bound && - predicate->in_column_index < batch.ColumnsCount()) { - batch.ColumnAt(predicate->in_column_index).SelectRowsByStringSet(predicate->literal_in_values, rows); - return; - } - - for (size_t row = 0; row < batch.RowsCount(); ++row) { - if (EvaluatePredicate(predicate, batch, row)) { - rows.push_back(row); - } - } -} - -template -static Schema BuildAggregateOutputSchema(const std::vector& bindings) { - Schema schema; - schema.columns.reserve(bindings.size()); - - for (const auto& binding : bindings) { - schema.columns.push_back(ColumnSchema(binding.aggregate.output_name, AggregateOutputType(binding.aggregate))); - } - - return schema; -} - -static Schema BuildSelectOutputSchema(const std::vector& select_items) { - Schema schema; - schema.columns.reserve(select_items.size()); - - for (const auto& item : select_items) { - schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); - } - - return schema; -} - -class FilterOperator final : public Operator { - public: - FilterOperator(std::unique_ptr child, PredicatePtr filter) - : child_(std::move(child)), filter_(std::move(filter)) {} - - std::optional Next() override { - while (auto batch = child_->Next()) { - std::vector selected_rows; - SelectRowsMatchingPredicate(filter_, *batch, selected_rows); - - if (selected_rows.empty()) { - continue; - } - - Batch filtered(batch->GetSchema(), selected_rows.size()); - filtered.AppendRowsSelectedFromBatch(*batch, selected_rows); - - return filtered; - } - - return std::nullopt; - } - - private: - std::unique_ptr child_; - PredicatePtr filter_; -}; - -class EnsureSchemaOperator final : public Operator { - public: - EnsureSchemaOperator(std::unique_ptr child, Schema schema) - : child_(std::move(child)), schema_(std::move(schema)) {} - - std::optional Next() override { - if (returned_empty_) { - return std::nullopt; - } - - if (auto batch = child_->Next()) { - returned_any_ = true; - return batch; - } - - if (returned_any_) { - return std::nullopt; - } - - returned_any_ = true; - returned_empty_ = true; - - return Batch(schema_); - } - - private: - std::unique_ptr child_; - Schema schema_; - bool returned_any_ = false; - bool returned_empty_ = false; -}; - -static bool UsesRowAggInput(const PlannedAgg& aggregate) { - return !aggregate.direct_numeric_argument && aggregate.argument_kind != AggArgumentKind::Column; -} - -static void ConsumeAggRow(const PlannedAgg& aggregate, const Batch& batch, const size_t row, AggState& state) { - if (aggregate.direct_numeric_argument) { - state.ConsumeInt128(batch.ColumnAt(aggregate.column_index).ValueAsInt128(row) + - aggregate.direct_numeric_offset); - return; - } - - if (aggregate.argument_kind == AggArgumentKind::Column) { - state.ConsumeValue(EvalExpr(aggregate.argument, batch, row)); - return; - } - - state.ConsumeRow(); -} - -static void ConsumeAggRows(const PlannedAgg& aggregate, const Batch& batch, const std::span rows, - AggState& state) { - if (UsesRowAggInput(aggregate)) { - state.ConsumeRows(rows.size()); - return; - } - - for (const size_t row : rows) { - ConsumeAggRow(aggregate, batch, row, state); - } -} - -struct AggBinding { - PlannedAgg aggregate; - std::unique_ptr state; -}; - -struct GroupKeyComponent { - ColumnType type = ColumnType::String; - Int128 int_value = 0; - std::string string_value; -}; - -static Int128 ParseGroupKeyIntValue(const ColumnType type, const std::string_view value) { - switch (type) { - case ColumnType::Boolean: - return ParseBoolean(value) ? 1 : 0; - case ColumnType::Int16: - return ParseInt16(value); - case ColumnType::Int32: - return ParseInt32(value); - case ColumnType::Int64: - return ParseInt64(value); - case ColumnType::Int128: - return ParseInt128(value); - case ColumnType::Date: - return ParseDate(value); - case ColumnType::Timestamp: - return ParseTimestamp(value); - case ColumnType::Character: - return ParseCharacter(value); - case ColumnType::String: - break; - } - - throw Error::Unsupported("executor", "unsupported group key type"); -} - -static std::string FormatGroupKeyValue(const GroupKeyComponent& value) { - switch (value.type) { - case ColumnType::Boolean: - return BooleanToString(value.int_value != 0); - case ColumnType::Int16: - case ColumnType::Int32: - case ColumnType::Int64: - case ColumnType::Int128: - return Int128ToString(value.int_value); - case ColumnType::Date: - return DateToString(static_cast(value.int_value)); - case ColumnType::Timestamp: - return TimestampToString(static_cast(value.int_value)); - case ColumnType::Character: - return std::string(1, static_cast(value.int_value)); - case ColumnType::String: - return value.string_value; - } - - return {}; -} - -static std::strong_ordering CompareGroupKeyValue(const GroupKeyComponent& lhs, const GroupKeyComponent& rhs) { - if (lhs.type != rhs.type) { - throw Error::InconsistentData("executor", "group key type mismatch"); - } - - if (lhs.type == ColumnType::String) { - return lhs.string_value <=> rhs.string_value; - } - - return lhs.int_value <=> rhs.int_value; -} - -static bool GroupKeyValueEquals(const GroupKeyComponent& lhs, const GroupKeyComponent& rhs) { - return CompareGroupKeyValue(lhs, rhs) == 0; -} - -static size_t HashCombine(const size_t seed, const size_t value) { - return seed ^ (value + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2)); -} - -static size_t HashInt128(const Int128 value) { - const auto bits = static_cast(value); - const uint64_t low = static_cast(bits); - const uint64_t high = static_cast(bits >> 64); - - return HashCombine(std::hash{}(high), std::hash{}(low)); -} - -struct TypedGroupKey { - std::vector values; -}; - -struct TypedGroupKeyHash { - size_t operator()(const TypedGroupKey& key) const { - size_t hash = 0; - - for (const GroupKeyComponent& value : key.values) { - hash = HashCombine(hash, std::hash{}(static_cast(value.type))); - hash = HashCombine(hash, value.type == ColumnType::String ? std::hash{}(value.string_value) - : HashInt128(value.int_value)); - } - - return hash; - } -}; - -struct TypedGroupKeyEqual { - bool operator()(const TypedGroupKey& lhs, const TypedGroupKey& rhs) const { - if (lhs.values.size() != rhs.values.size()) { - return false; - } - - for (size_t i = 0; i < lhs.values.size(); ++i) { - if (!GroupKeyValueEquals(lhs.values[i], rhs.values[i])) { - return false; - } - } - - return true; - } -}; - -class GroupKeyMaterializer { - public: - explicit GroupKeyMaterializer(std::vector group_keys) : group_keys_(std::move(group_keys)) {} - - TypedGroupKey Materialize(const Batch& batch, const size_t row) const { - TypedGroupKey key; - key.values.reserve(group_keys_.size()); - - for (const auto& group_key : group_keys_) { - if (group_key.expression && group_key.expression->kind == ExprKind::Column && - group_key.expression->column_index_bound && group_key.expression->column_index < batch.ColumnsCount()) { - const Column& column = batch.ColumnAt(group_key.expression->column_index); - if (group_key.column_type == ColumnType::String) { - key.values.push_back(GroupKeyComponent{ - .type = group_key.column_type, - .int_value = 0, - .string_value = column.ValueAsString(row), - }); - } else { - key.values.push_back(GroupKeyComponent{ - .type = group_key.column_type, - .int_value = column.ValueAsInt128(row), - .string_value = {}, - }); - } - continue; - } - - std::string value = EvalExpr(group_key.expression, batch, row); - if (group_key.column_type == ColumnType::String) { - key.values.push_back(GroupKeyComponent{ - .type = group_key.column_type, - .int_value = 0, - .string_value = std::move(value), - }); - } else { - key.values.push_back(GroupKeyComponent{ - .type = group_key.column_type, - .int_value = ParseGroupKeyIntValue(group_key.column_type, value), - .string_value = {}, - }); - } - } - - return key; - } - - std::vector group_keys_; -}; - -class AggOperator final : public Operator { - public: - AggOperator(std::unique_ptr child, const std::vector& aggregates, PredicatePtr filter) - : child_(std::move(child)), filter_(std::move(filter)) { - bindings_.reserve(aggregates.size()); - - for (const auto& aggregate : aggregates) { - bindings_.push_back(AggBinding{ - .aggregate = aggregate, - .state = CreateAggState(aggregate), - }); - } - } - - std::optional Next() override { - if (returned_) { - return std::nullopt; - } - - returned_ = true; - - while (auto batch = child_->Next()) { - std::vector selected_rows; - SelectRowsMatchingPredicate(filter_, *batch, selected_rows); - - for (const auto& binding : bindings_) { - ConsumeAggRows(binding.aggregate, *batch, selected_rows, *binding.state); - } - } - - Batch result(BuildAggregateOutputSchema(bindings_)); - - for (size_t i = 0; i < bindings_.size(); ++i) { - result.AppendValueFromString(i, bindings_[i].state->Finalize()); - } - - return result; - } - - private: - std::unique_ptr child_; - PredicatePtr filter_; - std::vector bindings_; - bool returned_ = false; -}; - -class GroupAggOperator final : public Operator { - public: - GroupAggOperator(std::unique_ptr child, std::vector group_keys, - const std::vector& aggregates, std::vector select_items, - PredicatePtr filter, PredicatePtr having, const bool sort_by_group_keys) - : child_(std::move(child)), - group_key_materializer_(std::move(group_keys)), - select_items_(std::move(select_items)), - filter_(std::move(filter)), - having_(std::move(having)), - sort_by_group_keys_(sort_by_group_keys) { - bindings_.reserve(aggregates.size()); - - for (const auto& aggregate : aggregates) { - bindings_.push_back(GroupAggBinding{ - .aggregate = aggregate, - }); - } - } - - std::optional Next() override { - if (returned_) { - return std::nullopt; - } - - returned_ = true; - - while (auto batch = child_->Next()) { - std::vector selected_rows; - SelectRowsMatchingPredicate(filter_, *batch, selected_rows); - - for (const size_t row : selected_rows) { - TypedGroupKey key = group_key_materializer_.Materialize(*batch, row); - - size_t group_index = 0; - const auto it = group_indexes_.find(key); - if (it == group_indexes_.end()) { - group_index = groups_.size(); - group_indexes_.emplace(key, group_index); - - groups_.push_back(GroupState{ - .key_values = std::move(key.values), - .states = CreateStates(), - }); - } else { - group_index = it->second; - } - - GroupState& group = groups_[group_index]; - for (size_t i = 0; i < bindings_.size(); ++i) { - ConsumeAggRow(bindings_[i].aggregate, *batch, row, *group.states[i]); - } - } - } - - const Schema schema = BuildSelectOutputSchema(select_items_); - std::vector output_order = BuildOutputOrder(); - Batch result(schema, output_order.size()); - - for (const size_t group_index : output_order) { - AppendGroupToBatch(groups_[group_index], result); - } - - if (having_) { - std::vector selected_rows; - selected_rows.reserve(result.RowsCount()); - - for (size_t row = 0; row < result.RowsCount(); ++row) { - if (EvaluatePredicate(having_, result, row)) { - selected_rows.push_back(row); - } - } - - Batch filtered(schema, selected_rows.size()); - filtered.AppendRowsSelectedFromBatch(result, selected_rows); - - return filtered; - } - - return result; - } - - private: - struct GroupState { - std::vector key_values; - std::vector> states; - }; - - struct GroupAggBinding { - PlannedAgg aggregate; - }; - - std::vector> CreateStates() const { - std::vector> states; - states.reserve(bindings_.size()); - - for (const auto& binding : bindings_) { - states.push_back(CreateAggState(binding.aggregate)); - } - - return states; - } - - void AppendGroupToBatch(const GroupState& group, const Batch& batch) const { - for (size_t column = 0; column < select_items_.size(); ++column) { - const PlannedSelectItem& item = select_items_[column]; - if (item.kind == SelectItemKind::GroupKey) { - batch.AppendValueFromString(column, FormatGroupKeyValue(group.key_values[item.index])); - } else { - batch.AppendValueFromString(column, group.states[item.index]->Finalize()); - } - } - } - - std::vector BuildOutputOrder() const { - std::vector order; - order.reserve(groups_.size()); - - for (size_t i = 0; i < groups_.size(); ++i) { - order.push_back(i); - } - - if (!sort_by_group_keys_) { - return order; - } - - std::vector group_key_columns; - - for (size_t i = 0; i < select_items_.size(); ++i) { - if (select_items_[i].kind == SelectItemKind::GroupKey) { - group_key_columns.push_back(i); - } - } - - if (group_key_columns.empty()) { - return order; - } - - std::ranges::sort(order, [&](const size_t lhs_index, const size_t rhs_index) { - const GroupState& lhs = groups_[lhs_index]; - const GroupState& rhs = groups_[rhs_index]; - for (const size_t column : group_key_columns) { - const PlannedSelectItem& item = select_items_[column]; - const std::strong_ordering comparison = - CompareGroupKeyValue(lhs.key_values[item.index], rhs.key_values[item.index]); - if (comparison != 0) { - return comparison < 0; - } - } - - return false; - }); - - return order; - } - - std::unique_ptr child_; - GroupKeyMaterializer group_key_materializer_; - std::vector bindings_; - std::vector select_items_; - PredicatePtr filter_; - PredicatePtr having_; - std::unordered_map group_indexes_; - std::vector groups_; - bool sort_by_group_keys_ = false; - bool returned_ = false; -}; - -class GroupAggTopKOperator final : public Operator { - public: - GroupAggTopKOperator(std::unique_ptr child, std::vector group_keys, - const std::vector& aggregates, std::vector select_items, - PredicatePtr filter, std::vector order_by, const size_t limit) - : child_(std::move(child)), - group_key_materializer_(std::move(group_keys)), - select_items_(std::move(select_items)), - filter_(std::move(filter)), - order_by_(std::move(order_by)), - limit_(limit) { - bindings_.reserve(aggregates.size()); - - for (const auto& aggregate : aggregates) { - bindings_.push_back(GroupAggBinding{ - .aggregate = aggregate, - }); - } - } - - std::optional Next() override { - if (returned_ || limit_ == 0) { - return std::nullopt; - } - - returned_ = true; - - while (auto batch = child_->Next()) { - std::vector selected_rows; - SelectRowsMatchingPredicate(filter_, *batch, selected_rows); - - for (const size_t row : selected_rows) { - TypedGroupKey key = group_key_materializer_.Materialize(*batch, row); - - size_t group_index = 0; - const auto it = group_indexes_.find(key); - if (it == group_indexes_.end()) { - group_index = groups_.size(); - group_indexes_.emplace(key, group_index); - - groups_.push_back(GroupState{ - .key_values = std::move(key.values), - .states = CreateStates(), - .aggregate_values = {}, - .aggregate_values_finalized = false, - }); - } else { - group_index = it->second; - } - - GroupState& group = groups_[group_index]; - for (size_t i = 0; i < bindings_.size(); ++i) { - ConsumeAggRow(bindings_[i].aggregate, *batch, row, *group.states[i]); - } - } - } - - const Schema schema = BuildSelectOutputSchema(select_items_); - const std::vector top_groups = BuildTopGroupIndexes(); - Batch result(schema, top_groups.size()); - - for (const size_t group_index : top_groups) { - AppendGroupToBatch(groups_[group_index], result); - } - - return result; - } - - private: - struct GroupState { - std::vector key_values; - std::vector> states; - mutable std::vector aggregate_values; - mutable bool aggregate_values_finalized = false; - }; - - struct GroupAggBinding { - PlannedAgg aggregate; - }; - - std::vector> CreateStates() const { - std::vector> states; - states.reserve(bindings_.size()); - - for (const auto& binding : bindings_) { - states.push_back(CreateAggState(binding.aggregate)); - } - - return states; - } - - static void FinalizeAggregateValues(const GroupState& group) { - if (group.aggregate_values_finalized) { - return; - } - - group.aggregate_values.reserve(group.states.size()); - for (const auto& state : group.states) { - group.aggregate_values.push_back(state->Finalize()); - } - group.aggregate_values_finalized = true; - } - - bool OrdersBefore(const size_t lhs_index, const size_t rhs_index) const { - const GroupState& lhs = groups_[lhs_index]; - const GroupState& rhs = groups_[rhs_index]; - - for (const PlannedOrderBy& order : order_by_) { - const PlannedSelectItem& item = select_items_[order.result_column_index]; - std::strong_ordering comparison = std::strong_ordering::equal; - - if (item.kind == SelectItemKind::GroupKey) { - comparison = CompareGroupKeyValue(lhs.key_values[item.index], rhs.key_values[item.index]); - } else { - FinalizeAggregateValues(lhs); - FinalizeAggregateValues(rhs); - comparison = - CompareValues(item.output_type, lhs.aggregate_values[item.index], rhs.aggregate_values[item.index]); - } - - if (comparison != 0) { - return order.descending ? comparison > 0 : comparison < 0; - } - } - - return lhs_index < rhs_index; - } - - void AppendGroupToBatch(const GroupState& group, const Batch& batch) const { - FinalizeAggregateValues(group); - - for (size_t column = 0; column < select_items_.size(); ++column) { - const PlannedSelectItem& item = select_items_[column]; - if (item.kind == SelectItemKind::GroupKey) { - batch.AppendValueFromString(column, FormatGroupKeyValue(group.key_values[item.index])); - } else { - batch.AppendValueFromString(column, group.aggregate_values[item.index]); - } - } - } - - std::vector BuildTopGroupIndexes() const { - std::vector top_groups; - top_groups.reserve(limit_); - - for (size_t i = 0; i < groups_.size(); ++i) { - if (top_groups.size() < limit_) { - top_groups.push_back(i); - std::ranges::push_heap(top_groups, - [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); - continue; - } - - if (OrdersBefore(i, top_groups.front())) { - std::ranges::pop_heap(top_groups, - [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); - top_groups.back() = i; - std::ranges::push_heap(top_groups, - [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); - } - } - - std::ranges::sort(top_groups, [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); - - return top_groups; - } - - std::unique_ptr child_; - GroupKeyMaterializer group_key_materializer_; - std::vector bindings_; - std::vector select_items_; - PredicatePtr filter_; - std::vector order_by_; - size_t limit_ = 0; - std::unordered_map group_indexes_; - mutable std::vector groups_; - bool returned_ = false; -}; - -struct SortedRow { - std::vector values; - size_t ordinal = 0; -}; - -class RowOrdering { - public: - explicit RowOrdering(std::vector order_by) : order_by_(std::move(order_by)) {} - - bool operator()(const SortedRow& lhs, const SortedRow& rhs) const { - for (const auto& order : order_by_) { - const std::strong_ordering comparison = CompareValues( - order.value_type, lhs.values[order.result_column_index], rhs.values[order.result_column_index]); - if (comparison != 0) { - return order.descending ? comparison > 0 : comparison < 0; - } - } - - return lhs.ordinal < rhs.ordinal; - } - - private: - std::vector order_by_; -}; - -static std::vector MaterializeBatchRow(const Batch& batch, const size_t row) { - std::vector values; - values.reserve(batch.ColumnsCount()); - - for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { - values.push_back(batch.ColumnAt(column_index).ValueAsString(row)); - } - - return values; -} - -static Batch BuildBatchFromSortedRows(const Schema& schema, const std::vector& rows) { - Batch result(schema, rows.size()); - - for (const auto& row : rows) { - for (size_t i = 0; i < row.values.size(); ++i) { - result.AppendValueFromString(i, row.values[i]); - } - } - - return result; -} - -class ProjectionOperator final : public Operator { - public: - ProjectionOperator(std::unique_ptr child, std::vector items, Schema source_schema) - : child_(std::move(child)), items_(std::move(items)), source_schema_(std::move(source_schema)) { - for (const auto& item : items_) { - if (item.expression && item.expression->kind == ExprKind::Star) { - for (const size_t source_index : StarColumnIndexes(source_schema_)) { - schema_.columns.push_back(source_schema_.columns[source_index]); - } - } else { - schema_.columns.emplace_back(item.output_name, ColumnType::String); - } - } - } - - std::optional Next() override { - while (auto batch = child_->Next()) { - if (!star_column_indexes_initialized_) { - star_column_indexes_ = StarColumnIndexes(batch->GetSchema()); - star_column_indexes_initialized_ = true; - } - - Batch projected(schema_, batch->RowsCount()); - - size_t output_column = 0; - for (const auto& item : items_) { - if (item.expression && item.expression->kind == ExprKind::Star) { - for (const size_t source_column : star_column_indexes_) { - projected.AppendColumnRange(output_column++, batch->ColumnAt(source_column), 0, - batch->RowsCount()); - } - continue; - } - - const size_t expression_column = output_column++; - for (size_t row = 0; row < batch->RowsCount(); ++row) { - projected.AppendValueFromString(expression_column, EvalExpr(item.expression, *batch, row)); - } - } - - if (projected.RowsCount() > 0) { - return projected; - } - } - - return std::nullopt; - } - - const Schema& GetSchema() const { return schema_; } - - private: - std::unique_ptr child_; - std::vector items_; - Schema source_schema_; - Schema schema_; - std::vector star_column_indexes_; - bool star_column_indexes_initialized_ = false; -}; - -class OrderByOperator final : public Operator { - public: - OrderByOperator(std::unique_ptr child, std::vector order_by, - std::vector /*select_items*/) - : child_(std::move(child)), ordering_(std::move(order_by)) {} - - std::optional Next() override { - if (returned_) { - return std::nullopt; - } - - returned_ = true; - - std::optional schema; - std::vector rows; - size_t ordinal = 0; - - while (auto batch = child_->Next()) { - if (!schema.has_value()) { - schema = batch->GetSchema(); - } - - for (size_t row = 0; row < batch->RowsCount(); ++row) { - rows.push_back(SortedRow{ - .values = MaterializeBatchRow(*batch, row), - .ordinal = ordinal++, - }); - } - } - - if (!schema.has_value()) { - return std::nullopt; - } - - SortRows(rows); - - return BuildBatchFromSortedRows(*schema, rows); - } - - private: - void SortRows(std::vector& rows) const { std::ranges::sort(rows, ordering_); } - - std::unique_ptr child_; - RowOrdering ordering_; - bool returned_ = false; -}; - -class TopKOperator final : public Operator { - public: - TopKOperator(std::unique_ptr child, std::vector order_by, - std::vector /*select_items*/, const size_t limit) - : child_(std::move(child)), ordering_(std::move(order_by)), limit_(limit) {} - - std::optional Next() override { - if (returned_ || limit_ == 0) { - return std::nullopt; - } - - returned_ = true; - - std::optional schema; - std::vector rows; - rows.reserve(limit_); - - size_t ordinal = 0; - - while (auto batch = child_->Next()) { - if (!schema.has_value()) { - schema = batch->GetSchema(); - } - - for (size_t row = 0; row < batch->RowsCount(); ++row) { - SortedRow candidate{ - .values = MaterializeBatchRow(*batch, row), - .ordinal = ordinal++, - }; - - if (rows.size() < limit_) { - rows.push_back(std::move(candidate)); - std::ranges::push_heap(rows, ordering_); - continue; - } - - if (ordering_(candidate, rows.front())) { - std::ranges::pop_heap(rows, ordering_); - rows.back() = std::move(candidate); - std::ranges::push_heap(rows, ordering_); - } - } - } - - if (!schema.has_value()) { - return std::nullopt; - } - - std::ranges::sort(rows, ordering_); - - return BuildBatchFromSortedRows(*schema, rows); - } - - private: - std::unique_ptr child_; - RowOrdering ordering_; - size_t limit_; - bool returned_ = false; -}; - -class LimitOperator final : public Operator { - public: - LimitOperator(std::unique_ptr child, const size_t limit) : child_(std::move(child)), remaining_(limit) {} - - std::optional Next() override { - if (remaining_ == 0) { - return std::nullopt; - } - - while (auto batch = child_->Next()) { - if (batch->RowsCount() <= remaining_) { - remaining_ -= batch->RowsCount(); - - return batch; - } - - Batch limited(batch->GetSchema(), remaining_); - limited.AppendRowsRangeFromBatch(*batch, 0, remaining_); - - remaining_ = 0; - - return limited; - } - - remaining_ = 0; - - return std::nullopt; - } - - private: - std::unique_ptr child_; - size_t remaining_; -}; - -class OffsetOperator final : public Operator { - public: - OffsetOperator(std::unique_ptr child, const size_t offset) - : child_(std::move(child)), remaining_(offset) {} - - std::optional Next() override { - while (const auto batch = child_->Next()) { - if (remaining_ >= batch->RowsCount()) { - remaining_ -= batch->RowsCount(); - continue; - } - - Batch shifted(batch->GetSchema(), batch->RowsCount() - remaining_); - shifted.AppendRowsRangeFromBatch(*batch, remaining_, batch->RowsCount() - remaining_); - remaining_ = 0; - - return shifted; - } - - return std::nullopt; - } - - private: - std::unique_ptr child_; - size_t remaining_; -}; - -static void ApplyOrderOffsetLimit(std::unique_ptr& root, const PlannedQuery& planned, - bool& limit_applied_by_top_k) { - if (!planned.order_by.empty()) { - if (planned.limit.has_value() && planned.offset == 0) { - root = - std::make_unique(std::move(root), planned.order_by, planned.select_items, *planned.limit); - limit_applied_by_top_k = true; - } else { - root = std::make_unique(std::move(root), planned.order_by, planned.select_items); - } - } - - if (planned.offset > 0) { - root = std::make_unique(std::move(root), planned.offset); - } - - if (planned.limit.has_value() && !limit_applied_by_top_k) { - root = std::make_unique(std::move(root), *planned.limit); - } -} +#include "executor/operators_internal.h" std::unique_ptr BuildPlan(const PlannedQuery& planned) { if (planned.metadata_count_only) { - return std::make_unique(planned.table_path, planned.aggregates.front().output_name); + return CreateMetadataCountOperator(planned.table_path, planned.aggregates.front().output_name); } if (planned.metadata_extrema_only) { - return std::make_unique(planned.table_path, planned.aggregates); + return CreateMetadataExtremaOperator(planned.table_path, planned.aggregates); } std::unique_ptr root = - std::make_unique(planned.table_path, planned.projection_indexes, planned.filter); + CreateScanOperator(planned.table_path, planned.projection_indexes, planned.filter); if (planned.filter && planned.plain_select) { - root = std::make_unique(std::move(root), planned.filter); + root = CreateFilterOperator(std::move(root), planned.filter); } if (planned.plain_select) { bool limit_applied_by_top_k = false; ApplyOrderOffsetLimit(root, planned, limit_applied_by_top_k); - auto projection = - std::make_unique(std::move(root), planned.plain_select_items, planned.table_schema); - const Schema output_schema = projection->GetSchema(); - root = std::move(projection); - root = std::make_unique(std::move(root), output_schema); + const Schema output_schema = ProjectionOutputSchema(planned.plain_select_items, planned.table_schema); + root = CreateProjectionOperator(std::move(root), planned.plain_select_items, planned.table_schema); + root = CreateEnsureSchemaOperator(std::move(root), output_schema); return root; } @@ -1645,20 +32,21 @@ std::unique_ptr BuildPlan(const PlannedQuery& planned) { if (!planned.group_keys.empty()) { if (!planned.order_by.empty() && planned.limit.has_value() && planned.offset == 0 && !planned.having) { - root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, - planned.select_items, planned.filter, planned.order_by, - *planned.limit); + root = CreateGroupAggTopKOperator(std::move(root), planned.group_keys, planned.aggregates, + planned.select_items, planned.filter, planned.order_by, + *planned.limit); limit_applied_by_top_k = true; } else { - root = std::make_unique(std::move(root), planned.group_keys, planned.aggregates, - planned.select_items, planned.filter, planned.having, - planned.order_by.empty()); + root = CreateGroupAggOperator(std::move(root), planned.group_keys, planned.aggregates, + planned.select_items, planned.filter, planned.having, + planned.order_by.empty()); } } else { - root = std::make_unique(std::move(root), planned.aggregates, planned.filter); + root = CreateAggOperator(std::move(root), planned.aggregates, planned.filter); } ApplyOrderOffsetLimit(root, planned, limit_applied_by_top_k); + root = CreateEnsureSchemaOperator(std::move(root), BuildSelectOutputSchema(planned.select_items)); return root; } diff --git a/src/executor/operators_aggregate.cpp b/src/executor/operators_aggregate.cpp new file mode 100644 index 0000000..dd06f7e --- /dev/null +++ b/src/executor/operators_aggregate.cpp @@ -0,0 +1,729 @@ +#include +#include +#include +#include +#include +#include + +#include "common/ascii.h" +#include "common/error.h" +#include "common/operator_utils.h" +#include "common/parsing.h" +#include "executor/aggregate_function.h" +#include "executor/aggregate_state.h" +#include "executor/operators_internal.h" + +constexpr std::string_view ExtractMinutePart = "MINUTE"; +constexpr std::string_view ExtractHourPart = "HOUR"; +constexpr int64_t MinuteMicros = 60'000'000; + +template +Schema BuildAggregateOutputSchema(const std::vector& bindings) { + Schema schema; + schema.columns.reserve(bindings.size()); + + for (const auto& binding : bindings) { + schema.columns.push_back(ColumnSchema(binding.aggregate.output_name, AggregateOutputType(binding.aggregate))); + } + + return schema; +} + +bool UsesRowAggInput(const PlannedAgg& aggregate) { + return !aggregate.direct_numeric_argument && aggregate.argument_kind != AggArgumentKind::Column; +} + +void ConsumeAggRow(const PlannedAgg& aggregate, const Batch& batch, const size_t row, AggState& state) { + if (aggregate.direct_numeric_argument) { + state.ConsumeInt128(batch.ColumnAt(aggregate.column_index).ValueAsInt128(row) + + aggregate.direct_numeric_offset); + return; + } + + if (aggregate.argument_kind == AggArgumentKind::Column) { + state.ConsumeValue(EvalExpr(aggregate.argument, batch, row)); + return; + } + + state.ConsumeRow(); +} + +void ConsumeAggRows(const PlannedAgg& aggregate, const Batch& batch, const std::span rows, + AggState& state) { + if (UsesRowAggInput(aggregate)) { + state.ConsumeRows(rows.size()); + return; + } + + for (const size_t row : rows) { + ConsumeAggRow(aggregate, batch, row, state); + } +} + +struct AggBinding { + PlannedAgg aggregate; + std::unique_ptr state; +}; + +struct GroupKeyComponent { + ColumnType type = ColumnType::String; + Int128 int_value = 0; + std::string string_value; +}; + +std::optional TryEvalTypedGroupKeyInt(const ExprPtr& expr, const Batch& batch, const size_t row) { + if (!expr) { + return std::nullopt; + } + + switch (expr->kind) { + case ExprKind::Column: + if (!expr->column_index_bound || expr->column_index >= batch.ColumnsCount()) { + return std::nullopt; + } + if (batch.ColumnAt(expr->column_index).Type() == ColumnType::String) { + return std::nullopt; + } + return batch.ColumnAt(expr->column_index).ValueAsInt128(row); + case ExprKind::Literal: + if (expr->literal.kind != LiteralKind::Numeric) { + return std::nullopt; + } + return ParseInt128(expr->literal.text); + case ExprKind::Binary: { + const auto lhs = TryEvalTypedGroupKeyInt(expr->left, batch, row); + const auto rhs = TryEvalTypedGroupKeyInt(expr->right, batch, row); + if (!lhs.has_value() || !rhs.has_value()) { + return std::nullopt; + } + return expr->binary_op == BinaryOp::Add ? *lhs + *rhs : *lhs - *rhs; + } + case ExprKind::Function: { + const std::string name = ToUpperAscii(expr->function_name); + + if (name == "EXTRACT" && expr->arguments.size() == 2 && expr->arguments[0] && + expr->arguments[0]->kind == ExprKind::Literal) { + const auto ts = TryEvalTypedGroupKeyInt(expr->arguments[1], batch, row); + if (!ts.has_value()) { + return std::nullopt; + } + + const std::string part = ToUpperAscii(expr->arguments[0]->literal.text); + const std::chrono::sys_time time{ + std::chrono::microseconds{static_cast(*ts)}}; + const auto day = std::chrono::floor(time); + const std::chrono::hh_mm_ss tod{time - day}; + + if (part == ExtractMinutePart) { + return tod.minutes().count(); + } + if (part == ExtractHourPart) { + return tod.hours().count(); + } + } + + if (name == "DATE_TRUNC" && expr->arguments.size() == 2 && expr->arguments[0] && + expr->arguments[0]->kind == ExprKind::Literal) { + const auto ts = TryEvalTypedGroupKeyInt(expr->arguments[1], batch, row); + if (!ts.has_value()) { + return std::nullopt; + } + + const std::string part = ToUpperAscii(NormalizeLiteralForEval(expr->arguments[0]->literal)); + if (part == ExtractMinutePart) { + const int64_t micros = static_cast(*ts); + return micros / MinuteMicros * MinuteMicros; + } + } + + return std::nullopt; + } + case ExprKind::Case: + case ExprKind::Star: + return std::nullopt; + } + + return std::nullopt; +} + +Int128 ParseGroupKeyIntValue(const ColumnType type, const std::string_view value) { + switch (type) { + case ColumnType::Boolean: + return ParseBoolean(value) ? 1 : 0; + case ColumnType::Int16: + return ParseInt16(value); + case ColumnType::Int32: + return ParseInt32(value); + case ColumnType::Int64: + return ParseInt64(value); + case ColumnType::Int128: + return ParseInt128(value); + case ColumnType::Date: + return ParseDate(value); + case ColumnType::Timestamp: + return ParseTimestamp(value); + case ColumnType::Character: + return ParseCharacter(value); + case ColumnType::String: + break; + } + + throw Error::Unsupported("executor", "unsupported group key type"); +} + +std::string FormatGroupKeyValue(const GroupKeyComponent& value) { + switch (value.type) { + case ColumnType::Boolean: + return BooleanToString(value.int_value != 0); + case ColumnType::Int16: + case ColumnType::Int32: + case ColumnType::Int64: + case ColumnType::Int128: + return Int128ToString(value.int_value); + case ColumnType::Date: + return DateToString(static_cast(value.int_value)); + case ColumnType::Timestamp: + return TimestampToString(static_cast(value.int_value)); + case ColumnType::Character: + return std::string(1, static_cast(value.int_value)); + case ColumnType::String: + return value.string_value; + } + + return {}; +} + +std::strong_ordering CompareGroupKeyValue(const GroupKeyComponent& lhs, const GroupKeyComponent& rhs) { + if (lhs.type != rhs.type) { + throw Error::InconsistentData("executor", "group key type mismatch"); + } + + if (lhs.type == ColumnType::String) { + return lhs.string_value <=> rhs.string_value; + } + + return lhs.int_value <=> rhs.int_value; +} + +bool GroupKeyValueEquals(const GroupKeyComponent& lhs, const GroupKeyComponent& rhs) { + return CompareGroupKeyValue(lhs, rhs) == 0; +} + +size_t HashCombine(const size_t seed, const size_t value) { + return seed ^ (value + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2)); +} + +size_t HashInt128(const Int128 value) { + const auto bits = static_cast(value); + const uint64_t low = static_cast(bits); + const uint64_t high = static_cast(bits >> 64); + + return HashCombine(std::hash{}(high), std::hash{}(low)); +} + +struct TypedGroupKey { + std::vector values; +}; + +struct TypedGroupKeyHash { + size_t operator()(const TypedGroupKey& key) const { + size_t hash = 0; + + for (const GroupKeyComponent& value : key.values) { + hash = HashCombine(hash, std::hash{}(static_cast(value.type))); + hash = HashCombine(hash, value.type == ColumnType::String ? std::hash{}(value.string_value) + : HashInt128(value.int_value)); + } + + return hash; + } +}; + +struct TypedGroupKeyEqual { + bool operator()(const TypedGroupKey& lhs, const TypedGroupKey& rhs) const { + if (lhs.values.size() != rhs.values.size()) { + return false; + } + + for (size_t i = 0; i < lhs.values.size(); ++i) { + if (!GroupKeyValueEquals(lhs.values[i], rhs.values[i])) { + return false; + } + } + + return true; + } +}; + +class GroupKeyMaterializer { + public: + explicit GroupKeyMaterializer(std::vector group_keys) : group_keys_(std::move(group_keys)) {} + + TypedGroupKey Materialize(const Batch& batch, const size_t row) const { + TypedGroupKey key; + key.values.reserve(group_keys_.size()); + + for (const auto& group_key : group_keys_) { + if (group_key.column_type != ColumnType::String) { + if (const auto typed_value = TryEvalTypedGroupKeyInt(group_key.expression, batch, row); + typed_value.has_value()) { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = *typed_value, + .string_value = {}, + }); + continue; + } + } else if (group_key.expression && group_key.expression->kind == ExprKind::Column && + group_key.expression->column_index_bound && + group_key.expression->column_index < batch.ColumnsCount()) { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = 0, + .string_value = batch.ColumnAt(group_key.expression->column_index).ValueAsString(row), + }); + continue; + } + + std::string value = EvalExpr(group_key.expression, batch, row); + if (group_key.column_type == ColumnType::String) { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = 0, + .string_value = std::move(value), + }); + } else { + key.values.push_back(GroupKeyComponent{ + .type = group_key.column_type, + .int_value = ParseGroupKeyIntValue(group_key.column_type, value), + .string_value = {}, + }); + } + } + + return key; + } + + private: + std::vector group_keys_; +}; + +class AggOperator final : public Operator { + public: + AggOperator(std::unique_ptr child, std::vector aggregates, PredicatePtr filter) + : child_(std::move(child)), filter_(std::move(filter)) { + bindings_.reserve(aggregates.size()); + + for (auto& aggregate : aggregates) { + std::unique_ptr state = CreateAggState(aggregate); + bindings_.push_back(AggBinding{ + .aggregate = std::move(aggregate), + .state = std::move(state), + }); + } + } + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + while (auto batch = child_->Next()) { + std::vector selected_rows; + SelectRowsMatchingPredicate(filter_, *batch, selected_rows); + + for (auto& binding : bindings_) { + ConsumeAggRows(binding.aggregate, *batch, selected_rows, *binding.state); + } + } + + Batch result(BuildAggregateOutputSchema(bindings_)); + + for (size_t i = 0; i < bindings_.size(); ++i) { + result.AppendValueFromString(i, bindings_[i].state->Finalize()); + } + + return result; + } + + private: + std::unique_ptr child_; + PredicatePtr filter_; + std::vector bindings_; + bool returned_ = false; +}; + +class GroupAggOperator final : public Operator { + public: + GroupAggOperator(std::unique_ptr child, std::vector group_keys, + const std::vector& aggregates, std::vector select_items, + PredicatePtr filter, PredicatePtr having, const bool sort_by_group_keys) + : child_(std::move(child)), + group_key_materializer_(std::move(group_keys)), + select_items_(std::move(select_items)), + filter_(std::move(filter)), + having_(std::move(having)), + sort_by_group_keys_(sort_by_group_keys) { + bindings_.reserve(aggregates.size()); + + for (const auto& aggregate : aggregates) { + bindings_.push_back(GroupAggBinding{ + .aggregate = aggregate, + }); + } + } + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + while (auto batch = child_->Next()) { + std::vector selected_rows; + SelectRowsMatchingPredicate(filter_, *batch, selected_rows); + + for (const size_t row : selected_rows) { + TypedGroupKey key = group_key_materializer_.Materialize(*batch, row); + + size_t group_index = 0; + const auto it = group_indexes_.find(key); + if (it == group_indexes_.end()) { + group_index = groups_.size(); + group_indexes_.emplace(key, group_index); + + groups_.push_back(GroupState{ + .key_values = std::move(key.values), + .states = CreateStates(), + }); + } else { + group_index = it->second; + } + + GroupState& group = groups_[group_index]; + for (size_t i = 0; i < bindings_.size(); ++i) { + ConsumeAggRow(bindings_[i].aggregate, *batch, row, *group.states[i]); + } + } + } + + const Schema schema = BuildSelectOutputSchema(select_items_); + std::vector output_order = BuildOutputOrder(); + Batch result(schema, output_order.size()); + + for (const size_t group_index : output_order) { + AppendGroupToBatch(groups_[group_index], result); + } + + if (having_) { + std::vector selected_rows; + selected_rows.reserve(result.RowsCount()); + + for (size_t row = 0; row < result.RowsCount(); ++row) { + if (EvaluatePredicate(having_, result, row)) { + selected_rows.push_back(row); + } + } + + Batch filtered(schema, selected_rows.size()); + filtered.AppendRowsSelectedFromBatch(result, selected_rows); + + return filtered; + } + + return result; + } + + private: + struct GroupState { + std::vector key_values; + std::vector> states; + }; + + struct GroupAggBinding { + PlannedAgg aggregate; + }; + + std::vector> CreateStates() const { + std::vector> states; + states.reserve(bindings_.size()); + + for (const auto& binding : bindings_) { + states.push_back(CreateAggState(binding.aggregate)); + } + + return states; + } + + void AppendGroupToBatch(const GroupState& group, const Batch& batch) const { + for (size_t column = 0; column < select_items_.size(); ++column) { + const PlannedSelectItem& item = select_items_[column]; + if (item.kind == SelectItemKind::GroupKey) { + batch.AppendValueFromString(column, FormatGroupKeyValue(group.key_values[item.index])); + } else { + batch.AppendValueFromString(column, group.states[item.index]->Finalize()); + } + } + } + + std::vector BuildOutputOrder() const { + std::vector order; + order.reserve(groups_.size()); + + for (size_t i = 0; i < groups_.size(); ++i) { + order.push_back(i); + } + + if (!sort_by_group_keys_) { + return order; + } + + std::vector group_key_columns; + + for (size_t i = 0; i < select_items_.size(); ++i) { + if (select_items_[i].kind == SelectItemKind::GroupKey) { + group_key_columns.push_back(i); + } + } + + if (group_key_columns.empty()) { + return order; + } + + std::sort(order.begin(), order.end(), [&](const size_t lhs_index, const size_t rhs_index) { + const GroupState& lhs = groups_[lhs_index]; + const GroupState& rhs = groups_[rhs_index]; + for (const size_t column : group_key_columns) { + const PlannedSelectItem& item = select_items_[column]; + const std::strong_ordering comparison = + CompareGroupKeyValue(lhs.key_values[item.index], rhs.key_values[item.index]); + if (comparison != 0) { + return comparison < 0; + } + } + + return false; + }); + + return order; + } + + std::unique_ptr child_; + GroupKeyMaterializer group_key_materializer_; + std::vector bindings_; + std::vector select_items_; + PredicatePtr filter_; + PredicatePtr having_; + std::unordered_map group_indexes_; + std::vector groups_; + bool sort_by_group_keys_ = false; + bool returned_ = false; +}; + +class GroupAggTopKOperator final : public Operator { + public: + GroupAggTopKOperator(std::unique_ptr child, std::vector group_keys, + const std::vector& aggregates, std::vector select_items, + PredicatePtr filter, std::vector order_by, const size_t limit) + : child_(std::move(child)), + group_key_materializer_(std::move(group_keys)), + select_items_(std::move(select_items)), + filter_(std::move(filter)), + order_by_(std::move(order_by)), + limit_(limit) { + bindings_.reserve(aggregates.size()); + + for (const auto& aggregate : aggregates) { + bindings_.push_back(GroupAggBinding{ + .aggregate = aggregate, + }); + } + } + + std::optional Next() override { + if (returned_ || limit_ == 0) { + return std::nullopt; + } + + returned_ = true; + + while (auto batch = child_->Next()) { + std::vector selected_rows; + SelectRowsMatchingPredicate(filter_, *batch, selected_rows); + + for (const size_t row : selected_rows) { + TypedGroupKey key = group_key_materializer_.Materialize(*batch, row); + + size_t group_index = 0; + const auto it = group_indexes_.find(key); + if (it == group_indexes_.end()) { + group_index = groups_.size(); + group_indexes_.emplace(key, group_index); + + groups_.push_back(GroupState{ + .key_values = std::move(key.values), + .states = CreateStates(), + .aggregate_values = {}, + .aggregate_values_finalized = false, + }); + } else { + group_index = it->second; + } + + GroupState& group = groups_[group_index]; + for (size_t i = 0; i < bindings_.size(); ++i) { + ConsumeAggRow(bindings_[i].aggregate, *batch, row, *group.states[i]); + } + } + } + + const Schema schema = BuildSelectOutputSchema(select_items_); + const std::vector top_groups = BuildTopGroupIndexes(); + Batch result(schema, top_groups.size()); + + for (const size_t group_index : top_groups) { + AppendGroupToBatch(groups_[group_index], result); + } + + return result; + } + + private: + struct GroupState { + std::vector key_values; + std::vector> states; + mutable std::vector aggregate_values; + mutable bool aggregate_values_finalized = false; + }; + + struct GroupAggBinding { + PlannedAgg aggregate; + }; + + std::vector> CreateStates() const { + std::vector> states; + states.reserve(bindings_.size()); + + for (const auto& binding : bindings_) { + states.push_back(CreateAggState(binding.aggregate)); + } + + return states; + } + + static void FinalizeAggregateValues(const GroupState& group) { + if (group.aggregate_values_finalized) { + return; + } + + group.aggregate_values.reserve(group.states.size()); + for (const auto& state : group.states) { + group.aggregate_values.push_back(state->Finalize()); + } + group.aggregate_values_finalized = true; + } + + bool OrdersBefore(const size_t lhs_index, const size_t rhs_index) const { + const GroupState& lhs = groups_[lhs_index]; + const GroupState& rhs = groups_[rhs_index]; + + for (const PlannedOrderBy& order : order_by_) { + const PlannedSelectItem& item = select_items_[order.result_column_index]; + std::strong_ordering comparison = std::strong_ordering::equal; + + if (item.kind == SelectItemKind::GroupKey) { + comparison = CompareGroupKeyValue(lhs.key_values[item.index], rhs.key_values[item.index]); + } else { + FinalizeAggregateValues(lhs); + FinalizeAggregateValues(rhs); + comparison = + CompareValues(item.output_type, lhs.aggregate_values[item.index], rhs.aggregate_values[item.index]); + } + + if (comparison != 0) { + return order.descending ? comparison > 0 : comparison < 0; + } + } + + return lhs_index < rhs_index; + } + + void AppendGroupToBatch(const GroupState& group, const Batch& batch) const { + FinalizeAggregateValues(group); + + for (size_t column = 0; column < select_items_.size(); ++column) { + const PlannedSelectItem& item = select_items_[column]; + if (item.kind == SelectItemKind::GroupKey) { + batch.AppendValueFromString(column, FormatGroupKeyValue(group.key_values[item.index])); + } else { + batch.AppendValueFromString(column, group.aggregate_values[item.index]); + } + } + } + + std::vector BuildTopGroupIndexes() const { + std::vector top_groups; + top_groups.reserve(limit_); + + for (size_t i = 0; i < groups_.size(); ++i) { + if (top_groups.size() < limit_) { + top_groups.push_back(i); + std::push_heap(top_groups.begin(), top_groups.end(), + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + continue; + } + + if (OrdersBefore(i, top_groups.front())) { + std::pop_heap(top_groups.begin(), top_groups.end(), + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + top_groups.back() = i; + std::push_heap(top_groups.begin(), top_groups.end(), + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + } + } + + std::sort(top_groups.begin(), top_groups.end(), + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + + return top_groups; + } + + std::unique_ptr child_; + GroupKeyMaterializer group_key_materializer_; + std::vector bindings_; + std::vector select_items_; + PredicatePtr filter_; + std::vector order_by_; + size_t limit_ = 0; + std::unordered_map group_indexes_; + mutable std::vector groups_; + bool returned_ = false; +}; + +std::unique_ptr CreateAggOperator(std::unique_ptr child, std::vector aggregates, + PredicatePtr filter) { + return std::make_unique(std::move(child), std::move(aggregates), std::move(filter)); +} + +std::unique_ptr CreateGroupAggOperator(std::unique_ptr child, + std::vector group_keys, + const std::vector& aggregates, + std::vector select_items, PredicatePtr filter, + PredicatePtr having, const bool sort_by_group_keys) { + return std::make_unique(std::move(child), std::move(group_keys), aggregates, + std::move(select_items), std::move(filter), std::move(having), + sort_by_group_keys); +} + +std::unique_ptr CreateGroupAggTopKOperator(std::unique_ptr child, + std::vector group_keys, + const std::vector& aggregates, + std::vector select_items, PredicatePtr filter, + std::vector order_by, const size_t limit) { + return std::make_unique(std::move(child), std::move(group_keys), aggregates, + std::move(select_items), std::move(filter), std::move(order_by), + limit); +} diff --git a/src/executor/operators_runtime.cpp b/src/executor/operators_runtime.cpp new file mode 100644 index 0000000..2905197 --- /dev/null +++ b/src/executor/operators_runtime.cpp @@ -0,0 +1,802 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "common/ascii.h" +#include "common/error.h" +#include "common/operator_utils.h" +#include "common/parsing.h" +#include "common/planner_utils.h" +#include "executor/operators_internal.h" + +constexpr std::string_view CountStarName = "COUNT(*)"; +constexpr std::string_view FallbackAggregateAlias = "c"; +constexpr std::string_view PageViewsAlias = "PageViews"; +constexpr std::string_view ExtractMinutePart = "MINUTE"; +constexpr std::string_view ExtractHourPart = "HOUR"; + +constexpr int64_t MinuteMicros = 60'000'000; + +constexpr std::string_view HttpScheme = "http://"; +constexpr std::string_view HttpsScheme = "https://"; +constexpr std::string_view WwwPrefix = "www."; +constexpr std::string_view RegexBackrefPlaceholder = "\\1"; +constexpr std::string_view RegexBackrefReplacement = "$1"; + +Int128 EvalInt(const std::string& value) { return ParseInt128(value); } + +bool IsAggregateLookupFunction(const std::string_view name) { + return name == "COUNT" || name == "SUM" || name == "AVG" || name == "MIN" || name == "MAX"; +} + +std::string FormatAggregateExprName(const ExprSpec& expr) { + const std::string name = ToUpperAscii(expr.function_name); + if (name == "COUNT" && !expr.arguments.empty() && expr.arguments.front()->kind == ExprKind::Star) { + return std::string(CountStarName); + } + + std::string out = name + "("; + + for (size_t i = 0; i < expr.arguments.size(); ++i) { + if (i > 0) { + out += ", "; + } + out += expr.arguments[i]->output_name; + } + + out += ")"; + + return out; +} + +std::optional TryResolveAggregateValue(const ExprSpec& expr, const Batch& batch, const size_t row) { + const std::string name = ToUpperAscii(expr.function_name); + if (!IsAggregateLookupFunction(name)) { + return std::nullopt; + } + + const std::string formatted_name = FormatAggregateExprName(expr); + for (const std::string_view candidate_name : + {std::string_view{formatted_name}, FallbackAggregateAlias, PageViewsAlias}) { + if (const auto column = TryFindBatchColumn(batch.GetSchema(), candidate_name); column.has_value()) { + return batch.ColumnAt(*column).ValueAsString(row); + } + } + + return std::nullopt; +} + +std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t row) { + if (const auto aggregate_value = TryResolveAggregateValue(expr, batch, row); aggregate_value.has_value()) { + return std::move(*aggregate_value); + } + + const std::string name = ToUpperAscii(expr.function_name); + + if (name == "STRLEN" || name == "LENGTH") { + return std::to_string(EvalExpr(expr.arguments.at(0), batch, row).size()); + } + + if (name == "EXTRACT") { + const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); + const int64_t ts = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); + + const std::chrono::sys_time time{std::chrono::microseconds{ts}}; + const auto day = std::chrono::floor(time); + + const std::chrono::hh_mm_ss tod{time - day}; + + if (part == ExtractMinutePart) { + return std::to_string(tod.minutes().count()); + } + if (part == ExtractHourPart) { + return std::to_string(tod.hours().count()); + } + + throw Error::Unsupported("executor", "unsupported EXTRACT part"); + } + + if (name == "DATE_TRUNC") { + const std::string part = ToUpperAscii(EvalExpr(expr.arguments.at(0), batch, row)); + int64_t micros = ParseTimestamp(EvalExpr(expr.arguments.at(1), batch, row)); + + if (part == ExtractMinutePart) { + micros = micros / MinuteMicros * MinuteMicros; + return TimestampToString(micros); + } + + throw Error::Unsupported("executor", "unsupported DATE_TRUNC part"); + } + + if (name == "REGEXP_REPLACE") { + const std::string source = EvalExpr(expr.arguments.at(0), batch, row); + const std::string pattern_text = EvalExpr(expr.arguments.at(1), batch, row); + std::string replacement = EvalExpr(expr.arguments.at(2), batch, row); + + if (pattern_text == R"(^https?://(?:www\.)?([^/]+)/.*$)" && replacement == R"(\1)") { + std::string_view rest = source; + if (rest.starts_with(HttpScheme)) { + rest.remove_prefix(HttpScheme.size()); + } else if (rest.starts_with(HttpsScheme)) { + rest.remove_prefix(HttpsScheme.size()); + } else { + return source; + } + + if (rest.starts_with(WwwPrefix)) { + rest.remove_prefix(WwwPrefix.size()); + } + + const size_t slash = rest.find('/'); + if (slash == std::string_view::npos || slash == 0) { + return source; + } + + const std::string_view suffix = rest.substr(slash + 1); + if (suffix.find('\n') != std::string_view::npos || suffix.find('\r') != std::string_view::npos) { + return source; + } + + return std::string(rest.substr(0, slash)); + } + + if (expr.regex_replace_bound) { + return std::regex_replace(source, expr.regex_pattern, expr.regex_replacement); + } + + for (size_t pos = 0; (pos = replacement.find(RegexBackrefPlaceholder, pos)) != std::string::npos;) { + replacement.replace(pos, RegexBackrefPlaceholder.size(), RegexBackrefReplacement); + pos += RegexBackrefReplacement.size(); + } + + const std::regex pattern(pattern_text); + + return std::regex_replace(source, pattern, replacement); + } + + throw Error::Unsupported("executor", "unsupported function '" + expr.function_name + "'"); +} + +ComparisonKind FlipComparison(const ComparisonKind comparison) { + switch (comparison) { + case ComparisonKind::Equal: + return ComparisonKind::Equal; + case ComparisonKind::NotEqual: + return ComparisonKind::NotEqual; + case ComparisonKind::Less: + return ComparisonKind::Greater; + case ComparisonKind::LessOrEqual: + return ComparisonKind::GreaterOrEqual; + case ComparisonKind::Greater: + return ComparisonKind::Less; + case ComparisonKind::GreaterOrEqual: + return ComparisonKind::LessOrEqual; + } + + return comparison; +} + +bool MatchesInt128Comparison(const Int128 lhs, const Int128 rhs, const ComparisonKind comparison) { + switch (comparison) { + case ComparisonKind::Equal: + return lhs == rhs; + case ComparisonKind::NotEqual: + return lhs != rhs; + case ComparisonKind::Less: + return lhs < rhs; + case ComparisonKind::LessOrEqual: + return lhs <= rhs; + case ComparisonKind::Greater: + return lhs > rhs; + case ComparisonKind::GreaterOrEqual: + return lhs >= rhs; + } + + return false; +} + +ValueComparison ToValueComparison(const ComparisonKind comparison) { + switch (comparison) { + case ComparisonKind::Equal: + return ValueComparison::Equal; + case ComparisonKind::NotEqual: + return ValueComparison::NotEqual; + case ComparisonKind::Less: + return ValueComparison::Less; + case ComparisonKind::LessOrEqual: + return ValueComparison::LessOrEqual; + case ComparisonKind::Greater: + return ValueComparison::Greater; + case ComparisonKind::GreaterOrEqual: + return ValueComparison::GreaterOrEqual; + } + + return ValueComparison::Equal; +} + +std::optional LiteralValueForColumnType(const QueryLiteral& literal, const ColumnType type) { + try { + if (literal.kind == LiteralKind::Numeric) { + return ParseInt128(literal.text); + } + + if (literal.kind == LiteralKind::String) { + const QueryLiteral normalized_literal{NormalizeLiteralForEval(literal), LiteralKind::String}; + if (type == ColumnType::Date) { + return ParseDate(normalized_literal.text); + } + if (type == ColumnType::Timestamp) { + return ParseTimestamp(normalized_literal.text); + } + } + } catch (const Error&) { + return std::nullopt; + } + + return std::nullopt; +} + +std::optional TryEvaluateTypedComparison(const ExprPtr& left, const ExprPtr& right, const Batch& batch, + const size_t row, const ComparisonKind comparison) { + if (!left || !right) { + return std::nullopt; + } + + if (left->kind != ExprKind::Column || right->kind != ExprKind::Literal) { + if (left->kind == ExprKind::Literal && right->kind == ExprKind::Column) { + return TryEvaluateTypedComparison(right, left, batch, row, FlipComparison(comparison)); + } + return std::nullopt; + } + + const auto column_index = TryFindBatchColumn(batch.GetSchema(), left->column.name); + if (!column_index.has_value()) { + return std::nullopt; + } + + const Column& column = batch.ColumnAt(*column_index); + if (column.Type() == ColumnType::String) { + return std::nullopt; + } + + const std::optional rhs = LiteralValueForColumnType(right->literal, column.Type()); + if (!rhs.has_value()) { + return std::nullopt; + } + + return MatchesInt128Comparison(column.ValueAsInt128(row), *rhs, comparison); +} + +ColumnType InferComparisonType(const std::string& lhs, const std::string& rhs) { + if (TryParseInt128(lhs).has_value() && TryParseInt128(rhs).has_value()) { + return ColumnType::Int128; + } + + if (TryParseDate(lhs).has_value() && TryParseDate(rhs).has_value()) { + return ColumnType::Date; + } + + if (TryParseTimestamp(lhs).has_value() && TryParseTimestamp(rhs).has_value()) { + return ColumnType::Timestamp; + } + + return ColumnType::String; +} + +struct SortedRow { + std::vector values; + size_t ordinal = 0; +}; + +class RowOrdering { + public: + explicit RowOrdering(std::vector order_by) : order_by_(std::move(order_by)) {} + + bool operator()(const SortedRow& lhs, const SortedRow& rhs) const { + for (const auto& order : order_by_) { + const std::strong_ordering comparison = CompareValues( + order.value_type, lhs.values[order.result_column_index], rhs.values[order.result_column_index]); + if (comparison != 0) { + return order.descending ? comparison > 0 : comparison < 0; + } + } + + return lhs.ordinal < rhs.ordinal; + } + + private: + std::vector order_by_; +}; + +std::vector MaterializeBatchRow(const Batch& batch, const size_t row) { + std::vector values; + values.reserve(batch.ColumnsCount()); + + for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { + values.push_back(batch.ColumnAt(column_index).ValueAsString(row)); + } + + return values; +} + +Batch BuildBatchFromSortedRows(const Schema& schema, const std::vector& rows) { + Batch result(schema, rows.size()); + + for (const auto& row : rows) { + for (size_t i = 0; i < row.values.size(); ++i) { + result.AppendValueFromString(i, row.values[i]); + } + } + + return result; +} + +class FilterOperator final : public Operator { + public: + FilterOperator(std::unique_ptr child, PredicatePtr filter) + : child_(std::move(child)), filter_(std::move(filter)) {} + + std::optional Next() override { + while (auto batch = child_->Next()) { + std::vector selected_rows; + SelectRowsMatchingPredicate(filter_, *batch, selected_rows); + + if (selected_rows.empty()) { + continue; + } + + Batch filtered(batch->GetSchema(), selected_rows.size()); + filtered.AppendRowsSelectedFromBatch(*batch, selected_rows); + + return filtered; + } + + return std::nullopt; + } + + private: + std::unique_ptr child_; + PredicatePtr filter_; +}; + +class EnsureSchemaOperator final : public Operator { + public: + EnsureSchemaOperator(std::unique_ptr child, Schema schema) + : child_(std::move(child)), schema_(std::move(schema)) {} + + std::optional Next() override { + if (returned_empty_) { + return std::nullopt; + } + + if (auto batch = child_->Next()) { + returned_any_ = true; + return batch; + } + + if (returned_any_) { + return std::nullopt; + } + + returned_any_ = true; + returned_empty_ = true; + + return Batch(schema_); + } + + private: + std::unique_ptr child_; + Schema schema_; + bool returned_any_ = false; + bool returned_empty_ = false; +}; + +class ProjectionOperator final : public Operator { + public: + ProjectionOperator(std::unique_ptr child, std::vector items, Schema source_schema) + : child_(std::move(child)), + items_(std::move(items)), + source_schema_(std::move(source_schema)), + schema_(ProjectionOutputSchema(items_, source_schema_)) {} + + std::optional Next() override { + while (auto batch = child_->Next()) { + if (!star_column_indexes_initialized_) { + star_column_indexes_ = StarColumnIndexes(batch->GetSchema()); + star_column_indexes_initialized_ = true; + } + + Batch projected(schema_, batch->RowsCount()); + + size_t output_column = 0; + for (const auto& item : items_) { + if (item.expression && item.expression->kind == ExprKind::Star) { + for (const size_t source_column : star_column_indexes_) { + projected.AppendColumnRange(output_column++, batch->ColumnAt(source_column), 0, + batch->RowsCount()); + } + continue; + } + + const size_t expression_column = output_column++; + for (size_t row = 0; row < batch->RowsCount(); ++row) { + projected.AppendValueFromString(expression_column, EvalExpr(item.expression, *batch, row)); + } + } + + if (projected.RowsCount() > 0) { + return projected; + } + } + + return std::nullopt; + } + + private: + std::unique_ptr child_; + std::vector items_; + Schema source_schema_; + Schema schema_; + std::vector star_column_indexes_; + bool star_column_indexes_initialized_ = false; +}; + +class OrderByOperator final : public Operator { + public: + OrderByOperator(std::unique_ptr child, std::vector order_by) + : child_(std::move(child)), ordering_(std::move(order_by)) {} + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + std::optional schema; + std::vector rows; + size_t ordinal = 0; + + while (auto batch = child_->Next()) { + if (!schema.has_value()) { + schema = batch->GetSchema(); + } + + for (size_t row = 0; row < batch->RowsCount(); ++row) { + rows.push_back(SortedRow{ + .values = MaterializeBatchRow(*batch, row), + .ordinal = ordinal++, + }); + } + } + + if (!schema.has_value()) { + return std::nullopt; + } + + std::sort(rows.begin(), rows.end(), ordering_); + + return BuildBatchFromSortedRows(*schema, rows); + } + + private: + std::unique_ptr child_; + RowOrdering ordering_; + bool returned_ = false; +}; + +class TopKOperator final : public Operator { + public: + TopKOperator(std::unique_ptr child, std::vector order_by, const size_t limit) + : child_(std::move(child)), ordering_(std::move(order_by)), limit_(limit) {} + + std::optional Next() override { + if (returned_ || limit_ == 0) { + return std::nullopt; + } + + returned_ = true; + + std::optional schema; + std::vector rows; + rows.reserve(limit_); + + size_t ordinal = 0; + + while (auto batch = child_->Next()) { + if (!schema.has_value()) { + schema = batch->GetSchema(); + } + + for (size_t row = 0; row < batch->RowsCount(); ++row) { + SortedRow candidate{ + .values = MaterializeBatchRow(*batch, row), + .ordinal = ordinal++, + }; + + if (rows.size() < limit_) { + rows.push_back(std::move(candidate)); + std::push_heap(rows.begin(), rows.end(), ordering_); + continue; + } + + if (ordering_(candidate, rows.front())) { + std::pop_heap(rows.begin(), rows.end(), ordering_); + rows.back() = std::move(candidate); + std::push_heap(rows.begin(), rows.end(), ordering_); + } + } + } + + if (!schema.has_value()) { + return std::nullopt; + } + + std::sort(rows.begin(), rows.end(), ordering_); + + return BuildBatchFromSortedRows(*schema, rows); + } + + private: + std::unique_ptr child_; + RowOrdering ordering_; + size_t limit_; + bool returned_ = false; +}; + +class LimitOperator final : public Operator { + public: + LimitOperator(std::unique_ptr child, const size_t limit) : child_(std::move(child)), remaining_(limit) {} + + std::optional Next() override { + if (remaining_ == 0) { + return std::nullopt; + } + + while (auto batch = child_->Next()) { + if (batch->RowsCount() <= remaining_) { + remaining_ -= batch->RowsCount(); + return batch; + } + + Batch limited(batch->GetSchema(), remaining_); + limited.AppendRowsRangeFromBatch(*batch, 0, remaining_); + remaining_ = 0; + + return limited; + } + + remaining_ = 0; + + return std::nullopt; + } + + private: + std::unique_ptr child_; + size_t remaining_; +}; + +class OffsetOperator final : public Operator { + public: + OffsetOperator(std::unique_ptr child, const size_t offset) + : child_(std::move(child)), remaining_(offset) {} + + std::optional Next() override { + while (const auto batch = child_->Next()) { + if (remaining_ >= batch->RowsCount()) { + remaining_ -= batch->RowsCount(); + continue; + } + + Batch shifted(batch->GetSchema(), batch->RowsCount() - remaining_); + shifted.AppendRowsRangeFromBatch(*batch, remaining_, batch->RowsCount() - remaining_); + remaining_ = 0; + + return shifted; + } + + return std::nullopt; + } + + private: + std::unique_ptr child_; + size_t remaining_; +}; + +std::string NormalizeLiteralForEval(const QueryLiteral& literal) { + if (literal.kind != LiteralKind::String) { + return literal.text; + } + + return UnescapeSqlString(literal.text); +} + +std::optional TryFindBatchColumn(const Schema& schema, const std::string_view name) { + for (size_t i = 0; i < schema.columns.size(); ++i) { + if (SameColumnName(schema.columns[i].name, name)) { + return i; + } + } + return std::nullopt; +} + +std::string EvalExpr(const ExprPtr& expr, const Batch& batch, const size_t row) { + switch (expr->kind) { + case ExprKind::Column: { + if (expr->column_index_bound && expr->column_index < batch.ColumnsCount() && + SameColumnName(batch.GetSchema().columns[expr->column_index].name, expr->column.name)) { + return batch.ColumnAt(expr->column_index).ValueAsString(row); + } + + const auto column = TryFindBatchColumn(batch.GetSchema(), expr->column.name); + if (!column.has_value()) { + throw Error::InvalidArgument("executor", "unknown column '" + expr->column.name + "'"); + } + + return batch.ColumnAt(*column).ValueAsString(row); + } + case ExprKind::Literal: + return NormalizeLiteralForEval(expr->literal); + case ExprKind::Binary: { + const Int128 lhs = EvalInt(EvalExpr(expr->left, batch, row)); + const Int128 rhs = EvalInt(EvalExpr(expr->right, batch, row)); + return Int128ToString(expr->binary_op == BinaryOp::Add ? lhs + rhs : lhs - rhs); + } + case ExprKind::Function: + return EvalFunction(*expr, batch, row); + case ExprKind::Case: + return EvaluatePredicate(expr->case_spec.condition, batch, row) + ? EvalExpr(expr->case_spec.then_expr, batch, row) + : EvalExpr(expr->case_spec.else_expr, batch, row); + case ExprKind::Star: + return "*"; + } + return {}; +} + +bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, const size_t row) { + if (!predicate) { + return true; + } + + switch (predicate->kind) { + case PredicateKind::And: + return EvaluatePredicate(predicate->lhs, batch, row) && EvaluatePredicate(predicate->rhs, batch, row); + case PredicateKind::Like: + return LikeMatches(EvalExpr(predicate->left, batch, row), EvalExpr(predicate->right, batch, row)); + case PredicateKind::NotLike: + return !LikeMatches(EvalExpr(predicate->left, batch, row), EvalExpr(predicate->right, batch, row)); + case PredicateKind::In: { + if (predicate->literal_in_set_bound && predicate->in_column_index < batch.ColumnsCount()) { + return predicate->literal_in_values.contains( + batch.ColumnAt(predicate->in_column_index).ValueAsString(row)); + } + + const std::string lhs = EvalExpr(predicate->left, batch, row); + + return std::ranges::any_of(predicate->values, + [&](const ExprPtr& value) { return lhs == EvalExpr(value, batch, row); }); + } + case PredicateKind::Comparison: { + if (predicate->typed_literal_comparison_bound && predicate->typed_column_index < batch.ColumnsCount()) { + return MatchesInt128Comparison(batch.ColumnAt(predicate->typed_column_index).ValueAsInt128(row), + predicate->typed_literal_value, predicate->typed_comparison); + } + + if (const std::optional result = + TryEvaluateTypedComparison(predicate->left, predicate->right, batch, row, predicate->comparison); + result.has_value()) { + return *result; + } + + const std::string lhs = EvalExpr(predicate->left, batch, row); + const std::string rhs = EvalExpr(predicate->right, batch, row); + + return MatchesComparison(InferComparisonType(lhs, rhs), lhs, rhs, predicate->comparison); + } + } + + return false; +} + +void SelectRowsMatchingPredicate(const PredicatePtr& predicate, const Batch& batch, std::vector& rows) { + rows.clear(); + rows.reserve(batch.RowsCount()); + + if (!predicate) { + for (size_t row = 0; row < batch.RowsCount(); ++row) { + rows.push_back(row); + } + return; + } + + if (predicate->kind == PredicateKind::Comparison && predicate->typed_literal_comparison_bound && + predicate->typed_column_index < batch.ColumnsCount()) { + batch.ColumnAt(predicate->typed_column_index) + .SelectRowsByInt128Comparison(predicate->typed_literal_value, + ToValueComparison(predicate->typed_comparison), rows); + return; + } + + if ((predicate->kind == PredicateKind::Like || predicate->kind == PredicateKind::NotLike) && + predicate->literal_like_pattern_bound && predicate->like_column_index < batch.ColumnsCount()) { + batch.ColumnAt(predicate->like_column_index) + .SelectRowsByLikePattern(predicate->like_pattern, predicate->like_negated, rows); + return; + } + + if (predicate->kind == PredicateKind::In && predicate->literal_in_set_bound && + predicate->in_column_index < batch.ColumnsCount()) { + batch.ColumnAt(predicate->in_column_index).SelectRowsByStringSet(predicate->literal_in_values, rows); + return; + } + + for (size_t row = 0; row < batch.RowsCount(); ++row) { + if (EvaluatePredicate(predicate, batch, row)) { + rows.push_back(row); + } + } +} + +Schema BuildSelectOutputSchema(const std::vector& select_items) { + Schema schema; + schema.columns.reserve(select_items.size()); + + for (const auto& item : select_items) { + schema.columns.push_back(ColumnSchema(item.output_name, item.output_type)); + } + + return schema; +} + +Schema ProjectionOutputSchema(const std::vector& items, const Schema& source_schema) { + Schema schema; + + for (const auto& item : items) { + if (item.expression && item.expression->kind == ExprKind::Star) { + for (const size_t source_index : StarColumnIndexes(source_schema)) { + schema.columns.push_back(source_schema.columns[source_index]); + } + } else { + schema.columns.emplace_back(item.output_name, ColumnType::String); + } + } + + return schema; +} + +std::unique_ptr CreateFilterOperator(std::unique_ptr child, PredicatePtr filter) { + return std::make_unique(std::move(child), std::move(filter)); +} + +std::unique_ptr CreateEnsureSchemaOperator(std::unique_ptr child, Schema schema) { + return std::make_unique(std::move(child), std::move(schema)); +} + +std::unique_ptr CreateProjectionOperator(std::unique_ptr child, std::vector items, + Schema source_schema) { + return std::make_unique(std::move(child), std::move(items), std::move(source_schema)); +} + +void ApplyOrderOffsetLimit(std::unique_ptr& root, const PlannedQuery& planned, bool& limit_applied_by_top_k) { + if (!planned.order_by.empty()) { + if (planned.limit.has_value() && planned.offset == 0) { + root = std::make_unique(std::move(root), planned.order_by, *planned.limit); + limit_applied_by_top_k = true; + } else { + root = std::make_unique(std::move(root), planned.order_by); + } + } + + if (planned.offset > 0) { + root = std::make_unique(std::move(root), planned.offset); + } + + if (planned.limit.has_value() && !limit_applied_by_top_k) { + root = std::make_unique(std::move(root), *planned.limit); + } +} diff --git a/src/executor/operators_scan.cpp b/src/executor/operators_scan.cpp new file mode 100644 index 0000000..5bc9008 --- /dev/null +++ b/src/executor/operators_scan.cpp @@ -0,0 +1,232 @@ +#include +#include + +#include "common/ascii.h" +#include "common/error.h" +#include "common/operator_utils.h" +#include "executor/aggregate_state.h" +#include "executor/operators_internal.h" +#include "io/columnar_batch.h" +#include "io/file.h" + +class MetadataCountOperator final : public Operator { + public: + MetadataCountOperator(std::filesystem::path path, std::string output_name) + : path_(std::move(path)), output_name_(std::move(output_name)) {} + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + uint64_t rows = 0; + + const ColumnarBatchReader reader(path_); + for (const auto& row_group : reader.GetMetadata().row_groups) { + rows += row_group.row_count; + } + + Batch result(Schema{{ColumnSchema(output_name_, ColumnType::Int64)}}, 1); + result.AppendValueFromString(0, std::to_string(rows)); + + return result; + } + + private: + std::filesystem::path path_; + std::string output_name_; + bool returned_ = false; +}; + +class MetadataExtremaOperator final : public Operator { + public: + MetadataExtremaOperator(std::filesystem::path path, std::vector aggregates) + : path_(std::move(path)), aggregates_(std::move(aggregates)) {} + + std::optional Next() override { + if (returned_) { + return std::nullopt; + } + + returned_ = true; + + const ColumnarBatchReader reader(path_); + const ColumnarMetadata& metadata = reader.GetMetadata(); + + Schema schema; + schema.columns.reserve(aggregates_.size()); + + std::vector> states; + states.reserve(aggregates_.size()); + + for (const PlannedAgg& aggregate : aggregates_) { + schema.columns.push_back(ColumnSchema(aggregate.output_name, AggregateOutputType(aggregate))); + states.push_back(CreateAggState(aggregate)); + } + + for (const auto& row_group : metadata.row_groups) { + for (size_t i = 0; i < aggregates_.size(); ++i) { + const PlannedAgg& aggregate = aggregates_[i]; + const auto& chunk = row_group.columns[aggregate.column_index]; + + if (!chunk.has_min_max) { + throw Error::InvalidState("executor", "metadata min/max statistics are missing", path_.string()); + } + + const std::string name = ToUpperAscii(aggregate.function->canonical_name); + states[i]->ConsumeInt128(name == "MIN" ? chunk.min_value : chunk.max_value); + } + } + + Batch result(schema); + for (size_t i = 0; i < states.size(); ++i) { + result.AppendValueFromString(i, states[i]->Finalize()); + } + + return result; + } + + private: + std::filesystem::path path_; + std::vector aggregates_; + bool returned_ = false; +}; + +bool MatchesRowGroupComparison(const ColumnChunkMetadata& chunk, const Int128 value, const ComparisonKind comparison) { + if (!chunk.has_min_max) { + return true; + } + + switch (comparison) { + case ComparisonKind::Equal: + return chunk.min_value <= value && value <= chunk.max_value; + case ComparisonKind::NotEqual: + return chunk.min_value != value || chunk.max_value != value; + case ComparisonKind::Less: + return chunk.min_value < value; + case ComparisonKind::LessOrEqual: + return chunk.min_value <= value; + case ComparisonKind::Greater: + return chunk.max_value > value; + case ComparisonKind::GreaterOrEqual: + return chunk.max_value >= value; + } + + return true; +} + +bool MayMatchRowGroup(const PredicatePtr& predicate, const RowGroupMetadata& row_group) { + if (!predicate) { + return true; + } + + switch (predicate->kind) { + case PredicateKind::And: + return MayMatchRowGroup(predicate->lhs, row_group) && MayMatchRowGroup(predicate->rhs, row_group); + case PredicateKind::Comparison: + if (!predicate->metadata_typed_literal_comparison_bound || + predicate->metadata_typed_column_index >= row_group.columns.size()) { + return true; + } + return MatchesRowGroupComparison(row_group.columns[predicate->metadata_typed_column_index], + predicate->metadata_typed_literal_value, + predicate->metadata_typed_comparison); + case PredicateKind::In: + if (!predicate->metadata_typed_in_set_bound || + predicate->metadata_typed_in_column_index >= row_group.columns.size()) { + return true; + } + + return std::any_of(predicate->metadata_typed_in_values.begin(), predicate->metadata_typed_in_values.end(), + [&](const Int128 value) { + return MatchesRowGroupComparison( + row_group.columns[predicate->metadata_typed_in_column_index], value, + ComparisonKind::Equal); + }); + case PredicateKind::Like: + case PredicateKind::NotLike: + return true; + } + + return true; +} + +class ScanOperator final : public Operator { + public: + ScanOperator(std::filesystem::path path, std::vector projection_indexes, PredicatePtr filter) + : path_(std::move(path)), + input_(path_), + metadata_(ColumnarBatchReader(path_).GetMetadata()), + filter_(std::move(filter)) { + if (metadata_.schema.columns.empty()) { + throw Error::MalformedData("executor", "columnar schema is empty", path_.string()); + } + + projection_indexes_ = std::move(projection_indexes); + projected_schema_.columns.reserve(projection_indexes_.size()); + + for (const size_t source_index : projection_indexes_) { + if (source_index >= metadata_.schema.columns.size()) { + throw Error::OutOfRange("executor", "projection index out of range", path_.string()); + } + + projected_schema_.columns.push_back(metadata_.schema.columns[source_index]); + } + } + + std::optional Next() override { + if (next_group_ >= metadata_.row_groups.size()) { + return std::nullopt; + } + + const RowGroupMetadata* row_group = nullptr; + while (next_group_ < metadata_.row_groups.size()) { + const RowGroupMetadata& candidate = metadata_.row_groups[next_group_++]; + if (MayMatchRowGroup(filter_, candidate)) { + row_group = &candidate; + break; + } + } + + if (row_group == nullptr) { + return std::nullopt; + } + + Batch batch(projected_schema_, row_group->row_count); + + for (size_t projected_index = 0; projected_index < projection_indexes_.size(); ++projected_index) { + const size_t source_index = projection_indexes_[projected_index]; + const auto& chunk = row_group->columns[source_index]; + + ReadBatchColumnChunk(path_, input_, chunk, row_group->row_count, batch, projected_index); + } + + return batch; + } + + private: + std::filesystem::path path_; + InputFile input_; + + ColumnarMetadata metadata_; + std::vector projection_indexes_; + Schema projected_schema_; + PredicatePtr filter_; + size_t next_group_ = 0; +}; + +std::unique_ptr CreateMetadataCountOperator(std::filesystem::path path, std::string output_name) { + return std::make_unique(std::move(path), std::move(output_name)); +} + +std::unique_ptr CreateMetadataExtremaOperator(std::filesystem::path path, + std::vector aggregates) { + return std::make_unique(std::move(path), std::move(aggregates)); +} + +std::unique_ptr CreateScanOperator(std::filesystem::path path, std::vector projection_indexes, + PredicatePtr filter) { + return std::make_unique(std::move(path), std::move(projection_indexes), std::move(filter)); +} diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index f48594c..409bf49 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -261,6 +261,65 @@ TEST(executor, rejects_unknown_aggregate_functions) { EXPECT_EQ(result.error().GetCode(), Error::Code::Unsupported); } +TEST(executor, supports_group_by_computed_numeric_and_timestamp_keys) { + const TempFile schema_file("executor_computed_schema"); + const TempFile data_file("executor_computed_data"); + const TempFile columnar_file("executor_computed_columnar"); + + WriteRows(schema_file.Path(), { + {"ClientIP", "int64"}, + {"EventTime", "timestamp"}, + }); + WriteRows(data_file.Path(), { + {"10", "2024-01-01 12:03:11"}, + {"11", "2024-01-01 12:03:49"}, + {"10", "2024-01-01 12:04:00"}, + }); + + ConvertCsvToColumnar(schema_file.Path(), data_file.Path(), columnar_file.Path(), 2); + + Executor executor; + executor.RegisterTable("hits", columnar_file.Path()); + + { + auto result = executor.Execute( + "SELECT ClientIP - 1, COUNT(*) AS c FROM hits GROUP BY ClientIP - 1 ORDER BY c DESC, ClientIP - 1 ASC;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); + EXPECT_EQ(BatchRows(result.value()), (std::vector>{ + {"9", "2"}, + {"10", "1"}, + })); + } + + { + auto result = executor.Execute( + "SELECT extract(minute FROM EventTime) AS m, COUNT(*) AS c FROM hits GROUP BY m ORDER BY m ASC;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); + EXPECT_EQ(BatchRows(result.value()), (std::vector>{ + {"3", "2"}, + {"4", "1"}, + })); + } + + { + auto result = executor.Execute( + "SELECT DATE_TRUNC('minute', EventTime) AS m, COUNT(*) AS c FROM hits GROUP BY m ORDER BY m ASC;"); + ASSERT_TRUE(result.has_value()) << result.error().what(); + EXPECT_EQ(BatchRows(result.value()), (std::vector>{ + {"2024-01-01 12:03:00", "2"}, + {"2024-01-01 12:04:00", "1"}, + })); + } +} + +TEST(executor, preserves_schema_for_empty_grouped_result_after_offset) { + const Batch batch = BuildHitsTable( + "SELECT RegionID, COUNT(*) AS c FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10 OFFSET 100;"); + + EXPECT_EQ(BatchColumnNames(batch), (std::vector{"RegionID", "c"})); + EXPECT_TRUE(BatchRows(batch).empty()); +} + TEST(executor, prunes_row_groups_for_typed_where_filters_before_reading_chunks) { const TempFile schema_file("executor_prune_schema"); const TempFile data_file("executor_prune_data"); @@ -301,11 +360,14 @@ TEST(executor, prunes_row_groups_for_typed_where_filters_before_reading_chunks) std::memcpy(&metadata_size, bytes.data() + footer_offset, sizeof(metadata_size)); const size_t metadata_offset = footer_offset - metadata_size; - bytes.erase(bytes.begin() + static_cast(metadata_offset), bytes.begin() + static_cast(footer_offset)); - bytes.insert(bytes.begin() + static_cast(metadata_offset), metadata_blob.begin(), metadata_blob.end()); + bytes.erase(bytes.begin() + static_cast(metadata_offset), + bytes.begin() + static_cast(footer_offset)); + bytes.insert(bytes.begin() + static_cast(metadata_offset), metadata_blob.begin(), + metadata_blob.end()); const uint64_t patched_metadata_size = metadata_blob.size(); - std::memcpy(bytes.data() + metadata_offset + metadata_blob.size(), &patched_metadata_size, sizeof(patched_metadata_size)); + std::memcpy(bytes.data() + metadata_offset + metadata_blob.size(), &patched_metadata_size, + sizeof(patched_metadata_size)); WriteFileBytes(columnar_file.Path(), bytes); From 8e3aef8b01c9cdc45f9a24c97465634e5f379ca5 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 14:22:31 +0300 Subject: [PATCH 05/13] upd: update dashboard --- README.md | 154 +++++++++++------------ benchmarks/results/readme_benchmarks.csv | 86 ++++++------- 2 files changed, 120 insertions(+), 120 deletions(-) diff --git a/README.md b/README.md index 23ae626..fb86f57 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## Benchmark Dashboard -`generated 2026-05-28 13:26:56` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) +`generated 2026-05-28 14:14:38` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) ### Summary @@ -13,14 +13,14 @@ | --- | ---: | | queries ok | 43 / 43 | | queries failed | 0 | -| median warm time | 239.79 ms | -| average warm time | 274.57 ms | -| p95 warm time | 813.63 ms | -| cold/warm delta | 6.7% | +| median warm time | 227.39 ms | +| average warm time | 227.85 ms | +| p95 warm time | 469.27 ms | +| cold/warm delta | 4.1% | | total output size | 20.3 KB | | max output size | 6.0 KB | -| fastest query | Q06 · 4.20 ms | -| slowest query | Q18 · 1146.61 ms | +| fastest query | Q19 · 4.30 ms | +| slowest query | Q28 · 961.46 ms | ### Storage @@ -34,10 +34,10 @@ | roundtrip csv size | 743.0 MB | | compression ratio | 6.71x | | columnar / csv | 14.9% | -| csv -> columnar | 5.52s | -| columnar -> csv | 9.65s | -| convert throughput | 145.4 MB/s | -| roundtrip throughput | 12.4 MB/s | +| csv -> columnar | 5.65s | +| columnar -> csv | 9.55s | +| convert throughput | 142.2 MB/s | +| roundtrip throughput | 12.5 MB/s | ### Heatmap @@ -45,89 +45,89 @@ | slot 1 | slot 2 | slot 3 | slot 4 | slot 5 | slot 6 | | ---: | ---: | ---: | ---: | ---: | ---: | -| Q00 🟩 `5.33` | Q01 🟩 `7.51` | Q02 🟩 `18.46` | Q03 🟩 `14.22` | Q04 🟩 `142.67` | Q05 🟩 `127.92` | -| Q06 🟩 `4.20` | Q07 🟩 `6.58` | Q08 🟩 `184.89` | Q09 🟩 `200.71` | Q10 🟥 `266.04` | Q11 🟥 `308.16` | -| Q12 🟦 `228.27` | Q13 🟦 `244.72` | Q14 🟦 `229.88` | Q15 🟩 `69.88` | Q16 🟩 `139.03` | Q17 🟩 `149.92` | -| Q18 🟥 `1146.61` | Q19 🟩 `4.85` | Q20 🟩 `144.36` | Q21 🟥 `283.42` | Q22 🟥 `371.82` | Q23 🟦 `245.27` | -| Q24 🟥 `275.45` | Q25 🟥 `329.49` | Q26 🟥 `269.75` | Q27 🟥 `439.36` | Q28 🟥 `957.04` | Q29 🟥 `390.44` | -| Q30 🟦 `239.79` | Q31 🟦 `261.03` | Q32 🟥 `828.89` | Q33 🟥 `369.78` | Q34 🟥 `482.41` | Q35 🟥 `676.27` | -| Q36 🟥 `324.34` | Q37 🟦 `235.02` | Q38 🟩 `69.98` | Q39 🟥 `416.78` | Q40 🟩 `90.83` | Q41 🟩 `32.00` | -| Q42 🟥 `573.07` | | | | | | +| Q00 🟩 `5.66` | Q01 🟩 `8.52` | Q02 🟩 `18.69` | Q03 🟩 `15.81` | Q04 🟩 `167.19` | Q05 🟩 `142.57` | +| Q06 🟩 `4.81` | Q07 🟩 `8.51` | Q08 🟩 `191.42` | Q09 🟦 `204.68` | Q10 🟥 `266.03` | Q11 🟥 `267.35` | +| Q12 🟦 `225.01` | Q13 🟦 `239.70` | Q14 🟦 `228.39` | Q15 🟩 `70.37` | Q16 🟩 `120.24` | Q17 🟩 `141.30` | +| Q18 🟥 `334.36` | Q19 🟩 `4.30` | Q20 🟩 `142.30` | Q21 🟥 `273.15` | Q22 🟥 `365.64` | Q23 🟦 `242.96` | +| Q24 🟥 `271.73` | Q25 🟦 `222.25` | Q26 🟥 `271.67` | Q27 🟥 `420.53` | Q28 🟥 `961.46` | Q29 🟥 `390.66` | +| Q30 🟦 `232.35` | Q31 🟥 `254.79` | Q32 🟥 `727.78` | Q33 🟥 `369.53` | Q34 🟥 `408.60` | Q35 🟦 `227.39` | +| Q36 🟥 `352.00` | Q37 🟦 `240.26` | Q38 🟩 `82.36` | Q39 🟥 `474.69` | Q40 🟩 `96.72` | Q41 🟩 `35.08` | +| Q42 🟩 `68.79` | | | | | | ### Largest Result Sets | Query | Output | Warm median, ms | SQL | | --- | ---: | ---: | --- | -| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 371.82 | SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT… | -| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 245.27 | SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime… | -| [Q29](benchmarks/queries/query_29.sql) | 990 B | 390.44 | SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(Reso… | -| [Q39](benchmarks/queries/query_39.sql) | 835 B | 416.78 | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (… | -| [Q24](benchmarks/queries/query_24.sql) | 658 B | 275.45 | SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY… | +| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 365.64 | SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT… | +| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 242.96 | SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime… | +| [Q29](benchmarks/queries/query_29.sql) | 990 B | 390.66 | SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(Reso… | +| [Q39](benchmarks/queries/query_39.sql) | 835 B | 474.69 | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (… | +| [Q24](benchmarks/queries/query_24.sql) | 658 B | 271.73 | SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY… | ### Fastest Queries | Query | SQL | Warm median, ms | First run, ms | | --- | --- | ---: | ---: | -| [Q06](benchmarks/queries/query_6.sql) | SELECT MIN(EventDate), MAX(EventDate) FROM hits; | 4.20 | 4.73 | -| [Q19](benchmarks/queries/query_19.sql) | SELECT UserID FROM hits WHERE UserID = 435090932899640449; | 4.85 | 5.58 | -| [Q00](benchmarks/queries/query_0.sql) | SELECT COUNT(*) FROM hits; | 5.33 | 10.91 | -| [Q07](benchmarks/queries/query_7.sql) | SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY … | 6.58 | 6.60 | -| [Q01](benchmarks/queries/query_1.sql) | SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; | 7.51 | 14.12 | +| [Q19](benchmarks/queries/query_19.sql) | SELECT UserID FROM hits WHERE UserID = 435090932899640449; | 4.30 | 5.03 | +| [Q06](benchmarks/queries/query_6.sql) | SELECT MIN(EventDate), MAX(EventDate) FROM hits; | 4.81 | 4.83 | +| [Q00](benchmarks/queries/query_0.sql) | SELECT COUNT(*) FROM hits; | 5.66 | 6.73 | +| [Q07](benchmarks/queries/query_7.sql) | SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY … | 8.51 | 11.26 | +| [Q01](benchmarks/queries/query_1.sql) | SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; | 8.52 | 7.83 | ### Slowest Queries | Query | SQL | Warm median, ms | Warm max, ms | | --- | --- | ---: | ---: | -| [Q18](benchmarks/queries/query_18.sql) | SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT… | 1146.61 | 1176.15 | -| [Q28](benchmarks/queries/query_28.sql) | SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1')… | 957.04 | 967.25 | -| [Q32](benchmarks/queries/query_32.sql) | SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(Resolution… | 828.89 | 953.89 | -| [Q35](benchmarks/queries/query_35.sql) | SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS … | 676.27 | 678.94 | -| [Q42](benchmarks/queries/query_42.sql) | SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM… | 573.07 | 574.43 | +| [Q28](benchmarks/queries/query_28.sql) | SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1')… | 961.46 | 965.33 | +| [Q32](benchmarks/queries/query_32.sql) | SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(Resolution… | 727.78 | 799.88 | +| [Q39](benchmarks/queries/query_39.sql) | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEn… | 474.69 | 479.44 | +| [Q27](benchmarks/queries/query_27.sql) | SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE … | 420.53 | 423.62 | +| [Q34](benchmarks/queries/query_34.sql) | SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC … | 408.60 | 497.95 | ### Query Table | Query | Output CSV | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | -| [Q00](benchmarks/queries/query_0.sql) | 7 B | 10.91 | 5.73 | 5.33 | 5.06 | 7.21 | ok | -| [Q01](benchmarks/queries/query_1.sql) | 6 B | 14.12 | 7.41 | 7.51 | 6.92 | 7.72 | ok | -| [Q02](benchmarks/queries/query_2.sql) | 18 B | 27.96 | 18.37 | 18.46 | 17.43 | 19.11 | ok | -| [Q03](benchmarks/queries/query_3.sql) | 20 B | 24.73 | 14.17 | 14.22 | 13.58 | 14.67 | ok | -| [Q04](benchmarks/queries/query_4.sql) | 6 B | 142.46 | 143.75 | 142.67 | 142.39 | 147.26 | ok | -| [Q05](benchmarks/queries/query_5.sql) | 6 B | 137.41 | 127.97 | 127.92 | 127.70 | 128.33 | ok | -| [Q06](benchmarks/queries/query_6.sql) | 22 B | 4.73 | 4.21 | 4.20 | 4.11 | 4.35 | ok | -| [Q07](benchmarks/queries/query_7.sql) | 31 B | 6.60 | 6.58 | 6.58 | 6.55 | 6.64 | ok | -| [Q08](benchmarks/queries/query_8.sql) | 81 B | 187.86 | 185.13 | 184.89 | 184.61 | 186.12 | ok | -| [Q09](benchmarks/queries/query_9.sql) | 237 B | 199.68 | 200.89 | 200.71 | 200.13 | 202.00 | ok | -| [Q10](benchmarks/queries/query_10.sql) | 102 B | 276.16 | 266.05 | 266.04 | 263.92 | 268.23 | ok | -| [Q11](benchmarks/queries/query_11.sql) | 116 B | 276.72 | 309.33 | 308.16 | 267.94 | 353.04 | ok | -| [Q12](benchmarks/queries/query_12.sql) | 387 B | 410.85 | 229.06 | 228.27 | 225.02 | 234.66 | ok | -| [Q13](benchmarks/queries/query_13.sql) | 350 B | 243.61 | 249.32 | 244.72 | 239.30 | 268.53 | ok | -| [Q14](benchmarks/queries/query_14.sql) | 400 B | 228.50 | 230.86 | 229.88 | 226.83 | 236.84 | ok | -| [Q15](benchmarks/queries/query_15.sql) | 236 B | 74.99 | 69.83 | 69.88 | 69.21 | 70.34 | ok | -| [Q16](benchmarks/queries/query_16.sql) | 244 B | 123.59 | 135.96 | 139.03 | 119.77 | 146.01 | ok | -| [Q17](benchmarks/queries/query_17.sql) | 414 B | 186.73 | 152.43 | 149.92 | 145.57 | 164.29 | ok | -| [Q18](benchmarks/queries/query_18.sql) | 264 B | 1168.73 | 1151.55 | 1146.61 | 1136.85 | 1176.15 | ok | -| [Q19](benchmarks/queries/query_19.sql) | 0 B | 5.58 | 4.83 | 4.85 | 4.65 | 4.96 | ok | -| [Q20](benchmarks/queries/query_20.sql) | 3 B | 188.34 | 144.23 | 144.36 | 142.56 | 145.65 | ok | -| [Q21](benchmarks/queries/query_21.sql) | 162 B | 277.03 | 282.07 | 283.42 | 270.81 | 290.61 | ok | -| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 410.96 | 372.51 | 371.82 | 362.17 | 384.20 | ok | -| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 276.18 | 245.86 | 245.27 | 242.46 | 250.46 | ok | -| [Q24](benchmarks/queries/query_24.sql) | 658 B | 272.21 | 287.04 | 275.45 | 272.69 | 324.55 | ok | -| [Q25](benchmarks/queries/query_25.sql) | 382 B | 287.75 | 410.16 | 329.49 | 263.83 | 717.82 | ok | -| [Q26](benchmarks/queries/query_26.sql) | 658 B | 316.19 | 270.07 | 269.75 | 268.76 | 272.01 | ok | -| [Q27](benchmarks/queries/query_27.sql) | 26 B | 464.09 | 448.40 | 439.36 | 421.28 | 493.62 | ok | -| [Q28](benchmarks/queries/query_28.sql) | 556 B | 1003.27 | 958.87 | 957.04 | 954.14 | 967.25 | ok | -| [Q29](benchmarks/queries/query_29.sql) | 990 B | 395.02 | 390.10 | 390.44 | 387.37 | 392.13 | ok | -| [Q30](benchmarks/queries/query_30.sql) | 244 B | 256.21 | 238.89 | 239.79 | 235.62 | 240.37 | ok | -| [Q31](benchmarks/queries/query_31.sql) | 398 B | 299.84 | 261.53 | 261.03 | 258.93 | 265.14 | ok | -| [Q32](benchmarks/queries/query_32.sql) | 376 B | 986.27 | 854.79 | 828.89 | 807.48 | 953.89 | ok | -| [Q33](benchmarks/queries/query_33.sql) | 464 B | 378.58 | 374.65 | 369.78 | 363.09 | 395.96 | ok | -| [Q34](benchmarks/queries/query_34.sql) | 484 B | 473.26 | 480.15 | 482.41 | 408.94 | 546.83 | ok | -| [Q35](benchmarks/queries/query_35.sql) | 493 B | 676.69 | 675.28 | 676.27 | 669.64 | 678.94 | ok | -| [Q36](benchmarks/queries/query_36.sql) | 569 B | 340.63 | 324.75 | 324.34 | 321.71 | 328.61 | ok | -| [Q37](benchmarks/queries/query_37.sql) | 550 B | 249.18 | 234.70 | 235.02 | 233.13 | 235.62 | ok | -| [Q38](benchmarks/queries/query_38.sql) | 480 B | 75.87 | 70.08 | 69.98 | 69.94 | 70.44 | ok | -| [Q39](benchmarks/queries/query_39.sql) | 835 B | 489.41 | 416.60 | 416.78 | 408.36 | 424.48 | ok | -| [Q40](benchmarks/queries/query_40.sql) | 356 B | 106.21 | 91.04 | 90.83 | 90.28 | 92.23 | ok | -| [Q41](benchmarks/queries/query_41.sql) | 132 B | 37.17 | 32.00 | 32.00 | 31.92 | 32.07 | ok | -| [Q42](benchmarks/queries/query_42.sql) | 240 B | 580.31 | 572.39 | 573.07 | 568.98 | 574.43 | ok | +| [Q00](benchmarks/queries/query_0.sql) | 7 B | 6.73 | 5.73 | 5.66 | 5.04 | 6.58 | ok | +| [Q01](benchmarks/queries/query_1.sql) | 6 B | 7.83 | 8.39 | 8.52 | 7.40 | 9.11 | ok | +| [Q02](benchmarks/queries/query_2.sql) | 18 B | 25.17 | 18.62 | 18.69 | 18.22 | 18.89 | ok | +| [Q03](benchmarks/queries/query_3.sql) | 20 B | 14.89 | 16.28 | 15.81 | 14.92 | 18.60 | ok | +| [Q04](benchmarks/queries/query_4.sql) | 6 B | 169.19 | 166.66 | 167.19 | 160.12 | 172.16 | ok | +| [Q05](benchmarks/queries/query_5.sql) | 6 B | 158.88 | 142.99 | 142.57 | 139.13 | 147.68 | ok | +| [Q06](benchmarks/queries/query_6.sql) | 22 B | 4.83 | 4.81 | 4.81 | 4.51 | 5.09 | ok | +| [Q07](benchmarks/queries/query_7.sql) | 31 B | 11.26 | 9.06 | 8.51 | 8.09 | 11.12 | ok | +| [Q08](benchmarks/queries/query_8.sql) | 81 B | 220.05 | 191.69 | 191.42 | 188.73 | 195.20 | ok | +| [Q09](benchmarks/queries/query_9.sql) | 237 B | 205.60 | 204.66 | 204.68 | 204.17 | 205.11 | ok | +| [Q10](benchmarks/queries/query_10.sql) | 102 B | 273.84 | 266.00 | 266.03 | 265.52 | 266.41 | ok | +| [Q11](benchmarks/queries/query_11.sql) | 116 B | 268.04 | 267.38 | 267.35 | 267.27 | 267.58 | ok | +| [Q12](benchmarks/queries/query_12.sql) | 387 B | 224.63 | 224.93 | 225.01 | 224.44 | 225.28 | ok | +| [Q13](benchmarks/queries/query_13.sql) | 350 B | 240.61 | 239.63 | 239.70 | 238.61 | 240.50 | ok | +| [Q14](benchmarks/queries/query_14.sql) | 400 B | 228.67 | 228.32 | 228.39 | 227.04 | 229.47 | ok | +| [Q15](benchmarks/queries/query_15.sql) | 236 B | 71.09 | 70.54 | 70.37 | 69.94 | 71.46 | ok | +| [Q16](benchmarks/queries/query_16.sql) | 244 B | 122.74 | 120.44 | 120.24 | 119.31 | 121.99 | ok | +| [Q17](benchmarks/queries/query_17.sql) | 414 B | 141.73 | 141.49 | 141.30 | 140.31 | 143.04 | ok | +| [Q18](benchmarks/queries/query_18.sql) | 264 B | 369.08 | 335.98 | 334.36 | 333.21 | 342.01 | ok | +| [Q19](benchmarks/queries/query_19.sql) | 0 B | 5.03 | 4.30 | 4.30 | 4.27 | 4.34 | ok | +| [Q20](benchmarks/queries/query_20.sql) | 3 B | 143.32 | 142.33 | 142.30 | 142.00 | 142.71 | ok | +| [Q21](benchmarks/queries/query_21.sql) | 162 B | 271.64 | 272.93 | 273.15 | 271.58 | 273.83 | ok | +| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 366.39 | 365.99 | 365.64 | 364.45 | 368.23 | ok | +| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 249.63 | 242.91 | 242.96 | 242.01 | 243.72 | ok | +| [Q24](benchmarks/queries/query_24.sql) | 658 B | 271.33 | 271.81 | 271.73 | 271.32 | 272.46 | ok | +| [Q25](benchmarks/queries/query_25.sql) | 382 B | 223.93 | 222.37 | 222.25 | 221.78 | 223.21 | ok | +| [Q26](benchmarks/queries/query_26.sql) | 658 B | 271.34 | 271.66 | 271.67 | 271.41 | 271.90 | ok | +| [Q27](benchmarks/queries/query_27.sql) | 26 B | 421.81 | 421.00 | 420.53 | 419.31 | 423.62 | ok | +| [Q28](benchmarks/queries/query_28.sql) | 556 B | 966.80 | 961.41 | 961.46 | 957.40 | 965.33 | ok | +| [Q29](benchmarks/queries/query_29.sql) | 990 B | 396.16 | 390.69 | 390.66 | 387.70 | 393.73 | ok | +| [Q30](benchmarks/queries/query_30.sql) | 244 B | 236.87 | 232.38 | 232.35 | 232.17 | 232.65 | ok | +| [Q31](benchmarks/queries/query_31.sql) | 398 B | 262.18 | 254.79 | 254.79 | 252.74 | 256.84 | ok | +| [Q32](benchmarks/queries/query_32.sql) | 376 B | 967.89 | 741.95 | 727.78 | 712.38 | 799.88 | ok | +| [Q33](benchmarks/queries/query_33.sql) | 464 B | 368.98 | 380.07 | 369.53 | 362.34 | 418.87 | ok | +| [Q34](benchmarks/queries/query_34.sql) | 484 B | 386.18 | 427.59 | 408.60 | 395.22 | 497.95 | ok | +| [Q35](benchmarks/queries/query_35.sql) | 493 B | 149.27 | 215.79 | 227.39 | 164.34 | 244.05 | ok | +| [Q36](benchmarks/queries/query_36.sql) | 569 B | 367.61 | 349.09 | 352.00 | 328.04 | 364.33 | ok | +| [Q37](benchmarks/queries/query_37.sql) | 550 B | 255.09 | 253.91 | 240.26 | 238.20 | 296.91 | ok | +| [Q38](benchmarks/queries/query_38.sql) | 480 B | 78.61 | 81.87 | 82.36 | 72.82 | 89.95 | ok | +| [Q39](benchmarks/queries/query_39.sql) | 835 B | 546.84 | 468.97 | 474.69 | 447.05 | 479.44 | ok | +| [Q40](benchmarks/queries/query_40.sql) | 356 B | 113.76 | 96.55 | 96.72 | 94.74 | 98.03 | ok | +| [Q41](benchmarks/queries/query_41.sql) | 132 B | 39.03 | 35.20 | 35.08 | 34.74 | 35.88 | ok | +| [Q42](benchmarks/queries/query_42.sql) | 240 B | 78.57 | 68.56 | 68.79 | 67.35 | 69.30 | ok | diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index 0ad6d04..23f171b 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,44 +1,44 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,output_csv_bytes,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,10.91,5.73,5.33,5.06,7.21,7,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,14.12,7.41,7.51,6.92,7.72,6,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",27.96,18.37,18.46,17.43,19.11,18,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,24.73,14.17,14.22,13.58,14.67,20,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,142.46,143.75,142.67,142.39,147.26,6,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,137.41,127.97,127.92,127.70,128.33,6,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",4.73,4.21,4.20,4.11,4.35,22,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",6.60,6.58,6.58,6.55,6.64,31,ok, -8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",187.86,185.13,184.89,184.61,186.12,81,ok, -9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",199.68,200.89,200.71,200.13,202.00,237,ok, -10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",276.16,266.05,266.04,263.92,268.23,102,ok, -11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",276.72,309.33,308.16,267.94,353.04,116,ok, -12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",410.85,229.06,228.27,225.02,234.66,387,ok, -13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",243.61,249.32,244.72,239.30,268.53,350,ok, -14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",228.50,230.86,229.88,226.83,236.84,400,ok, -15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",74.99,69.83,69.88,69.21,70.34,236,ok, -16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",123.59,135.96,139.03,119.77,146.01,244,ok, -17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",186.73,152.43,149.92,145.57,164.29,414,ok, -18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",1168.73,1151.55,1146.61,1136.85,1176.15,264,ok, -19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,5.58,4.83,4.85,4.65,4.96,0,ok, -20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,188.34,144.23,144.36,142.56,145.65,3,ok, -21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",277.03,282.07,283.42,270.81,290.61,162,ok, -22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",410.96,372.51,371.82,362.17,384.20,6188,ok, -23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,276.18,245.86,245.27,242.46,250.46,2633,ok, -24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,272.21,287.04,275.45,272.69,324.55,658,ok, -25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,287.75,410.16,329.49,263.83,717.82,382,ok, -26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",316.19,270.07,269.75,268.76,272.01,658,ok, -27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",464.09,448.40,439.36,421.28,493.62,26,ok, -28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",1003.27,958.87,957.04,954.14,967.25,556,ok, -29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",395.02,390.10,390.44,387.37,392.13,990,ok, -30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",256.21,238.89,239.79,235.62,240.37,244,ok, -31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",299.84,261.53,261.03,258.93,265.14,398,ok, -32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",986.27,854.79,828.89,807.48,953.89,376,ok, -33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",378.58,374.65,369.78,363.09,395.96,464,ok, -34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",473.26,480.15,482.41,408.94,546.83,484,ok, -35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",676.69,675.28,676.27,669.64,678.94,493,ok, -36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",340.63,324.75,324.34,321.71,328.61,569,ok, -37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",249.18,234.70,235.02,233.13,235.62,550,ok, -38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",75.87,70.08,69.98,69.94,70.44,480,ok, -39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",489.41,416.60,416.78,408.36,424.48,835,ok, -40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",106.21,91.04,90.83,90.28,92.23,356,ok, -41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",37.17,32.00,32.00,31.92,32.07,132,ok, -42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",580.31,572.39,573.07,568.98,574.43,240,ok, +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,6.73,5.73,5.66,5.04,6.58,7,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,7.83,8.39,8.52,7.40,9.11,6,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",25.17,18.62,18.69,18.22,18.89,18,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,14.89,16.28,15.81,14.92,18.60,20,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,169.19,166.66,167.19,160.12,172.16,6,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,158.88,142.99,142.57,139.13,147.68,6,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",4.83,4.81,4.81,4.51,5.09,22,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",11.26,9.06,8.51,8.09,11.12,31,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",220.05,191.69,191.42,188.73,195.20,81,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",205.60,204.66,204.68,204.17,205.11,237,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",273.84,266.00,266.03,265.52,266.41,102,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",268.04,267.38,267.35,267.27,267.58,116,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",224.63,224.93,225.01,224.44,225.28,387,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",240.61,239.63,239.70,238.61,240.50,350,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",228.67,228.32,228.39,227.04,229.47,400,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",71.09,70.54,70.37,69.94,71.46,236,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",122.74,120.44,120.24,119.31,121.99,244,ok, +17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",141.73,141.49,141.30,140.31,143.04,414,ok, +18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",369.08,335.98,334.36,333.21,342.01,264,ok, +19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,5.03,4.30,4.30,4.27,4.34,0,ok, +20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,143.32,142.33,142.30,142.00,142.71,3,ok, +21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",271.64,272.93,273.15,271.58,273.83,162,ok, +22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",366.39,365.99,365.64,364.45,368.23,6188,ok, +23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,249.63,242.91,242.96,242.01,243.72,2633,ok, +24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,271.33,271.81,271.73,271.32,272.46,658,ok, +25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,223.93,222.37,222.25,221.78,223.21,382,ok, +26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",271.34,271.66,271.67,271.41,271.90,658,ok, +27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",421.81,421.00,420.53,419.31,423.62,26,ok, +28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",966.80,961.41,961.46,957.40,965.33,556,ok, +29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",396.16,390.69,390.66,387.70,393.73,990,ok, +30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",236.87,232.38,232.35,232.17,232.65,244,ok, +31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",262.18,254.79,254.79,252.74,256.84,398,ok, +32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",967.89,741.95,727.78,712.38,799.88,376,ok, +33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",368.98,380.07,369.53,362.34,418.87,464,ok, +34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",386.18,427.59,408.60,395.22,497.95,484,ok, +35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",149.27,215.79,227.39,164.34,244.05,493,ok, +36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",367.61,349.09,352.00,328.04,364.33,569,ok, +37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",255.09,253.91,240.26,238.20,296.91,550,ok, +38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",78.61,81.87,82.36,72.82,89.95,480,ok, +39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",546.84,468.97,474.69,447.05,479.44,835,ok, +40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",113.76,96.55,96.72,94.74,98.03,356,ok, +41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",39.03,35.20,35.08,34.74,35.88,132,ok, +42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",78.57,68.56,68.79,67.35,69.30,240,ok, From d202b8f4f41df8485751ccecb509f58faa437d73 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 14:46:19 +0300 Subject: [PATCH 06/13] feat: added string arena --- include/common/string_arena.h | 37 +++++++++++++++++++++ src/CMakeLists.txt | 1 + src/common/string_arena.cpp | 48 ++++++++++++++++++++++++++++ src/executor/aggregate_states.cpp | 6 ++-- src/executor/operators_aggregate.cpp | 21 ++++++------ tests/test_executor.cpp | 8 ----- 6 files changed, 102 insertions(+), 19 deletions(-) create mode 100644 include/common/string_arena.h create mode 100644 src/common/string_arena.cpp diff --git a/include/common/string_arena.h b/include/common/string_arena.h new file mode 100644 index 0000000..5411ae7 --- /dev/null +++ b/include/common/string_arena.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include +#include +#include + +class StringArena { + public: + explicit StringArena(size_t chunk_size = 4096); + + std::string_view Store(std::string_view value); + void Clear(); + + private: + struct Chunk { + std::unique_ptr data; + size_t capacity = 0; + size_t used = 0; + }; + + Chunk& EnsureChunk(size_t size); + + size_t chunk_size_ = 0; + std::vector chunks_; +}; + +struct StringViewHash { + size_t operator()(std::string_view value) const noexcept; +}; + +struct StringViewEqual { + bool operator()(std::string_view lhs, std::string_view rhs) const noexcept; +}; + +using StringViewSet = std::unordered_set; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 72b2d7b..9ff24dd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,6 +23,7 @@ add_library(columnar_engine_core common/parsing.cpp common/operator_utils.cpp common/planner_utils.cpp + common/string_arena.cpp ) target_include_directories(columnar_engine_core PUBLIC ${COLUMNAR_INCLUDE_DIRS}) diff --git a/src/common/string_arena.cpp b/src/common/string_arena.cpp new file mode 100644 index 0000000..b1165dd --- /dev/null +++ b/src/common/string_arena.cpp @@ -0,0 +1,48 @@ +#include "common/string_arena.h" + +#include +#include +#include + +StringArena::StringArena(const size_t chunk_size) : chunk_size_(chunk_size) {} + +std::string_view StringArena::Store(const std::string_view value) { + if (value.empty()) { + return {}; + } + + Chunk& chunk = EnsureChunk(value.size()); + const size_t offset = chunk.used; + std::memcpy(chunk.data.get() + static_cast(offset), value.data(), value.size()); + chunk.used = offset + value.size(); + return {chunk.data.get() + static_cast(offset), value.size()}; +} + +void StringArena::Clear() { + for (Chunk& chunk : chunks_) { + chunk.used = 0; + } +} + +StringArena::Chunk& StringArena::EnsureChunk(const size_t size) { + if (!chunks_.empty() && chunks_.back().capacity - chunks_.back().used >= size) { + return chunks_.back(); + } + + const size_t capacity = std::max(chunk_size_, size); + chunks_.push_back(Chunk{ + .data = std::make_unique(capacity), + .capacity = capacity, + .used = 0, + }); + + return chunks_.back(); +} + +size_t StringViewHash::operator()(const std::string_view value) const noexcept { + return std::hash{}(value); +} + +bool StringViewEqual::operator()(const std::string_view lhs, const std::string_view rhs) const noexcept { + return lhs == rhs; +} diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index d86669f..9ea9c8c 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -8,6 +8,7 @@ #include "common/error.h" #include "common/int128.h" #include "common/parsing.h" +#include "common/string_arena.h" #include "executor/aggregate_function.h" #include "executor/aggregate_state.h" #include "model/column_traits.h" @@ -176,7 +177,7 @@ class DistinctAS final : public AggState { explicit DistinctAS(std::unique_ptr nested) : nested_(std::move(nested)) {} void ConsumeValue(const std::string_view value) override { - if (seen_.insert(std::string(value)).second) { + if (seen_.insert(arena_.Store(value)).second) { nested_->ConsumeValue(value); } } @@ -187,7 +188,8 @@ class DistinctAS final : public AggState { private: std::unique_ptr nested_; - std::unordered_set seen_; + StringArena arena_; + StringViewSet seen_; }; static bool SupportsAnyType(const ColumnType) { return true; } diff --git a/src/executor/operators_aggregate.cpp b/src/executor/operators_aggregate.cpp index dd06f7e..2f00d76 100644 --- a/src/executor/operators_aggregate.cpp +++ b/src/executor/operators_aggregate.cpp @@ -9,6 +9,7 @@ #include "common/error.h" #include "common/operator_utils.h" #include "common/parsing.h" +#include "common/string_arena.h" #include "executor/aggregate_function.h" #include "executor/aggregate_state.h" #include "executor/operators_internal.h" @@ -68,7 +69,7 @@ struct AggBinding { struct GroupKeyComponent { ColumnType type = ColumnType::String; Int128 int_value = 0; - std::string string_value; + std::string_view string_value; }; std::optional TryEvalTypedGroupKeyInt(const ExprPtr& expr, const Batch& batch, const size_t row) { @@ -187,7 +188,7 @@ std::string FormatGroupKeyValue(const GroupKeyComponent& value) { case ColumnType::Character: return std::string(1, static_cast(value.int_value)); case ColumnType::String: - return value.string_value; + return std::string(value.string_value); } return {}; @@ -231,8 +232,8 @@ struct TypedGroupKeyHash { for (const GroupKeyComponent& value : key.values) { hash = HashCombine(hash, std::hash{}(static_cast(value.type))); - hash = HashCombine(hash, value.type == ColumnType::String ? std::hash{}(value.string_value) - : HashInt128(value.int_value)); + hash = HashCombine(hash, value.type == ColumnType::String ? StringViewHash{}(value.string_value) + : HashInt128(value.int_value)); } return hash; @@ -259,7 +260,7 @@ class GroupKeyMaterializer { public: explicit GroupKeyMaterializer(std::vector group_keys) : group_keys_(std::move(group_keys)) {} - TypedGroupKey Materialize(const Batch& batch, const size_t row) const { + TypedGroupKey Materialize(const Batch& batch, const size_t row, StringArena& arena) const { TypedGroupKey key; key.values.reserve(group_keys_.size()); @@ -280,7 +281,7 @@ class GroupKeyMaterializer { key.values.push_back(GroupKeyComponent{ .type = group_key.column_type, .int_value = 0, - .string_value = batch.ColumnAt(group_key.expression->column_index).ValueAsString(row), + .string_value = arena.Store(batch.ColumnAt(group_key.expression->column_index).ValueAsString(row)), }); continue; } @@ -290,7 +291,7 @@ class GroupKeyMaterializer { key.values.push_back(GroupKeyComponent{ .type = group_key.column_type, .int_value = 0, - .string_value = std::move(value), + .string_value = arena.Store(value), }); } else { key.values.push_back(GroupKeyComponent{ @@ -387,7 +388,7 @@ class GroupAggOperator final : public Operator { SelectRowsMatchingPredicate(filter_, *batch, selected_rows); for (const size_t row : selected_rows) { - TypedGroupKey key = group_key_materializer_.Materialize(*batch, row); + TypedGroupKey key = group_key_materializer_.Materialize(*batch, row, string_arena_); size_t group_index = 0; const auto it = group_indexes_.find(key); @@ -517,6 +518,7 @@ class GroupAggOperator final : public Operator { std::vector select_items_; PredicatePtr filter_; PredicatePtr having_; + StringArena string_arena_; std::unordered_map group_indexes_; std::vector groups_; bool sort_by_group_keys_ = false; @@ -555,7 +557,7 @@ class GroupAggTopKOperator final : public Operator { SelectRowsMatchingPredicate(filter_, *batch, selected_rows); for (const size_t row : selected_rows) { - TypedGroupKey key = group_key_materializer_.Materialize(*batch, row); + TypedGroupKey key = group_key_materializer_.Materialize(*batch, row, string_arena_); size_t group_index = 0; const auto it = group_indexes_.find(key); @@ -698,6 +700,7 @@ class GroupAggTopKOperator final : public Operator { PredicatePtr filter_; std::vector order_by_; size_t limit_ = 0; + StringArena string_arena_; std::unordered_map group_indexes_; mutable std::vector groups_; bool returned_ = false; diff --git a/tests/test_executor.cpp b/tests/test_executor.cpp index 409bf49..3dd9b80 100644 --- a/tests/test_executor.cpp +++ b/tests/test_executor.cpp @@ -312,14 +312,6 @@ TEST(executor, supports_group_by_computed_numeric_and_timestamp_keys) { } } -TEST(executor, preserves_schema_for_empty_grouped_result_after_offset) { - const Batch batch = BuildHitsTable( - "SELECT RegionID, COUNT(*) AS c FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10 OFFSET 100;"); - - EXPECT_EQ(BatchColumnNames(batch), (std::vector{"RegionID", "c"})); - EXPECT_TRUE(BatchRows(batch).empty()); -} - TEST(executor, prunes_row_groups_for_typed_where_filters_before_reading_chunks) { const TempFile schema_file("executor_prune_schema"); const TempFile data_file("executor_prune_data"); From ae5bb96b3b0c757fbe1bfefab1e0e07e5df1c0a3 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 14:57:35 +0300 Subject: [PATCH 07/13] upd: update dashboard --- README.md | 154 +++++++++++------------ benchmarks/results/readme_benchmarks.csv | 86 ++++++------- 2 files changed, 120 insertions(+), 120 deletions(-) diff --git a/README.md b/README.md index fb86f57..f1823de 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## Benchmark Dashboard -`generated 2026-05-28 14:14:38` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) +`generated 2026-05-28 14:48:45` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) ### Summary @@ -13,14 +13,14 @@ | --- | ---: | | queries ok | 43 / 43 | | queries failed | 0 | -| median warm time | 227.39 ms | -| average warm time | 227.85 ms | -| p95 warm time | 469.27 ms | -| cold/warm delta | 4.1% | +| median warm time | 225.66 ms | +| average warm time | 221.06 ms | +| p95 warm time | 422.19 ms | +| cold/warm delta | 8.0% | | total output size | 20.3 KB | | max output size | 6.0 KB | -| fastest query | Q19 · 4.30 ms | -| slowest query | Q28 · 961.46 ms | +| fastest query | Q19 · 4.33 ms | +| slowest query | Q28 · 947.30 ms | ### Storage @@ -34,10 +34,10 @@ | roundtrip csv size | 743.0 MB | | compression ratio | 6.71x | | columnar / csv | 14.9% | -| csv -> columnar | 5.65s | -| columnar -> csv | 9.55s | -| convert throughput | 142.2 MB/s | -| roundtrip throughput | 12.5 MB/s | +| csv -> columnar | 5.51s | +| columnar -> csv | 9.40s | +| convert throughput | 145.8 MB/s | +| roundtrip throughput | 12.7 MB/s | ### Heatmap @@ -45,89 +45,89 @@ | slot 1 | slot 2 | slot 3 | slot 4 | slot 5 | slot 6 | | ---: | ---: | ---: | ---: | ---: | ---: | -| Q00 🟩 `5.66` | Q01 🟩 `8.52` | Q02 🟩 `18.69` | Q03 🟩 `15.81` | Q04 🟩 `167.19` | Q05 🟩 `142.57` | -| Q06 🟩 `4.81` | Q07 🟩 `8.51` | Q08 🟩 `191.42` | Q09 🟦 `204.68` | Q10 🟥 `266.03` | Q11 🟥 `267.35` | -| Q12 🟦 `225.01` | Q13 🟦 `239.70` | Q14 🟦 `228.39` | Q15 🟩 `70.37` | Q16 🟩 `120.24` | Q17 🟩 `141.30` | -| Q18 🟥 `334.36` | Q19 🟩 `4.30` | Q20 🟩 `142.30` | Q21 🟥 `273.15` | Q22 🟥 `365.64` | Q23 🟦 `242.96` | -| Q24 🟥 `271.73` | Q25 🟦 `222.25` | Q26 🟥 `271.67` | Q27 🟥 `420.53` | Q28 🟥 `961.46` | Q29 🟥 `390.66` | -| Q30 🟦 `232.35` | Q31 🟥 `254.79` | Q32 🟥 `727.78` | Q33 🟥 `369.53` | Q34 🟥 `408.60` | Q35 🟦 `227.39` | -| Q36 🟥 `352.00` | Q37 🟦 `240.26` | Q38 🟩 `82.36` | Q39 🟥 `474.69` | Q40 🟩 `96.72` | Q41 🟩 `35.08` | -| Q42 🟩 `68.79` | | | | | | +| Q00 🟩 `4.66` | Q01 🟩 `6.11` | Q02 🟩 `17.51` | Q03 🟩 `15.12` | Q04 🟩 `151.46` | Q05 🟩 `131.94` | +| Q06 🟩 `4.54` | Q07 🟩 `7.43` | Q08 🟩 `199.65` | Q09 🟦 `214.43` | Q10 🟥 `264.49` | Q11 🟥 `270.13` | +| Q12 🟦 `231.61` | Q13 🟦 `238.39` | Q14 🟦 `225.66` | Q15 🟩 `70.48` | Q16 🟩 `119.72` | Q17 🟩 `140.51` | +| Q18 🟥 `336.95` | Q19 🟩 `4.33` | Q20 🟩 `141.07` | Q21 🟥 `271.62` | Q22 🟥 `365.63` | Q23 🟦 `241.79` | +| Q24 🟥 `268.95` | Q25 🟦 `220.43` | Q26 🟥 `269.25` | Q27 🟥 `416.50` | Q28 🟥 `947.30` | Q29 🟥 `382.06` | +| Q30 🟦 `230.35` | Q31 🟥 `253.73` | Q32 🟥 `704.87` | Q33 🟥 `365.28` | Q34 🟥 `384.90` | Q35 🟩 `143.84` | +| Q36 🟥 `321.13` | Q37 🟦 `235.24` | Q38 🟩 `72.92` | Q39 🟥 `422.82` | Q40 🟩 `90.24` | Q41 🟩 `32.69` | +| Q42 🟩 `67.86` | | | | | | ### Largest Result Sets | Query | Output | Warm median, ms | SQL | | --- | ---: | ---: | --- | -| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 365.64 | SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT… | -| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 242.96 | SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime… | -| [Q29](benchmarks/queries/query_29.sql) | 990 B | 390.66 | SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(Reso… | -| [Q39](benchmarks/queries/query_39.sql) | 835 B | 474.69 | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (… | -| [Q24](benchmarks/queries/query_24.sql) | 658 B | 271.73 | SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY… | +| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 365.63 | SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT… | +| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 241.79 | SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime… | +| [Q29](benchmarks/queries/query_29.sql) | 990 B | 382.06 | SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(Reso… | +| [Q39](benchmarks/queries/query_39.sql) | 835 B | 422.82 | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (… | +| [Q24](benchmarks/queries/query_24.sql) | 658 B | 268.95 | SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY… | ### Fastest Queries | Query | SQL | Warm median, ms | First run, ms | | --- | --- | ---: | ---: | -| [Q19](benchmarks/queries/query_19.sql) | SELECT UserID FROM hits WHERE UserID = 435090932899640449; | 4.30 | 5.03 | -| [Q06](benchmarks/queries/query_6.sql) | SELECT MIN(EventDate), MAX(EventDate) FROM hits; | 4.81 | 4.83 | -| [Q00](benchmarks/queries/query_0.sql) | SELECT COUNT(*) FROM hits; | 5.66 | 6.73 | -| [Q07](benchmarks/queries/query_7.sql) | SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY … | 8.51 | 11.26 | -| [Q01](benchmarks/queries/query_1.sql) | SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; | 8.52 | 7.83 | +| [Q19](benchmarks/queries/query_19.sql) | SELECT UserID FROM hits WHERE UserID = 435090932899640449; | 4.33 | 5.07 | +| [Q06](benchmarks/queries/query_6.sql) | SELECT MIN(EventDate), MAX(EventDate) FROM hits; | 4.54 | 5.32 | +| [Q00](benchmarks/queries/query_0.sql) | SELECT COUNT(*) FROM hits; | 4.66 | 32.52 | +| [Q01](benchmarks/queries/query_1.sql) | SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; | 6.11 | 15.03 | +| [Q07](benchmarks/queries/query_7.sql) | SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY … | 7.43 | 6.70 | ### Slowest Queries | Query | SQL | Warm median, ms | Warm max, ms | | --- | --- | ---: | ---: | -| [Q28](benchmarks/queries/query_28.sql) | SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1')… | 961.46 | 965.33 | -| [Q32](benchmarks/queries/query_32.sql) | SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(Resolution… | 727.78 | 799.88 | -| [Q39](benchmarks/queries/query_39.sql) | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEn… | 474.69 | 479.44 | -| [Q27](benchmarks/queries/query_27.sql) | SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE … | 420.53 | 423.62 | -| [Q34](benchmarks/queries/query_34.sql) | SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC … | 408.60 | 497.95 | +| [Q28](benchmarks/queries/query_28.sql) | SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1')… | 947.30 | 947.70 | +| [Q32](benchmarks/queries/query_32.sql) | SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(Resolution… | 704.87 | 709.81 | +| [Q39](benchmarks/queries/query_39.sql) | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEn… | 422.82 | 433.09 | +| [Q27](benchmarks/queries/query_27.sql) | SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE … | 416.50 | 418.36 | +| [Q34](benchmarks/queries/query_34.sql) | SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC … | 384.90 | 387.53 | ### Query Table | Query | Output CSV | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | -| [Q00](benchmarks/queries/query_0.sql) | 7 B | 6.73 | 5.73 | 5.66 | 5.04 | 6.58 | ok | -| [Q01](benchmarks/queries/query_1.sql) | 6 B | 7.83 | 8.39 | 8.52 | 7.40 | 9.11 | ok | -| [Q02](benchmarks/queries/query_2.sql) | 18 B | 25.17 | 18.62 | 18.69 | 18.22 | 18.89 | ok | -| [Q03](benchmarks/queries/query_3.sql) | 20 B | 14.89 | 16.28 | 15.81 | 14.92 | 18.60 | ok | -| [Q04](benchmarks/queries/query_4.sql) | 6 B | 169.19 | 166.66 | 167.19 | 160.12 | 172.16 | ok | -| [Q05](benchmarks/queries/query_5.sql) | 6 B | 158.88 | 142.99 | 142.57 | 139.13 | 147.68 | ok | -| [Q06](benchmarks/queries/query_6.sql) | 22 B | 4.83 | 4.81 | 4.81 | 4.51 | 5.09 | ok | -| [Q07](benchmarks/queries/query_7.sql) | 31 B | 11.26 | 9.06 | 8.51 | 8.09 | 11.12 | ok | -| [Q08](benchmarks/queries/query_8.sql) | 81 B | 220.05 | 191.69 | 191.42 | 188.73 | 195.20 | ok | -| [Q09](benchmarks/queries/query_9.sql) | 237 B | 205.60 | 204.66 | 204.68 | 204.17 | 205.11 | ok | -| [Q10](benchmarks/queries/query_10.sql) | 102 B | 273.84 | 266.00 | 266.03 | 265.52 | 266.41 | ok | -| [Q11](benchmarks/queries/query_11.sql) | 116 B | 268.04 | 267.38 | 267.35 | 267.27 | 267.58 | ok | -| [Q12](benchmarks/queries/query_12.sql) | 387 B | 224.63 | 224.93 | 225.01 | 224.44 | 225.28 | ok | -| [Q13](benchmarks/queries/query_13.sql) | 350 B | 240.61 | 239.63 | 239.70 | 238.61 | 240.50 | ok | -| [Q14](benchmarks/queries/query_14.sql) | 400 B | 228.67 | 228.32 | 228.39 | 227.04 | 229.47 | ok | -| [Q15](benchmarks/queries/query_15.sql) | 236 B | 71.09 | 70.54 | 70.37 | 69.94 | 71.46 | ok | -| [Q16](benchmarks/queries/query_16.sql) | 244 B | 122.74 | 120.44 | 120.24 | 119.31 | 121.99 | ok | -| [Q17](benchmarks/queries/query_17.sql) | 414 B | 141.73 | 141.49 | 141.30 | 140.31 | 143.04 | ok | -| [Q18](benchmarks/queries/query_18.sql) | 264 B | 369.08 | 335.98 | 334.36 | 333.21 | 342.01 | ok | -| [Q19](benchmarks/queries/query_19.sql) | 0 B | 5.03 | 4.30 | 4.30 | 4.27 | 4.34 | ok | -| [Q20](benchmarks/queries/query_20.sql) | 3 B | 143.32 | 142.33 | 142.30 | 142.00 | 142.71 | ok | -| [Q21](benchmarks/queries/query_21.sql) | 162 B | 271.64 | 272.93 | 273.15 | 271.58 | 273.83 | ok | -| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 366.39 | 365.99 | 365.64 | 364.45 | 368.23 | ok | -| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 249.63 | 242.91 | 242.96 | 242.01 | 243.72 | ok | -| [Q24](benchmarks/queries/query_24.sql) | 658 B | 271.33 | 271.81 | 271.73 | 271.32 | 272.46 | ok | -| [Q25](benchmarks/queries/query_25.sql) | 382 B | 223.93 | 222.37 | 222.25 | 221.78 | 223.21 | ok | -| [Q26](benchmarks/queries/query_26.sql) | 658 B | 271.34 | 271.66 | 271.67 | 271.41 | 271.90 | ok | -| [Q27](benchmarks/queries/query_27.sql) | 26 B | 421.81 | 421.00 | 420.53 | 419.31 | 423.62 | ok | -| [Q28](benchmarks/queries/query_28.sql) | 556 B | 966.80 | 961.41 | 961.46 | 957.40 | 965.33 | ok | -| [Q29](benchmarks/queries/query_29.sql) | 990 B | 396.16 | 390.69 | 390.66 | 387.70 | 393.73 | ok | -| [Q30](benchmarks/queries/query_30.sql) | 244 B | 236.87 | 232.38 | 232.35 | 232.17 | 232.65 | ok | -| [Q31](benchmarks/queries/query_31.sql) | 398 B | 262.18 | 254.79 | 254.79 | 252.74 | 256.84 | ok | -| [Q32](benchmarks/queries/query_32.sql) | 376 B | 967.89 | 741.95 | 727.78 | 712.38 | 799.88 | ok | -| [Q33](benchmarks/queries/query_33.sql) | 464 B | 368.98 | 380.07 | 369.53 | 362.34 | 418.87 | ok | -| [Q34](benchmarks/queries/query_34.sql) | 484 B | 386.18 | 427.59 | 408.60 | 395.22 | 497.95 | ok | -| [Q35](benchmarks/queries/query_35.sql) | 493 B | 149.27 | 215.79 | 227.39 | 164.34 | 244.05 | ok | -| [Q36](benchmarks/queries/query_36.sql) | 569 B | 367.61 | 349.09 | 352.00 | 328.04 | 364.33 | ok | -| [Q37](benchmarks/queries/query_37.sql) | 550 B | 255.09 | 253.91 | 240.26 | 238.20 | 296.91 | ok | -| [Q38](benchmarks/queries/query_38.sql) | 480 B | 78.61 | 81.87 | 82.36 | 72.82 | 89.95 | ok | -| [Q39](benchmarks/queries/query_39.sql) | 835 B | 546.84 | 468.97 | 474.69 | 447.05 | 479.44 | ok | -| [Q40](benchmarks/queries/query_40.sql) | 356 B | 113.76 | 96.55 | 96.72 | 94.74 | 98.03 | ok | -| [Q41](benchmarks/queries/query_41.sql) | 132 B | 39.03 | 35.20 | 35.08 | 34.74 | 35.88 | ok | -| [Q42](benchmarks/queries/query_42.sql) | 240 B | 78.57 | 68.56 | 68.79 | 67.35 | 69.30 | ok | +| [Q00](benchmarks/queries/query_0.sql) | 7 B | 32.52 | 4.63 | 4.66 | 4.45 | 4.74 | ok | +| [Q01](benchmarks/queries/query_1.sql) | 6 B | 15.03 | 6.21 | 6.11 | 6.08 | 6.53 | ok | +| [Q02](benchmarks/queries/query_2.sql) | 18 B | 27.26 | 17.87 | 17.51 | 17.33 | 19.14 | ok | +| [Q03](benchmarks/queries/query_3.sql) | 20 B | 28.56 | 14.98 | 15.12 | 13.54 | 16.16 | ok | +| [Q04](benchmarks/queries/query_4.sql) | 6 B | 149.26 | 180.46 | 151.46 | 142.19 | 276.73 | ok | +| [Q05](benchmarks/queries/query_5.sql) | 6 B | 150.42 | 132.22 | 131.94 | 131.34 | 133.66 | ok | +| [Q06](benchmarks/queries/query_6.sql) | 22 B | 5.32 | 4.58 | 4.54 | 4.21 | 5.05 | ok | +| [Q07](benchmarks/queries/query_7.sql) | 31 B | 6.70 | 7.46 | 7.43 | 7.19 | 7.81 | ok | +| [Q08](benchmarks/queries/query_8.sql) | 81 B | 201.60 | 199.34 | 199.65 | 194.41 | 203.66 | ok | +| [Q09](benchmarks/queries/query_9.sql) | 237 B | 212.51 | 218.14 | 214.43 | 210.86 | 232.86 | ok | +| [Q10](benchmarks/queries/query_10.sql) | 102 B | 276.50 | 264.53 | 264.49 | 263.64 | 265.52 | ok | +| [Q11](benchmarks/queries/query_11.sql) | 116 B | 267.70 | 271.02 | 270.13 | 265.75 | 278.06 | ok | +| [Q12](benchmarks/queries/query_12.sql) | 387 B | 233.29 | 249.14 | 231.61 | 228.06 | 305.29 | ok | +| [Q13](benchmarks/queries/query_13.sql) | 350 B | 253.15 | 238.86 | 238.39 | 237.53 | 241.14 | ok | +| [Q14](benchmarks/queries/query_14.sql) | 400 B | 225.86 | 225.66 | 225.66 | 225.40 | 225.93 | ok | +| [Q15](benchmarks/queries/query_15.sql) | 236 B | 70.78 | 70.34 | 70.48 | 69.77 | 70.64 | ok | +| [Q16](benchmarks/queries/query_16.sql) | 244 B | 119.70 | 119.67 | 119.72 | 119.02 | 120.24 | ok | +| [Q17](benchmarks/queries/query_17.sql) | 414 B | 139.96 | 140.43 | 140.51 | 139.65 | 141.07 | ok | +| [Q18](benchmarks/queries/query_18.sql) | 264 B | 402.60 | 336.78 | 336.95 | 334.78 | 338.43 | ok | +| [Q19](benchmarks/queries/query_19.sql) | 0 B | 5.07 | 4.35 | 4.33 | 4.26 | 4.49 | ok | +| [Q20](benchmarks/queries/query_20.sql) | 3 B | 186.80 | 141.09 | 141.07 | 140.79 | 141.45 | ok | +| [Q21](benchmarks/queries/query_21.sql) | 162 B | 271.28 | 271.51 | 271.62 | 270.68 | 272.11 | ok | +| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 408.96 | 366.24 | 365.63 | 362.18 | 371.51 | ok | +| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 279.14 | 242.44 | 241.79 | 240.90 | 245.29 | ok | +| [Q24](benchmarks/queries/query_24.sql) | 658 B | 270.04 | 269.01 | 268.95 | 268.55 | 269.59 | ok | +| [Q25](benchmarks/queries/query_25.sql) | 382 B | 220.27 | 220.38 | 220.43 | 219.87 | 220.80 | ok | +| [Q26](benchmarks/queries/query_26.sql) | 658 B | 270.29 | 270.39 | 269.25 | 268.48 | 274.56 | ok | +| [Q27](benchmarks/queries/query_27.sql) | 26 B | 420.06 | 416.45 | 416.50 | 414.43 | 418.36 | ok | +| [Q28](benchmarks/queries/query_28.sql) | 556 B | 972.67 | 947.10 | 947.30 | 946.08 | 947.70 | ok | +| [Q29](benchmarks/queries/query_29.sql) | 990 B | 386.30 | 383.47 | 382.06 | 381.66 | 388.12 | ok | +| [Q30](benchmarks/queries/query_30.sql) | 244 B | 232.60 | 230.63 | 230.35 | 228.93 | 232.88 | ok | +| [Q31](benchmarks/queries/query_31.sql) | 398 B | 269.77 | 260.09 | 253.73 | 250.18 | 282.74 | ok | +| [Q32](benchmarks/queries/query_32.sql) | 376 B | 1062.67 | 704.38 | 704.87 | 697.97 | 709.81 | ok | +| [Q33](benchmarks/queries/query_33.sql) | 464 B | 368.34 | 365.48 | 365.28 | 364.19 | 367.15 | ok | +| [Q34](benchmarks/queries/query_34.sql) | 484 B | 384.09 | 385.42 | 384.90 | 384.34 | 387.53 | ok | +| [Q35](benchmarks/queries/query_35.sql) | 493 B | 144.54 | 143.79 | 143.84 | 142.83 | 144.66 | ok | +| [Q36](benchmarks/queries/query_36.sql) | 569 B | 329.99 | 320.62 | 321.13 | 318.63 | 321.57 | ok | +| [Q37](benchmarks/queries/query_37.sql) | 550 B | 236.91 | 235.07 | 235.24 | 233.68 | 236.12 | ok | +| [Q38](benchmarks/queries/query_38.sql) | 480 B | 79.61 | 73.12 | 72.92 | 70.87 | 75.78 | ok | +| [Q39](benchmarks/queries/query_39.sql) | 835 B | 436.78 | 424.89 | 422.82 | 420.83 | 433.09 | ok | +| [Q40](benchmarks/queries/query_40.sql) | 356 B | 106.64 | 90.29 | 90.24 | 90.09 | 90.58 | ok | +| [Q41](benchmarks/queries/query_41.sql) | 132 B | 37.17 | 32.69 | 32.69 | 32.52 | 32.87 | ok | +| [Q42](benchmarks/queries/query_42.sql) | 240 B | 67.63 | 67.83 | 67.86 | 66.65 | 68.96 | ok | diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index 23f171b..a09ebbe 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,44 +1,44 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,output_csv_bytes,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,6.73,5.73,5.66,5.04,6.58,7,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,7.83,8.39,8.52,7.40,9.11,6,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",25.17,18.62,18.69,18.22,18.89,18,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,14.89,16.28,15.81,14.92,18.60,20,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,169.19,166.66,167.19,160.12,172.16,6,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,158.88,142.99,142.57,139.13,147.68,6,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",4.83,4.81,4.81,4.51,5.09,22,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",11.26,9.06,8.51,8.09,11.12,31,ok, -8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",220.05,191.69,191.42,188.73,195.20,81,ok, -9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",205.60,204.66,204.68,204.17,205.11,237,ok, -10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",273.84,266.00,266.03,265.52,266.41,102,ok, -11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",268.04,267.38,267.35,267.27,267.58,116,ok, -12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",224.63,224.93,225.01,224.44,225.28,387,ok, -13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",240.61,239.63,239.70,238.61,240.50,350,ok, -14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",228.67,228.32,228.39,227.04,229.47,400,ok, -15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",71.09,70.54,70.37,69.94,71.46,236,ok, -16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",122.74,120.44,120.24,119.31,121.99,244,ok, -17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",141.73,141.49,141.30,140.31,143.04,414,ok, -18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",369.08,335.98,334.36,333.21,342.01,264,ok, -19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,5.03,4.30,4.30,4.27,4.34,0,ok, -20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,143.32,142.33,142.30,142.00,142.71,3,ok, -21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",271.64,272.93,273.15,271.58,273.83,162,ok, -22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",366.39,365.99,365.64,364.45,368.23,6188,ok, -23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,249.63,242.91,242.96,242.01,243.72,2633,ok, -24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,271.33,271.81,271.73,271.32,272.46,658,ok, -25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,223.93,222.37,222.25,221.78,223.21,382,ok, -26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",271.34,271.66,271.67,271.41,271.90,658,ok, -27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",421.81,421.00,420.53,419.31,423.62,26,ok, -28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",966.80,961.41,961.46,957.40,965.33,556,ok, -29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",396.16,390.69,390.66,387.70,393.73,990,ok, -30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",236.87,232.38,232.35,232.17,232.65,244,ok, -31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",262.18,254.79,254.79,252.74,256.84,398,ok, -32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",967.89,741.95,727.78,712.38,799.88,376,ok, -33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",368.98,380.07,369.53,362.34,418.87,464,ok, -34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",386.18,427.59,408.60,395.22,497.95,484,ok, -35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",149.27,215.79,227.39,164.34,244.05,493,ok, -36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",367.61,349.09,352.00,328.04,364.33,569,ok, -37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",255.09,253.91,240.26,238.20,296.91,550,ok, -38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",78.61,81.87,82.36,72.82,89.95,480,ok, -39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",546.84,468.97,474.69,447.05,479.44,835,ok, -40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",113.76,96.55,96.72,94.74,98.03,356,ok, -41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",39.03,35.20,35.08,34.74,35.88,132,ok, -42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",78.57,68.56,68.79,67.35,69.30,240,ok, +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,32.52,4.63,4.66,4.45,4.74,7,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,15.03,6.21,6.11,6.08,6.53,6,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",27.26,17.87,17.51,17.33,19.14,18,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,28.56,14.98,15.12,13.54,16.16,20,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,149.26,180.46,151.46,142.19,276.73,6,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,150.42,132.22,131.94,131.34,133.66,6,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",5.32,4.58,4.54,4.21,5.05,22,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",6.70,7.46,7.43,7.19,7.81,31,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",201.60,199.34,199.65,194.41,203.66,81,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",212.51,218.14,214.43,210.86,232.86,237,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",276.50,264.53,264.49,263.64,265.52,102,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",267.70,271.02,270.13,265.75,278.06,116,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",233.29,249.14,231.61,228.06,305.29,387,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",253.15,238.86,238.39,237.53,241.14,350,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",225.86,225.66,225.66,225.40,225.93,400,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",70.78,70.34,70.48,69.77,70.64,236,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",119.70,119.67,119.72,119.02,120.24,244,ok, +17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",139.96,140.43,140.51,139.65,141.07,414,ok, +18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",402.60,336.78,336.95,334.78,338.43,264,ok, +19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,5.07,4.35,4.33,4.26,4.49,0,ok, +20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,186.80,141.09,141.07,140.79,141.45,3,ok, +21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",271.28,271.51,271.62,270.68,272.11,162,ok, +22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",408.96,366.24,365.63,362.18,371.51,6188,ok, +23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,279.14,242.44,241.79,240.90,245.29,2633,ok, +24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,270.04,269.01,268.95,268.55,269.59,658,ok, +25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,220.27,220.38,220.43,219.87,220.80,382,ok, +26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",270.29,270.39,269.25,268.48,274.56,658,ok, +27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",420.06,416.45,416.50,414.43,418.36,26,ok, +28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",972.67,947.10,947.30,946.08,947.70,556,ok, +29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",386.30,383.47,382.06,381.66,388.12,990,ok, +30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",232.60,230.63,230.35,228.93,232.88,244,ok, +31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",269.77,260.09,253.73,250.18,282.74,398,ok, +32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",1062.67,704.38,704.87,697.97,709.81,376,ok, +33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",368.34,365.48,365.28,364.19,367.15,464,ok, +34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",384.09,385.42,384.90,384.34,387.53,484,ok, +35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",144.54,143.79,143.84,142.83,144.66,493,ok, +36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",329.99,320.62,321.13,318.63,321.57,569,ok, +37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",236.91,235.07,235.24,233.68,236.12,550,ok, +38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",79.61,73.12,72.92,70.87,75.78,480,ok, +39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",436.78,424.89,422.82,420.83,433.09,835,ok, +40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",106.64,90.29,90.24,90.09,90.58,356,ok, +41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",37.17,32.69,32.69,32.52,32.87,132,ok, +42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",67.63,67.83,67.86,66.65,68.96,240,ok, From e3a5dc86b0694519f422578e586ccf81be16f906 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 15:08:40 +0300 Subject: [PATCH 08/13] upd: update dashboard --- README.md | 208 +++++++++++------------ benchmarks/results/readme_benchmarks.csv | 86 +++++----- scripts/dashboard.py | 148 ++++++++-------- 3 files changed, 212 insertions(+), 230 deletions(-) diff --git a/README.md b/README.md index f1823de..2b700be 100644 --- a/README.md +++ b/README.md @@ -5,39 +5,50 @@ ## Benchmark Dashboard -`generated 2026-05-28 14:48:45` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) - -### Summary - -| Metric | Value | -| --- | ---: | -| queries ok | 43 / 43 | -| queries failed | 0 | -| median warm time | 225.66 ms | -| average warm time | 221.06 ms | -| p95 warm time | 422.19 ms | -| cold/warm delta | 8.0% | -| total output size | 20.3 KB | -| max output size | 6.0 KB | -| fastest query | Q19 · 4.33 ms | -| slowest query | Q28 · 947.30 ms | - -### Storage - -| Metric | Value | -| --- | ---: | -| source csv | `benchmarks/hits_sample.csv` | -| schema | `benchmarks/scheme.csv` | -| compression | `lz4` | -| source size | 802.6 MB | -| columnar size | 119.7 MB | -| roundtrip csv size | 743.0 MB | -| compression ratio | 6.71x | -| columnar / csv | 14.9% | -| csv -> columnar | 5.51s | -| columnar -> csv | 9.40s | -| convert throughput | 145.8 MB/s | -| roundtrip throughput | 12.7 MB/s | +`generated 2026-05-28 15:08:08` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) + + + + + + +
+

Summary

+ + + + + + + + + + + + + + +
MetricValue
queries ok43 / 43
queries failed0
median warm time229.34 ms
average warm time230.90 ms
p95 warm time446.96 ms
cold/warm delta6.3%
total output size20.3 KB
max output size6.0 KB
fastest queryQ06 · 4.26 ms
slowest queryQ28 · 979.97 ms
+
+

Storage

+ + + + + + + + + + + + + + + + +
MetricValue
source csvbenchmarks/hits_sample.csv
schemabenchmarks/scheme.csv
compressionlz4
source size802.6 MB
columnar size119.7 MB
roundtrip csv size743.0 MB
compression ratio6.71x
columnar / csv14.9%
csv -> columnar5.67s
columnar -> csv9.59s
convert throughput141.7 MB/s
roundtrip throughput12.5 MB/s
+
### Heatmap @@ -45,89 +56,60 @@ | slot 1 | slot 2 | slot 3 | slot 4 | slot 5 | slot 6 | | ---: | ---: | ---: | ---: | ---: | ---: | -| Q00 🟩 `4.66` | Q01 🟩 `6.11` | Q02 🟩 `17.51` | Q03 🟩 `15.12` | Q04 🟩 `151.46` | Q05 🟩 `131.94` | -| Q06 🟩 `4.54` | Q07 🟩 `7.43` | Q08 🟩 `199.65` | Q09 🟦 `214.43` | Q10 🟥 `264.49` | Q11 🟥 `270.13` | -| Q12 🟦 `231.61` | Q13 🟦 `238.39` | Q14 🟦 `225.66` | Q15 🟩 `70.48` | Q16 🟩 `119.72` | Q17 🟩 `140.51` | -| Q18 🟥 `336.95` | Q19 🟩 `4.33` | Q20 🟩 `141.07` | Q21 🟥 `271.62` | Q22 🟥 `365.63` | Q23 🟦 `241.79` | -| Q24 🟥 `268.95` | Q25 🟦 `220.43` | Q26 🟥 `269.25` | Q27 🟥 `416.50` | Q28 🟥 `947.30` | Q29 🟥 `382.06` | -| Q30 🟦 `230.35` | Q31 🟥 `253.73` | Q32 🟥 `704.87` | Q33 🟥 `365.28` | Q34 🟥 `384.90` | Q35 🟩 `143.84` | -| Q36 🟥 `321.13` | Q37 🟦 `235.24` | Q38 🟩 `72.92` | Q39 🟥 `422.82` | Q40 🟩 `90.24` | Q41 🟩 `32.69` | -| Q42 🟩 `67.86` | | | | | | -### Largest Result Sets - -| Query | Output | Warm median, ms | SQL | -| --- | ---: | ---: | --- | -| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 365.63 | SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT… | -| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 241.79 | SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime… | -| [Q29](benchmarks/queries/query_29.sql) | 990 B | 382.06 | SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(Reso… | -| [Q39](benchmarks/queries/query_39.sql) | 835 B | 422.82 | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (… | -| [Q24](benchmarks/queries/query_24.sql) | 658 B | 268.95 | SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY… | - -### Fastest Queries - -| Query | SQL | Warm median, ms | First run, ms | -| --- | --- | ---: | ---: | -| [Q19](benchmarks/queries/query_19.sql) | SELECT UserID FROM hits WHERE UserID = 435090932899640449; | 4.33 | 5.07 | -| [Q06](benchmarks/queries/query_6.sql) | SELECT MIN(EventDate), MAX(EventDate) FROM hits; | 4.54 | 5.32 | -| [Q00](benchmarks/queries/query_0.sql) | SELECT COUNT(*) FROM hits; | 4.66 | 32.52 | -| [Q01](benchmarks/queries/query_1.sql) | SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; | 6.11 | 15.03 | -| [Q07](benchmarks/queries/query_7.sql) | SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY … | 7.43 | 6.70 | - -### Slowest Queries - -| Query | SQL | Warm median, ms | Warm max, ms | -| --- | --- | ---: | ---: | -| [Q28](benchmarks/queries/query_28.sql) | SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1')… | 947.30 | 947.70 | -| [Q32](benchmarks/queries/query_32.sql) | SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(Resolution… | 704.87 | 709.81 | -| [Q39](benchmarks/queries/query_39.sql) | SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEn… | 422.82 | 433.09 | -| [Q27](benchmarks/queries/query_27.sql) | SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE … | 416.50 | 418.36 | -| [Q34](benchmarks/queries/query_34.sql) | SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC … | 384.90 | 387.53 | +| Q00 🟩 `4.87` | Q01 🟩 `7.24` | Q02 🟩 `18.79` | Q03 🟩 `14.52` | Q04 🟩 `167.49` | Q05 🟩 `132.28` | +| Q06 🟩 `4.26` | Q07 🟩 `7.09` | Q08 🟩 `204.97` | Q09 🟦 `217.93` | Q10 🟥 `268.97` | Q11 🟥 `269.99` | +| Q12 🟦 `229.34` | Q13 🟦 `246.09` | Q14 🟦 `232.91` | Q15 🟩 `75.99` | Q16 🟩 `129.28` | Q17 🟩 `152.80` | +| Q18 🟥 `374.64` | Q19 🟩 `5.12` | Q20 🟩 `145.69` | Q21 🟥 `277.66` | Q22 🟥 `374.66` | Q23 🟦 `244.56` | +| Q24 🟥 `268.40` | Q25 🟦 `219.57` | Q26 🟥 `270.06` | Q27 🟥 `420.86` | Q28 🟥 `979.97` | Q29 🟥 `391.93` | +| Q30 🟦 `237.88` | Q31 🟥 `266.14` | Q32 🟥 `776.24` | Q33 🟥 `405.27` | Q34 🟥 `422.68` | Q35 🟩 `155.09` | +| Q36 🟥 `342.61` | Q37 🟦 `243.88` | Q38 🟩 `74.44` | Q39 🟥 `449.66` | Q40 🟩 `93.72` | Q41 🟩 `34.09` | +| Q42 🟩 `69.12` | | | | | | ### Query Table | Query | Output CSV | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | -| [Q00](benchmarks/queries/query_0.sql) | 7 B | 32.52 | 4.63 | 4.66 | 4.45 | 4.74 | ok | -| [Q01](benchmarks/queries/query_1.sql) | 6 B | 15.03 | 6.21 | 6.11 | 6.08 | 6.53 | ok | -| [Q02](benchmarks/queries/query_2.sql) | 18 B | 27.26 | 17.87 | 17.51 | 17.33 | 19.14 | ok | -| [Q03](benchmarks/queries/query_3.sql) | 20 B | 28.56 | 14.98 | 15.12 | 13.54 | 16.16 | ok | -| [Q04](benchmarks/queries/query_4.sql) | 6 B | 149.26 | 180.46 | 151.46 | 142.19 | 276.73 | ok | -| [Q05](benchmarks/queries/query_5.sql) | 6 B | 150.42 | 132.22 | 131.94 | 131.34 | 133.66 | ok | -| [Q06](benchmarks/queries/query_6.sql) | 22 B | 5.32 | 4.58 | 4.54 | 4.21 | 5.05 | ok | -| [Q07](benchmarks/queries/query_7.sql) | 31 B | 6.70 | 7.46 | 7.43 | 7.19 | 7.81 | ok | -| [Q08](benchmarks/queries/query_8.sql) | 81 B | 201.60 | 199.34 | 199.65 | 194.41 | 203.66 | ok | -| [Q09](benchmarks/queries/query_9.sql) | 237 B | 212.51 | 218.14 | 214.43 | 210.86 | 232.86 | ok | -| [Q10](benchmarks/queries/query_10.sql) | 102 B | 276.50 | 264.53 | 264.49 | 263.64 | 265.52 | ok | -| [Q11](benchmarks/queries/query_11.sql) | 116 B | 267.70 | 271.02 | 270.13 | 265.75 | 278.06 | ok | -| [Q12](benchmarks/queries/query_12.sql) | 387 B | 233.29 | 249.14 | 231.61 | 228.06 | 305.29 | ok | -| [Q13](benchmarks/queries/query_13.sql) | 350 B | 253.15 | 238.86 | 238.39 | 237.53 | 241.14 | ok | -| [Q14](benchmarks/queries/query_14.sql) | 400 B | 225.86 | 225.66 | 225.66 | 225.40 | 225.93 | ok | -| [Q15](benchmarks/queries/query_15.sql) | 236 B | 70.78 | 70.34 | 70.48 | 69.77 | 70.64 | ok | -| [Q16](benchmarks/queries/query_16.sql) | 244 B | 119.70 | 119.67 | 119.72 | 119.02 | 120.24 | ok | -| [Q17](benchmarks/queries/query_17.sql) | 414 B | 139.96 | 140.43 | 140.51 | 139.65 | 141.07 | ok | -| [Q18](benchmarks/queries/query_18.sql) | 264 B | 402.60 | 336.78 | 336.95 | 334.78 | 338.43 | ok | -| [Q19](benchmarks/queries/query_19.sql) | 0 B | 5.07 | 4.35 | 4.33 | 4.26 | 4.49 | ok | -| [Q20](benchmarks/queries/query_20.sql) | 3 B | 186.80 | 141.09 | 141.07 | 140.79 | 141.45 | ok | -| [Q21](benchmarks/queries/query_21.sql) | 162 B | 271.28 | 271.51 | 271.62 | 270.68 | 272.11 | ok | -| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 408.96 | 366.24 | 365.63 | 362.18 | 371.51 | ok | -| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 279.14 | 242.44 | 241.79 | 240.90 | 245.29 | ok | -| [Q24](benchmarks/queries/query_24.sql) | 658 B | 270.04 | 269.01 | 268.95 | 268.55 | 269.59 | ok | -| [Q25](benchmarks/queries/query_25.sql) | 382 B | 220.27 | 220.38 | 220.43 | 219.87 | 220.80 | ok | -| [Q26](benchmarks/queries/query_26.sql) | 658 B | 270.29 | 270.39 | 269.25 | 268.48 | 274.56 | ok | -| [Q27](benchmarks/queries/query_27.sql) | 26 B | 420.06 | 416.45 | 416.50 | 414.43 | 418.36 | ok | -| [Q28](benchmarks/queries/query_28.sql) | 556 B | 972.67 | 947.10 | 947.30 | 946.08 | 947.70 | ok | -| [Q29](benchmarks/queries/query_29.sql) | 990 B | 386.30 | 383.47 | 382.06 | 381.66 | 388.12 | ok | -| [Q30](benchmarks/queries/query_30.sql) | 244 B | 232.60 | 230.63 | 230.35 | 228.93 | 232.88 | ok | -| [Q31](benchmarks/queries/query_31.sql) | 398 B | 269.77 | 260.09 | 253.73 | 250.18 | 282.74 | ok | -| [Q32](benchmarks/queries/query_32.sql) | 376 B | 1062.67 | 704.38 | 704.87 | 697.97 | 709.81 | ok | -| [Q33](benchmarks/queries/query_33.sql) | 464 B | 368.34 | 365.48 | 365.28 | 364.19 | 367.15 | ok | -| [Q34](benchmarks/queries/query_34.sql) | 484 B | 384.09 | 385.42 | 384.90 | 384.34 | 387.53 | ok | -| [Q35](benchmarks/queries/query_35.sql) | 493 B | 144.54 | 143.79 | 143.84 | 142.83 | 144.66 | ok | -| [Q36](benchmarks/queries/query_36.sql) | 569 B | 329.99 | 320.62 | 321.13 | 318.63 | 321.57 | ok | -| [Q37](benchmarks/queries/query_37.sql) | 550 B | 236.91 | 235.07 | 235.24 | 233.68 | 236.12 | ok | -| [Q38](benchmarks/queries/query_38.sql) | 480 B | 79.61 | 73.12 | 72.92 | 70.87 | 75.78 | ok | -| [Q39](benchmarks/queries/query_39.sql) | 835 B | 436.78 | 424.89 | 422.82 | 420.83 | 433.09 | ok | -| [Q40](benchmarks/queries/query_40.sql) | 356 B | 106.64 | 90.29 | 90.24 | 90.09 | 90.58 | ok | -| [Q41](benchmarks/queries/query_41.sql) | 132 B | 37.17 | 32.69 | 32.69 | 32.52 | 32.87 | ok | -| [Q42](benchmarks/queries/query_42.sql) | 240 B | 67.63 | 67.83 | 67.86 | 66.65 | 68.96 | ok | +| [Q00](benchmarks/queries/query_0.sql) | 7 B | 9.49 | 4.96 | 4.87 | 4.59 | 5.51 | ok | +| [Q01](benchmarks/queries/query_1.sql) | 6 B | 15.88 | 7.12 | 7.24 | 6.19 | 7.81 | ok | +| [Q02](benchmarks/queries/query_2.sql) | 18 B | 28.03 | 19.08 | 18.79 | 17.92 | 20.83 | ok | +| [Q03](benchmarks/queries/query_3.sql) | 20 B | 27.65 | 14.48 | 14.52 | 14.06 | 14.81 | ok | +| [Q04](benchmarks/queries/query_4.sql) | 6 B | 156.02 | 190.53 | 167.49 | 153.12 | 274.03 | ok | +| [Q05](benchmarks/queries/query_5.sql) | 6 B | 142.67 | 132.55 | 132.28 | 132.05 | 133.58 | ok | +| [Q06](benchmarks/queries/query_6.sql) | 22 B | 4.78 | 4.27 | 4.26 | 4.12 | 4.43 | ok | +| [Q07](benchmarks/queries/query_7.sql) | 31 B | 7.78 | 7.08 | 7.09 | 6.68 | 7.45 | ok | +| [Q08](benchmarks/queries/query_8.sql) | 81 B | 204.57 | 205.39 | 204.97 | 201.88 | 209.73 | ok | +| [Q09](benchmarks/queries/query_9.sql) | 237 B | 217.28 | 218.49 | 217.93 | 215.12 | 222.97 | ok | +| [Q10](benchmarks/queries/query_10.sql) | 102 B | 281.97 | 269.05 | 268.97 | 268.52 | 269.74 | ok | +| [Q11](benchmarks/queries/query_11.sql) | 116 B | 273.22 | 270.25 | 269.99 | 269.17 | 271.86 | ok | +| [Q12](benchmarks/queries/query_12.sql) | 387 B | 227.97 | 230.01 | 229.34 | 227.64 | 233.73 | ok | +| [Q13](benchmarks/queries/query_13.sql) | 350 B | 249.96 | 246.34 | 246.09 | 244.41 | 248.77 | ok | +| [Q14](benchmarks/queries/query_14.sql) | 400 B | 232.70 | 233.59 | 232.91 | 231.02 | 237.51 | ok | +| [Q15](benchmarks/queries/query_15.sql) | 236 B | 76.95 | 75.84 | 75.99 | 74.37 | 77.04 | ok | +| [Q16](benchmarks/queries/query_16.sql) | 244 B | 130.83 | 129.57 | 129.28 | 129.09 | 130.63 | ok | +| [Q17](benchmarks/queries/query_17.sql) | 414 B | 153.61 | 153.56 | 152.80 | 152.21 | 156.42 | ok | +| [Q18](benchmarks/queries/query_18.sql) | 264 B | 445.38 | 374.33 | 374.64 | 372.72 | 375.32 | ok | +| [Q19](benchmarks/queries/query_19.sql) | 0 B | 5.02 | 5.16 | 5.12 | 4.54 | 5.86 | ok | +| [Q20](benchmarks/queries/query_20.sql) | 3 B | 194.65 | 145.74 | 145.69 | 145.27 | 146.31 | ok | +| [Q21](benchmarks/queries/query_21.sql) | 162 B | 278.26 | 277.79 | 277.66 | 277.15 | 278.69 | ok | +| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 426.01 | 375.71 | 374.66 | 374.01 | 379.49 | ok | +| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 285.01 | 244.43 | 244.56 | 242.75 | 245.86 | ok | +| [Q24](benchmarks/queries/query_24.sql) | 658 B | 268.59 | 268.79 | 268.40 | 267.75 | 270.62 | ok | +| [Q25](benchmarks/queries/query_25.sql) | 382 B | 222.70 | 219.63 | 219.57 | 219.47 | 219.92 | ok | +| [Q26](benchmarks/queries/query_26.sql) | 658 B | 268.76 | 270.05 | 270.06 | 269.16 | 270.92 | ok | +| [Q27](benchmarks/queries/query_27.sql) | 26 B | 425.51 | 420.92 | 420.86 | 416.18 | 425.80 | ok | +| [Q28](benchmarks/queries/query_28.sql) | 556 B | 979.03 | 1016.62 | 979.97 | 974.72 | 1131.81 | ok | +| [Q29](benchmarks/queries/query_29.sql) | 990 B | 395.85 | 393.43 | 391.93 | 390.74 | 399.12 | ok | +| [Q30](benchmarks/queries/query_30.sql) | 244 B | 250.48 | 238.09 | 237.88 | 236.56 | 240.02 | ok | +| [Q31](benchmarks/queries/query_31.sql) | 398 B | 280.27 | 266.13 | 266.14 | 265.27 | 266.99 | ok | +| [Q32](benchmarks/queries/query_32.sql) | 376 B | 1030.95 | 777.39 | 776.24 | 769.61 | 787.48 | ok | +| [Q33](benchmarks/queries/query_33.sql) | 464 B | 403.13 | 404.75 | 405.27 | 402.76 | 405.69 | ok | +| [Q34](benchmarks/queries/query_34.sql) | 484 B | 453.77 | 422.89 | 422.68 | 419.71 | 426.50 | ok | +| [Q35](benchmarks/queries/query_35.sql) | 493 B | 156.60 | 155.02 | 155.09 | 153.26 | 156.64 | ok | +| [Q36](benchmarks/queries/query_36.sql) | 569 B | 347.88 | 342.61 | 342.61 | 342.31 | 342.92 | ok | +| [Q37](benchmarks/queries/query_37.sql) | 550 B | 244.48 | 243.77 | 243.88 | 242.73 | 244.58 | ok | +| [Q38](benchmarks/queries/query_38.sql) | 480 B | 80.37 | 74.58 | 74.44 | 74.34 | 75.09 | ok | +| [Q39](benchmarks/queries/query_39.sql) | 835 B | 453.18 | 450.37 | 449.66 | 447.42 | 454.74 | ok | +| [Q40](benchmarks/queries/query_40.sql) | 356 B | 111.18 | 93.89 | 93.72 | 93.59 | 94.54 | ok | +| [Q41](benchmarks/queries/query_41.sql) | 132 B | 37.92 | 34.08 | 34.09 | 33.77 | 34.37 | ok | +| [Q42](benchmarks/queries/query_42.sql) | 240 B | 68.61 | 69.07 | 69.12 | 68.65 | 69.40 | ok | diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index a09ebbe..e62935a 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,44 +1,44 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,output_csv_bytes,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,32.52,4.63,4.66,4.45,4.74,7,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,15.03,6.21,6.11,6.08,6.53,6,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",27.26,17.87,17.51,17.33,19.14,18,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,28.56,14.98,15.12,13.54,16.16,20,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,149.26,180.46,151.46,142.19,276.73,6,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,150.42,132.22,131.94,131.34,133.66,6,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",5.32,4.58,4.54,4.21,5.05,22,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",6.70,7.46,7.43,7.19,7.81,31,ok, -8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",201.60,199.34,199.65,194.41,203.66,81,ok, -9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",212.51,218.14,214.43,210.86,232.86,237,ok, -10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",276.50,264.53,264.49,263.64,265.52,102,ok, -11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",267.70,271.02,270.13,265.75,278.06,116,ok, -12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",233.29,249.14,231.61,228.06,305.29,387,ok, -13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",253.15,238.86,238.39,237.53,241.14,350,ok, -14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",225.86,225.66,225.66,225.40,225.93,400,ok, -15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",70.78,70.34,70.48,69.77,70.64,236,ok, -16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",119.70,119.67,119.72,119.02,120.24,244,ok, -17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",139.96,140.43,140.51,139.65,141.07,414,ok, -18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",402.60,336.78,336.95,334.78,338.43,264,ok, -19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,5.07,4.35,4.33,4.26,4.49,0,ok, -20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,186.80,141.09,141.07,140.79,141.45,3,ok, -21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",271.28,271.51,271.62,270.68,272.11,162,ok, -22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",408.96,366.24,365.63,362.18,371.51,6188,ok, -23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,279.14,242.44,241.79,240.90,245.29,2633,ok, -24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,270.04,269.01,268.95,268.55,269.59,658,ok, -25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,220.27,220.38,220.43,219.87,220.80,382,ok, -26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",270.29,270.39,269.25,268.48,274.56,658,ok, -27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",420.06,416.45,416.50,414.43,418.36,26,ok, -28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",972.67,947.10,947.30,946.08,947.70,556,ok, -29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",386.30,383.47,382.06,381.66,388.12,990,ok, -30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",232.60,230.63,230.35,228.93,232.88,244,ok, -31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",269.77,260.09,253.73,250.18,282.74,398,ok, -32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",1062.67,704.38,704.87,697.97,709.81,376,ok, -33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",368.34,365.48,365.28,364.19,367.15,464,ok, -34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",384.09,385.42,384.90,384.34,387.53,484,ok, -35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",144.54,143.79,143.84,142.83,144.66,493,ok, -36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",329.99,320.62,321.13,318.63,321.57,569,ok, -37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",236.91,235.07,235.24,233.68,236.12,550,ok, -38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",79.61,73.12,72.92,70.87,75.78,480,ok, -39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",436.78,424.89,422.82,420.83,433.09,835,ok, -40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",106.64,90.29,90.24,90.09,90.58,356,ok, -41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",37.17,32.69,32.69,32.52,32.87,132,ok, -42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",67.63,67.83,67.86,66.65,68.96,240,ok, +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,9.49,4.96,4.87,4.59,5.51,7,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,15.88,7.12,7.24,6.19,7.81,6,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",28.03,19.08,18.79,17.92,20.83,18,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,27.65,14.48,14.52,14.06,14.81,20,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,156.02,190.53,167.49,153.12,274.03,6,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,142.67,132.55,132.28,132.05,133.58,6,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",4.78,4.27,4.26,4.12,4.43,22,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",7.78,7.08,7.09,6.68,7.45,31,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",204.57,205.39,204.97,201.88,209.73,81,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",217.28,218.49,217.93,215.12,222.97,237,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",281.97,269.05,268.97,268.52,269.74,102,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",273.22,270.25,269.99,269.17,271.86,116,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",227.97,230.01,229.34,227.64,233.73,387,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",249.96,246.34,246.09,244.41,248.77,350,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",232.70,233.59,232.91,231.02,237.51,400,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",76.95,75.84,75.99,74.37,77.04,236,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",130.83,129.57,129.28,129.09,130.63,244,ok, +17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",153.61,153.56,152.80,152.21,156.42,414,ok, +18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",445.38,374.33,374.64,372.72,375.32,264,ok, +19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,5.02,5.16,5.12,4.54,5.86,0,ok, +20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,194.65,145.74,145.69,145.27,146.31,3,ok, +21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",278.26,277.79,277.66,277.15,278.69,162,ok, +22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",426.01,375.71,374.66,374.01,379.49,6188,ok, +23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,285.01,244.43,244.56,242.75,245.86,2633,ok, +24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,268.59,268.79,268.40,267.75,270.62,658,ok, +25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,222.70,219.63,219.57,219.47,219.92,382,ok, +26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",268.76,270.05,270.06,269.16,270.92,658,ok, +27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",425.51,420.92,420.86,416.18,425.80,26,ok, +28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",979.03,1016.62,979.97,974.72,1131.81,556,ok, +29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",395.85,393.43,391.93,390.74,399.12,990,ok, +30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",250.48,238.09,237.88,236.56,240.02,244,ok, +31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",280.27,266.13,266.14,265.27,266.99,398,ok, +32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",1030.95,777.39,776.24,769.61,787.48,376,ok, +33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",403.13,404.75,405.27,402.76,405.69,464,ok, +34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",453.77,422.89,422.68,419.71,426.50,484,ok, +35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",156.60,155.02,155.09,153.26,156.64,493,ok, +36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",347.88,342.61,342.61,342.31,342.92,569,ok, +37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",244.48,243.77,243.88,242.73,244.58,550,ok, +38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",80.37,74.58,74.44,74.34,75.09,480,ok, +39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",453.18,450.37,449.66,447.42,454.74,835,ok, +40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",111.18,93.89,93.72,93.59,94.54,356,ok, +41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",37.92,34.08,34.09,33.77,34.37,132,ok, +42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",68.61,69.07,69.12,68.65,69.40,240,ok, diff --git a/scripts/dashboard.py b/scripts/dashboard.py index 8b8ce3f..01c4254 100644 --- a/scripts/dashboard.py +++ b/scripts/dashboard.py @@ -134,6 +134,30 @@ def markdown_escape(text: str) -> str: return text.replace("|", "\\|") +def html_escape(text: str) -> str: + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + ) + + +def render_html_metric_table(title: str, rows: list[tuple[str, str]]) -> list[str]: + lines = [ + f"

{html_escape(title)}

", + '', + '', + "", + ] + for metric, value in rows: + lines.append( + f"" + ) + lines.extend(["", "
MetricValue
{html_escape(str(metric))}{html_escape(str(value))}
"]) + return lines + + def parse_ms(raw: str) -> float | None: if not raw: return None @@ -435,99 +459,75 @@ def render_markdown( cold_penalty = ((statistics.fmean(first_run_values) / avg_value) - 1.0) * 100.0 if avg_value and first_run_values else 0.0 fastest = min(ok_rows, key=query_metric) if ok_rows else None slowest = max(ok_rows, key=query_metric) if ok_rows else None - top_slowest = sorted(ok_rows, key=query_metric, reverse=True)[:5] - top_fastest = sorted(ok_rows, key=query_metric)[:5] - largest_outputs = sorted(ok_rows, key=lambda row: int(row["output_csv_bytes"]), reverse=True)[:5] + summary_rows = [ + ("queries ok", f"{len(ok_rows)} / {len(rows)}"), + ("queries failed", str(len(failed_rows))), + ("median warm time", f"{format_ms(median_value)} ms"), + ("average warm time", f"{format_ms(avg_value)} ms"), + ("p95 warm time", f"{format_ms(p95_value)} ms"), + ("cold/warm delta", format_percent(cold_penalty)), + ("total output size", format_size(sum(output_sizes))), + ("max output size", format_size(max(output_sizes)) if output_sizes else "—"), + ( + "fastest query", + f"Q{int(fastest['query_id']):02d} · {format_ms(query_metric(fastest))} ms" if fastest else "—", + ), + ( + "slowest query", + f"Q{int(slowest['query_id']):02d} · {format_ms(query_metric(slowest))} ms" if slowest else "—", + ), + ] lines = [ "## Benchmark Dashboard", "", f"`generated {generated_at}` · `queries {len(rows)}/{count}` · `warm runs {warm_runs}` · [`csv`]({csv_path})", "", - "### Summary", - "", - "| Metric | Value |", - "| --- | ---: |", - f"| queries ok | {len(ok_rows)} / {len(rows)} |", - f"| queries failed | {len(failed_rows)} |", - f"| median warm time | {format_ms(median_value)} ms |", - f"| average warm time | {format_ms(avg_value)} ms |", - f"| p95 warm time | {format_ms(p95_value)} ms |", - f"| cold/warm delta | {format_percent(cold_penalty)} |", - f"| total output size | {format_size(sum(output_sizes))} |", - f"| max output size | {format_size(max(output_sizes)) if output_sizes else '—'} |", - f"| fastest query | Q{int(fastest['query_id']):02d} · {format_ms(query_metric(fastest))} ms |" if fastest else "| fastest query | — |", - f"| slowest query | Q{int(slowest['query_id']):02d} · {format_ms(query_metric(slowest))} ms |" if slowest else "| slowest query | — |", - "", ] if conversion_stats is not None: + storage_rows = [ + ("source csv", conversion_stats["source_csv_path"]), + ("schema", conversion_stats["schema_path"]), + ("compression", conversion_stats["compression"]), + ("source size", format_size(int(conversion_stats["source_csv_bytes"]))), + ("columnar size", format_size(int(conversion_stats["columnar_bytes"]))), + ("roundtrip csv size", format_size(int(conversion_stats["roundtrip_csv_bytes"]))), + ("compression ratio", format_ratio(float(conversion_stats["compression_ratio"]))), + ("columnar / csv", format_percent(float(conversion_stats["columnar_vs_csv_percent"]))), + ("csv -> columnar", format_seconds(float(conversion_stats["convert_elapsed_seconds"]))), + ("columnar -> csv", format_seconds(float(conversion_stats["roundtrip_elapsed_seconds"]))), + ("convert throughput", f"{float(conversion_stats['convert_throughput_mb_s']):.1f} MB/s"), + ("roundtrip throughput", f"{float(conversion_stats['roundtrip_throughput_mb_s']):.1f} MB/s"), + ] lines.extend( [ - "### Storage", + '', + '', + '", + '", + "", + "
', + *render_html_metric_table("Summary", summary_rows), + "', + *render_html_metric_table("Storage", storage_rows), + "
", + "", + ] + ) + else: + lines.extend( + [ + "### Summary", "", "| Metric | Value |", "| --- | ---: |", - f"| source csv | `{conversion_stats['source_csv_path']}` |", - f"| schema | `{conversion_stats['schema_path']}` |", - f"| compression | `{conversion_stats['compression']}` |", - f"| source size | {format_size(int(conversion_stats['source_csv_bytes']))} |", - f"| columnar size | {format_size(int(conversion_stats['columnar_bytes']))} |", - f"| roundtrip csv size | {format_size(int(conversion_stats['roundtrip_csv_bytes']))} |", - f"| compression ratio | {format_ratio(float(conversion_stats['compression_ratio']))} |", - f"| columnar / csv | {format_percent(float(conversion_stats['columnar_vs_csv_percent']))} |", - f"| csv -> columnar | {format_seconds(float(conversion_stats['convert_elapsed_seconds']))} |", - f"| columnar -> csv | {format_seconds(float(conversion_stats['roundtrip_elapsed_seconds']))} |", - f"| convert throughput | {float(conversion_stats['convert_throughput_mb_s']):.1f} MB/s |", - f"| roundtrip throughput | {float(conversion_stats['roundtrip_throughput_mb_s']):.1f} MB/s |", + *[f"| {markdown_escape(metric)} | {markdown_escape(str(value))} |" for metric, value in summary_rows], "", ] ) lines.extend(render_heatmap(rows, median_value)) - lines.extend( - [ - "### Largest Result Sets", - "", - "| Query | Output | Warm median, ms | SQL |", - "| --- | ---: | ---: | --- |", - ] - ) - for row in largest_outputs: - lines.append( - f"| [Q{int(row['query_id']):02d}]({row['query_file']}) | {format_size(int(row['output_csv_bytes']))} | " - f"{format_ms(query_metric(row))} | {markdown_escape(compact_sql(row['sql'], 64))} |" - ) - - lines.extend( - [ - "", - "### Fastest Queries", - "", - "| Query | SQL | Warm median, ms | First run, ms |", - "| --- | --- | ---: | ---: |", - ] - ) - for row in top_fastest: - lines.append( - f"| [Q{int(row['query_id']):02d}]({row['query_file']}) | {markdown_escape(compact_sql(row['sql'], 72))} | " - f"{format_ms(query_metric(row))} | {row['first_run_ms'] or '—'} |" - ) - - lines.extend( - [ - "", - "### Slowest Queries", - "", - "| Query | SQL | Warm median, ms | Warm max, ms |", - "| --- | --- | ---: | ---: |", - ] - ) - for row in top_slowest: - lines.append( - f"| [Q{int(row['query_id']):02d}]({row['query_file']}) | {markdown_escape(compact_sql(row['sql'], 72))} | " - f"{format_ms(query_metric(row))} | {row['warm_max_ms'] or '—'} |" - ) lines.extend( [ From 9480b0ae9dab6c32fe0ac0f94f6bbaf63a9ea198 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 21:18:26 +0300 Subject: [PATCH 09/13] fix: refactoring --- benchmarks/support/query_result_checks.cpp | 4 +- include/common/string_pattern_utils.h | 5 + .../comparison_utils.h} | 2 +- include/executor/operators_internal.h | 3 +- .../query_utils.h} | 6 + include/executor/typed_value_utils.h | 14 ++ src/CMakeLists.txt | 6 +- ...tor_utils.cpp => string_pattern_utils.cpp} | 71 +-------- src/executor/aggregate_states.cpp | 74 +-------- src/executor/comparison_utils.cpp | 70 +++++++++ src/executor/operators_aggregate.cpp | 57 ++----- src/executor/operators_runtime.cpp | 74 ++------- src/executor/operators_scan.cpp | 12 +- src/executor/query_planner.cpp | 142 ++---------------- .../query_utils.cpp} | 59 +++++++- src/executor/typed_value_utils.cpp | 107 +++++++++++++ src/model/column_string.cpp | 2 +- 17 files changed, 316 insertions(+), 392 deletions(-) create mode 100644 include/common/string_pattern_utils.h rename include/{common/operator_utils.h => executor/comparison_utils.h} (83%) rename include/{common/planner_utils.h => executor/query_utils.h} (72%) create mode 100644 include/executor/typed_value_utils.h rename src/common/{operator_utils.cpp => string_pattern_utils.cpp} (58%) create mode 100644 src/executor/comparison_utils.cpp rename src/{common/planner_utils.cpp => executor/query_utils.cpp} (67%) create mode 100644 src/executor/typed_value_utils.cpp diff --git a/benchmarks/support/query_result_checks.cpp b/benchmarks/support/query_result_checks.cpp index bbdcc03..78549a1 100644 --- a/benchmarks/support/query_result_checks.cpp +++ b/benchmarks/support/query_result_checks.cpp @@ -9,8 +9,8 @@ #include "common/ascii.h" #include "common/error.h" -#include "common/operator_utils.h" -#include "common/planner_utils.h" +#include "executor/comparison_utils.h" +#include "executor/query_utils.h" #include "io/csv.h" #include "io/csv_batch.h" diff --git a/include/common/string_pattern_utils.h b/include/common/string_pattern_utils.h new file mode 100644 index 0000000..7a70759 --- /dev/null +++ b/include/common/string_pattern_utils.h @@ -0,0 +1,5 @@ +#pragma once + +#include + +bool LikeMatches(std::string_view value, std::string_view pattern); diff --git a/include/common/operator_utils.h b/include/executor/comparison_utils.h similarity index 83% rename from include/common/operator_utils.h rename to include/executor/comparison_utils.h index 3d1a93d..43f13d4 100644 --- a/include/common/operator_utils.h +++ b/include/executor/comparison_utils.h @@ -7,5 +7,5 @@ std::strong_ordering CompareValues(ColumnType type, std::string_view lhs, std::string_view rhs); bool MatchesComparison(ColumnType type, std::string_view lhs, std::string_view rhs, ComparisonKind kind); -bool LikeMatches(std::string_view value, std::string_view pattern); + ColumnType AggregateOutputType(const PlannedAgg& aggregate); diff --git a/include/executor/operators_internal.h b/include/executor/operators_internal.h index 47dee2f..c0fd102 100644 --- a/include/executor/operators_internal.h +++ b/include/executor/operators_internal.h @@ -8,6 +8,7 @@ #include #include "executor/operator.h" +#include "executor/query_utils.h" std::unique_ptr CreateMetadataCountOperator(std::filesystem::path path, std::string output_name); std::unique_ptr CreateMetadataExtremaOperator(std::filesystem::path path, std::vector aggregates); @@ -33,8 +34,6 @@ std::unique_ptr CreateGroupAggTopKOperator(std::unique_ptr c void ApplyOrderOffsetLimit(std::unique_ptr& root, const PlannedQuery& planned, bool& limit_applied_by_top_k); Schema ProjectionOutputSchema(const std::vector& items, const Schema& source_schema); -std::string NormalizeLiteralForEval(const QueryLiteral& literal); -std::optional TryFindBatchColumn(const Schema& schema, std::string_view name); std::string EvalExpr(const ExprPtr& expr, const Batch& batch, size_t row); bool EvaluatePredicate(const PredicatePtr& predicate, const Batch& batch, size_t row); void SelectRowsMatchingPredicate(const PredicatePtr& predicate, const Batch& batch, std::vector& rows); diff --git a/include/common/planner_utils.h b/include/executor/query_utils.h similarity index 72% rename from include/common/planner_utils.h rename to include/executor/query_utils.h index a4a59e9..e8ff824 100644 --- a/include/common/planner_utils.h +++ b/include/executor/query_utils.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -11,9 +12,14 @@ size_t FindColumnIndex(const Schema& schema, std::string_view column_name); void ValidateColumnQualifier(const Query& query, const ColumnRef& column); std::string UnescapeSqlString(std::string_view text); std::string NormalizeLiteral(const QueryLiteral& literal, ColumnType type); +std::string NormalizeLiteralForEval(const QueryLiteral& literal); +void NormalizeRegexReplacement(std::string& replacement); bool SameColumnName(std::string_view lhs, std::string_view rhs); bool SameColumnRef(const ColumnRef& lhs, const ColumnRef& rhs); bool SameSelectItem(const SelectItemSpec& lhs, const SelectItemSpec& rhs); bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs); +bool SameExpr(const ExprPtr& lhs, const ExprPtr& rhs); + std::vector StarColumnIndexes(const Schema& schema); +std::optional TryFindBatchColumn(const Schema& schema, std::string_view name); diff --git a/include/executor/typed_value_utils.h b/include/executor/typed_value_utils.h new file mode 100644 index 0000000..95aa863 --- /dev/null +++ b/include/executor/typed_value_utils.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include +#include + +#include "executor/query.h" +#include "model/schema.h" + +bool IsNumericColumnType(ColumnType type); +Int128 ParseColumnValueAsInt128(ColumnType type, std::string_view value); +std::optional TryParseLiteralValueAsInt128(const QueryLiteral& literal, ColumnType type); +std::string FormatInt128Value(ColumnType type, Int128 value); +ComparisonKind FlipComparison(ComparisonKind comparison); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9ff24dd..79c6e65 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,8 +21,7 @@ add_library(columnar_engine_core common/error.cpp common/ascii.cpp common/parsing.cpp - common/operator_utils.cpp - common/planner_utils.cpp + common/string_pattern_utils.cpp common/string_arena.cpp ) @@ -54,6 +53,9 @@ target_link_libraries(columnar_engine_convert ) add_library(columnar_engine_executor + executor/comparison_utils.cpp + executor/query_utils.cpp + executor/typed_value_utils.cpp executor/executor.cpp executor/query_parser.cpp executor/query_planner.cpp diff --git a/src/common/operator_utils.cpp b/src/common/string_pattern_utils.cpp similarity index 58% rename from src/common/operator_utils.cpp rename to src/common/string_pattern_utils.cpp index 9639f50..48252cd 100644 --- a/src/common/operator_utils.cpp +++ b/src/common/string_pattern_utils.cpp @@ -1,49 +1,11 @@ -#include "common/operator_utils.h" +#include "common/string_pattern_utils.h" -#include -#include +#include #include -#include "common/ascii.h" -#include "common/error.h" -#include "model/column_traits.h" - constexpr char LikeWildcardAny = '%'; constexpr char LikeWildcardSingle = '_'; -static bool EvaluateOrdering(const std::strong_ordering ordering, const ComparisonKind kind) { - if (kind == ComparisonKind::Equal) { - return ordering == 0; - } - - if (kind == ComparisonKind::NotEqual) { - return ordering != 0; - } - - if (kind == ComparisonKind::Less) { - return ordering < 0; - } - - if (kind == ComparisonKind::LessOrEqual) { - return ordering <= 0; - } - - if (kind == ComparisonKind::Greater) { - return ordering > 0; - } - - if (kind == ComparisonKind::GreaterOrEqual) { - return ordering >= 0; - } - - throw Error::Unsupported("executor", "unsupported comparison kind"); -} - -template -static std::strong_ordering CompareValuesImpl(const T left, const T right) { - return left <=> right; -} - static bool LikeMatchesPercentOnly(const std::string_view value, const std::string_view pattern) { std::vector segments; @@ -118,17 +80,6 @@ static bool LikeMatchesWithUnderscore(const std::string_view value, const std::s return pattern_pos == pattern.size(); } -std::strong_ordering CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { - return VisitColumnType(type, [&]() { - return CompareValuesImpl(ColumnValueTraits::Parse(lhs), ColumnValueTraits::Parse(rhs)); - }); -} - -bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, - const ComparisonKind kind) { - return EvaluateOrdering(CompareValues(type, lhs, rhs), kind); -} - bool LikeMatches(const std::string_view value, const std::string_view pattern) { if (pattern.find(LikeWildcardSingle) == std::string_view::npos) { return LikeMatchesPercentOnly(value, pattern); @@ -136,21 +87,3 @@ bool LikeMatches(const std::string_view value, const std::string_view pattern) { return LikeMatchesWithUnderscore(value, pattern); } - -ColumnType AggregateOutputType(const PlannedAgg& aggregate) { - const std::string name = ToUpperAscii(aggregate.function->canonical_name); - - if (name == "COUNT") { - return ColumnType::Int64; - } - - if (name == "SUM" || name == "AVG") { - return ColumnType::Int128; - } - - if (name == "MIN" || name == "MAX") { - return aggregate.input_type; - } - - throw Error::Unsupported("executor", "unsupported aggregate result type"); -} diff --git a/src/executor/aggregate_states.cpp b/src/executor/aggregate_states.cpp index 9ea9c8c..6083cb3 100644 --- a/src/executor/aggregate_states.cpp +++ b/src/executor/aggregate_states.cpp @@ -11,6 +11,7 @@ #include "common/string_arena.h" #include "executor/aggregate_function.h" #include "executor/aggregate_state.h" +#include "executor/typed_value_utils.h" #include "model/column_traits.h" void AggState::ConsumeInt128(const Int128 value) { ConsumeValue(Int128ToString(value)); } @@ -31,29 +32,6 @@ static bool ShouldReplaceExtremum(const ColumnType type, const std::string_view }); } -static Int128 ParseToInt128(const ColumnType type, const std::string_view value) { - switch (type) { - case ColumnType::Boolean: - return ParseBoolean(value) ? 1 : 0; - case ColumnType::Int16: - return ParseInt16(value); - case ColumnType::Int32: - return ParseInt32(value); - case ColumnType::Int64: - return ParseInt64(value); - case ColumnType::Int128: - return ParseInt128(value); - case ColumnType::Character: - return ParseCharacter(value); - case ColumnType::String: - case ColumnType::Date: - case ColumnType::Timestamp: - break; - } - - throw Error::Unsupported("executor", "unsupported numeric conversion"); -} - static std::string FormatAverage(const Int128 sum, const uint64_t count) { if (count == 0) { return "0"; @@ -61,28 +39,6 @@ static std::string FormatAverage(const Int128 sum, const uint64_t count) { return Int128ToString(sum / static_cast(count)); } -static std::string FormatInt128AsType(const ColumnType type, const Int128 value) { - switch (type) { - case ColumnType::Boolean: - return BooleanToString(value != 0); - case ColumnType::Int16: - case ColumnType::Int32: - case ColumnType::Int64: - case ColumnType::Int128: - return Int128ToString(value); - case ColumnType::Date: - return DateToString(static_cast(value)); - case ColumnType::Timestamp: - return TimestampToString(static_cast(value)); - case ColumnType::Character: - return std::string(1, static_cast(value)); - case ColumnType::String: - break; - } - - throw Error::Unsupported("executor", "unsupported typed aggregate result"); -} - class CountAS final : public AggState { public: void ConsumeValue(const std::string_view) override { ConsumeRow(); } @@ -100,7 +56,7 @@ class SumAS final : public AggState { public: explicit SumAS(const ColumnType type) : type_(type) {} - void ConsumeValue(const std::string_view value) override { sum_ += ParseToInt128(type_, value); } + void ConsumeValue(const std::string_view value) override { sum_ += ParseColumnValueAsInt128(type_, value); } void ConsumeInt128(const Int128 value) override { sum_ += value; } void ConsumeRow() override { throw Error::InvalidState("executor", "SUM requires a value"); } @@ -116,7 +72,7 @@ class AvgAS final : public AggState { explicit AvgAS(const ColumnType type) : type_(type) {} void ConsumeValue(const std::string_view value) override { - sum_ += ParseToInt128(type_, value); + sum_ += ParseColumnValueAsInt128(type_, value); ++count_; } @@ -160,7 +116,7 @@ class ExtremumAS final : public AggState { std::string Finalize() const override { if (typed_extremum_.has_value()) { - return FormatInt128AsType(type_, *typed_extremum_); + return FormatInt128Value(type_, *typed_extremum_); } return extremum_.value_or(""); } @@ -194,24 +150,6 @@ class DistinctAS final : public AggState { static bool SupportsAnyType(const ColumnType) { return true; } -static bool SupportsNumericType(const ColumnType type) { - switch (type) { - case ColumnType::Boolean: - case ColumnType::Int16: - case ColumnType::Int32: - case ColumnType::Int64: - case ColumnType::Int128: - case ColumnType::Character: - return true; - case ColumnType::String: - case ColumnType::Date: - case ColumnType::Timestamp: - return false; - } - - return false; -} - static std::unique_ptr CreateCountState(const PlannedAgg&) { return std::make_unique(); } static std::unique_ptr CreateSumState(const PlannedAgg& aggregate) { @@ -256,13 +194,13 @@ AggRegistrar::AggRegistrar(const AggFuncDefinition& def) { AggRegistry::Instance [[maybe_unused]] static AggRegistrar register_sum({ .canonical_name = "SUM", - .supports_type = &SupportsNumericType, + .supports_type = &IsNumericColumnType, .factory = &CreateSumState, }); [[maybe_unused]] static AggRegistrar register_avg({ .canonical_name = "AVG", - .supports_type = &SupportsNumericType, + .supports_type = &IsNumericColumnType, .factory = &CreateAvgState, }); diff --git a/src/executor/comparison_utils.cpp b/src/executor/comparison_utils.cpp new file mode 100644 index 0000000..50565fa --- /dev/null +++ b/src/executor/comparison_utils.cpp @@ -0,0 +1,70 @@ +#include "executor/comparison_utils.h" + +#include +#include + +#include "common/ascii.h" +#include "common/error.h" +#include "model/column_traits.h" + +static bool EvaluateOrdering(const std::strong_ordering ordering, const ComparisonKind kind) { + if (kind == ComparisonKind::Equal) { + return ordering == 0; + } + + if (kind == ComparisonKind::NotEqual) { + return ordering != 0; + } + + if (kind == ComparisonKind::Less) { + return ordering < 0; + } + + if (kind == ComparisonKind::LessOrEqual) { + return ordering <= 0; + } + + if (kind == ComparisonKind::Greater) { + return ordering > 0; + } + + if (kind == ComparisonKind::GreaterOrEqual) { + return ordering >= 0; + } + + throw Error::Unsupported("executor", "unsupported comparison kind"); +} + +template +static std::strong_ordering CompareValuesImpl(const T left, const T right) { + return left <=> right; +} + +std::strong_ordering CompareValues(const ColumnType type, const std::string_view lhs, const std::string_view rhs) { + return VisitColumnType(type, [&]() { + return CompareValuesImpl(ColumnValueTraits::Parse(lhs), ColumnValueTraits::Parse(rhs)); + }); +} + +bool MatchesComparison(const ColumnType type, const std::string_view lhs, const std::string_view rhs, + const ComparisonKind kind) { + return EvaluateOrdering(CompareValues(type, lhs, rhs), kind); +} + +ColumnType AggregateOutputType(const PlannedAgg& aggregate) { + const std::string name = ToUpperAscii(aggregate.function->canonical_name); + + if (name == "COUNT") { + return ColumnType::Int64; + } + + if (name == "SUM" || name == "AVG") { + return ColumnType::Int128; + } + + if (name == "MIN" || name == "MAX") { + return aggregate.input_type; + } + + throw Error::Unsupported("executor", "unsupported aggregate result type"); +} diff --git a/src/executor/operators_aggregate.cpp b/src/executor/operators_aggregate.cpp index 2f00d76..1d3173b 100644 --- a/src/executor/operators_aggregate.cpp +++ b/src/executor/operators_aggregate.cpp @@ -7,12 +7,13 @@ #include "common/ascii.h" #include "common/error.h" -#include "common/operator_utils.h" #include "common/parsing.h" #include "common/string_arena.h" #include "executor/aggregate_function.h" #include "executor/aggregate_state.h" +#include "executor/comparison_utils.h" #include "executor/operators_internal.h" +#include "executor/typed_value_utils.h" constexpr std::string_view ExtractMinutePart = "MINUTE"; constexpr std::string_view ExtractHourPart = "HOUR"; @@ -147,46 +148,17 @@ std::optional TryEvalTypedGroupKeyInt(const ExprPtr& expr, const Batch& return std::nullopt; } -Int128 ParseGroupKeyIntValue(const ColumnType type, const std::string_view value) { - switch (type) { - case ColumnType::Boolean: - return ParseBoolean(value) ? 1 : 0; - case ColumnType::Int16: - return ParseInt16(value); - case ColumnType::Int32: - return ParseInt32(value); - case ColumnType::Int64: - return ParseInt64(value); - case ColumnType::Int128: - return ParseInt128(value); - case ColumnType::Date: - return ParseDate(value); - case ColumnType::Timestamp: - return ParseTimestamp(value); - case ColumnType::Character: - return ParseCharacter(value); - case ColumnType::String: - break; - } - - throw Error::Unsupported("executor", "unsupported group key type"); -} - std::string FormatGroupKeyValue(const GroupKeyComponent& value) { switch (value.type) { case ColumnType::Boolean: - return BooleanToString(value.int_value != 0); case ColumnType::Int16: case ColumnType::Int32: case ColumnType::Int64: case ColumnType::Int128: - return Int128ToString(value.int_value); case ColumnType::Date: - return DateToString(static_cast(value.int_value)); case ColumnType::Timestamp: - return TimestampToString(static_cast(value.int_value)); case ColumnType::Character: - return std::string(1, static_cast(value.int_value)); + return FormatInt128Value(value.type, value.int_value); case ColumnType::String: return std::string(value.string_value); } @@ -232,8 +204,8 @@ struct TypedGroupKeyHash { for (const GroupKeyComponent& value : key.values) { hash = HashCombine(hash, std::hash{}(static_cast(value.type))); - hash = HashCombine(hash, value.type == ColumnType::String ? StringViewHash{}(value.string_value) - : HashInt128(value.int_value)); + hash = HashCombine(hash, value.type == ColumnType::String ? StringViewHash{}(value.string_value) + : HashInt128(value.int_value)); } return hash; @@ -296,7 +268,7 @@ class GroupKeyMaterializer { } else { key.values.push_back(GroupKeyComponent{ .type = group_key.column_type, - .int_value = ParseGroupKeyIntValue(group_key.column_type, value), + .int_value = ParseColumnValueAsInt128(group_key.column_type, value), .string_value = {}, }); } @@ -494,7 +466,7 @@ class GroupAggOperator final : public Operator { return order; } - std::sort(order.begin(), order.end(), [&](const size_t lhs_index, const size_t rhs_index) { + std::ranges::sort(order, [&](const size_t lhs_index, const size_t rhs_index) { const GroupState& lhs = groups_[lhs_index]; const GroupState& rhs = groups_[rhs_index]; for (const size_t column : group_key_columns) { @@ -673,22 +645,21 @@ class GroupAggTopKOperator final : public Operator { for (size_t i = 0; i < groups_.size(); ++i) { if (top_groups.size() < limit_) { top_groups.push_back(i); - std::push_heap(top_groups.begin(), top_groups.end(), - [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + std::ranges::push_heap(top_groups, + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); continue; } if (OrdersBefore(i, top_groups.front())) { - std::pop_heap(top_groups.begin(), top_groups.end(), - [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + std::ranges::pop_heap(top_groups, + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); top_groups.back() = i; - std::push_heap(top_groups.begin(), top_groups.end(), - [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + std::ranges::push_heap(top_groups, + [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); } } - std::sort(top_groups.begin(), top_groups.end(), - [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); + std::ranges::sort(top_groups, [&](const size_t lhs, const size_t rhs) { return OrdersBefore(lhs, rhs); }); return top_groups; } diff --git a/src/executor/operators_runtime.cpp b/src/executor/operators_runtime.cpp index 2905197..841e794 100644 --- a/src/executor/operators_runtime.cpp +++ b/src/executor/operators_runtime.cpp @@ -8,10 +8,12 @@ #include "common/ascii.h" #include "common/error.h" -#include "common/operator_utils.h" #include "common/parsing.h" -#include "common/planner_utils.h" +#include "common/string_pattern_utils.h" +#include "executor/comparison_utils.h" #include "executor/operators_internal.h" +#include "executor/query_utils.h" +#include "executor/typed_value_utils.h" constexpr std::string_view CountStarName = "COUNT(*)"; constexpr std::string_view FallbackAggregateAlias = "c"; @@ -161,25 +163,6 @@ std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t throw Error::Unsupported("executor", "unsupported function '" + expr.function_name + "'"); } -ComparisonKind FlipComparison(const ComparisonKind comparison) { - switch (comparison) { - case ComparisonKind::Equal: - return ComparisonKind::Equal; - case ComparisonKind::NotEqual: - return ComparisonKind::NotEqual; - case ComparisonKind::Less: - return ComparisonKind::Greater; - case ComparisonKind::LessOrEqual: - return ComparisonKind::GreaterOrEqual; - case ComparisonKind::Greater: - return ComparisonKind::Less; - case ComparisonKind::GreaterOrEqual: - return ComparisonKind::LessOrEqual; - } - - return comparison; -} - bool MatchesInt128Comparison(const Int128 lhs, const Int128 rhs, const ComparisonKind comparison) { switch (comparison) { case ComparisonKind::Equal: @@ -218,28 +201,6 @@ ValueComparison ToValueComparison(const ComparisonKind comparison) { return ValueComparison::Equal; } -std::optional LiteralValueForColumnType(const QueryLiteral& literal, const ColumnType type) { - try { - if (literal.kind == LiteralKind::Numeric) { - return ParseInt128(literal.text); - } - - if (literal.kind == LiteralKind::String) { - const QueryLiteral normalized_literal{NormalizeLiteralForEval(literal), LiteralKind::String}; - if (type == ColumnType::Date) { - return ParseDate(normalized_literal.text); - } - if (type == ColumnType::Timestamp) { - return ParseTimestamp(normalized_literal.text); - } - } - } catch (const Error&) { - return std::nullopt; - } - - return std::nullopt; -} - std::optional TryEvaluateTypedComparison(const ExprPtr& left, const ExprPtr& right, const Batch& batch, const size_t row, const ComparisonKind comparison) { if (!left || !right) { @@ -263,7 +224,7 @@ std::optional TryEvaluateTypedComparison(const ExprPtr& left, const ExprPt return std::nullopt; } - const std::optional rhs = LiteralValueForColumnType(right->literal, column.Type()); + const std::optional rhs = TryParseLiteralValueAsInt128(right->literal, column.Type()); if (!rhs.has_value()) { return std::nullopt; } @@ -520,14 +481,14 @@ class TopKOperator final : public Operator { if (rows.size() < limit_) { rows.push_back(std::move(candidate)); - std::push_heap(rows.begin(), rows.end(), ordering_); + std::ranges::push_heap(rows, ordering_); continue; } if (ordering_(candidate, rows.front())) { - std::pop_heap(rows.begin(), rows.end(), ordering_); + std::ranges::pop_heap(rows, ordering_); rows.back() = std::move(candidate); - std::push_heap(rows.begin(), rows.end(), ordering_); + std::ranges::push_heap(rows, ordering_); } } } @@ -536,7 +497,7 @@ class TopKOperator final : public Operator { return std::nullopt; } - std::sort(rows.begin(), rows.end(), ordering_); + std::ranges::sort(rows, ordering_); return BuildBatchFromSortedRows(*schema, rows); } @@ -607,23 +568,6 @@ class OffsetOperator final : public Operator { size_t remaining_; }; -std::string NormalizeLiteralForEval(const QueryLiteral& literal) { - if (literal.kind != LiteralKind::String) { - return literal.text; - } - - return UnescapeSqlString(literal.text); -} - -std::optional TryFindBatchColumn(const Schema& schema, const std::string_view name) { - for (size_t i = 0; i < schema.columns.size(); ++i) { - if (SameColumnName(schema.columns[i].name, name)) { - return i; - } - } - return std::nullopt; -} - std::string EvalExpr(const ExprPtr& expr, const Batch& batch, const size_t row) { switch (expr->kind) { case ExprKind::Column: { diff --git a/src/executor/operators_scan.cpp b/src/executor/operators_scan.cpp index 5bc9008..39a026f 100644 --- a/src/executor/operators_scan.cpp +++ b/src/executor/operators_scan.cpp @@ -3,8 +3,8 @@ #include "common/ascii.h" #include "common/error.h" -#include "common/operator_utils.h" #include "executor/aggregate_state.h" +#include "executor/comparison_utils.h" #include "executor/operators_internal.h" #include "io/columnar_batch.h" #include "io/file.h" @@ -139,12 +139,10 @@ bool MayMatchRowGroup(const PredicatePtr& predicate, const RowGroupMetadata& row return true; } - return std::any_of(predicate->metadata_typed_in_values.begin(), predicate->metadata_typed_in_values.end(), - [&](const Int128 value) { - return MatchesRowGroupComparison( - row_group.columns[predicate->metadata_typed_in_column_index], value, - ComparisonKind::Equal); - }); + return std::ranges::any_of(predicate->metadata_typed_in_values, [&](const Int128 value) { + return MatchesRowGroupComparison(row_group.columns[predicate->metadata_typed_in_column_index], value, + ComparisonKind::Equal); + }); case PredicateKind::Like: case PredicateKind::NotLike: return true; diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index 282f2c7..8fd4819 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -4,46 +4,14 @@ #include "common/ascii.h" #include "common/error.h" -#include "common/operator_utils.h" #include "common/parsing.h" -#include "common/planner_utils.h" +#include "executor/comparison_utils.h" +#include "executor/query_utils.h" +#include "executor/typed_value_utils.h" #include "io/columnar_batch.h" constexpr size_t SqlOrdinalBase = 1; -static bool SameExpr(const ExprPtr& lhs, const ExprPtr& rhs) { - if (!lhs || !rhs || lhs->kind != rhs->kind) { - return false; - } - - switch (lhs->kind) { - case ExprKind::Column: - return SameColumnRef(lhs->column, rhs->column); - case ExprKind::Literal: - return lhs->literal.text == rhs->literal.text && lhs->literal.kind == rhs->literal.kind; - case ExprKind::Star: - return true; - case ExprKind::Binary: - return lhs->binary_op == rhs->binary_op && SameExpr(lhs->left, rhs->left) && - SameExpr(lhs->right, rhs->right); - case ExprKind::Function: - if (!SameColumnName(lhs->function_name, rhs->function_name) || - lhs->arguments.size() != rhs->arguments.size()) { - return false; - } - for (size_t i = 0; i < lhs->arguments.size(); ++i) { - if (!SameExpr(lhs->arguments[i], rhs->arguments[i])) { - return false; - } - } - return true; - case ExprKind::Case: - return lhs->output_name == rhs->output_name; - } - - return false; -} - static ColumnType ColumnTypeOf(const Query& query, const Schema& schema, const ExprPtr& expr); static ColumnType ColumnTypeOfColumn(const Query& query, const Schema& schema, const ColumnRef& column) { @@ -334,20 +302,6 @@ static std::optional TryNumericLiteralValue(const ExprPtr& expr) { } } -static std::string LiteralTextForEval(const QueryLiteral& literal) { - if (literal.kind == LiteralKind::String) { - return UnescapeSqlString(literal.text); - } - return literal.text; -} - -static void NormalizeRegexReplacement(std::string& replacement) { - for (size_t pos = 0; (pos = replacement.find("\\1", pos)) != std::string::npos;) { - replacement.replace(pos, 2, "$1"); - pos += 2; - } -} - static void BindRegexReplace(const ExprPtr& expr) { if (!expr || expr->kind != ExprKind::Function || !SameColumnName(expr->function_name, "REGEXP_REPLACE") || expr->arguments.size() != 3 || !expr->arguments[1] || !expr->arguments[2] || @@ -356,9 +310,9 @@ static void BindRegexReplace(const ExprPtr& expr) { } try { - std::string replacement = LiteralTextForEval(expr->arguments[2]->literal); + std::string replacement = NormalizeLiteralForEval(expr->arguments[2]->literal); NormalizeRegexReplacement(replacement); - expr->regex_pattern = std::regex(LiteralTextForEval(expr->arguments[1]->literal)); + expr->regex_pattern = std::regex(NormalizeLiteralForEval(expr->arguments[1]->literal)); expr->regex_replacement = std::move(replacement); expr->regex_replace_bound = true; } catch (const std::regex_error&) { @@ -366,44 +320,6 @@ static void BindRegexReplace(const ExprPtr& expr) { } } -static std::optional TryLiteralValueForColumnType(const QueryLiteral& literal, const ColumnType type) { - try { - if (literal.kind == LiteralKind::Numeric) { - return ParseInt128(literal.text); - } - - if (literal.kind != LiteralKind::String) { - return std::nullopt; - } - - const std::string normalized = UnescapeSqlString(literal.text); - switch (type) { - case ColumnType::Boolean: - return ParseBoolean(normalized) ? 1 : 0; - case ColumnType::Int16: - return ParseInt16(normalized); - case ColumnType::Int32: - return ParseInt32(normalized); - case ColumnType::Int64: - return ParseInt64(normalized); - case ColumnType::Int128: - return ParseInt128(normalized); - case ColumnType::Date: - return ParseDate(normalized); - case ColumnType::Timestamp: - return ParseTimestamp(normalized); - case ColumnType::Character: - return ParseCharacter(normalized); - case ColumnType::String: - return std::nullopt; - } - } catch (const Error&) { - return std::nullopt; - } - - return std::nullopt; -} - static void BindLiteralInSet(const Query& query_ast, const Schema& schema, const std::vector& projection_indexes, const PredicatePtr& predicate) { if (!predicate || predicate->kind != PredicateKind::In || !predicate->left || @@ -425,7 +341,7 @@ static void BindLiteralInSet(const Query& query_ast, const Schema& schema, if (!value || value->kind != ExprKind::Literal) { return; } - values.insert(LiteralTextForEval(value->literal)); + values.insert(NormalizeLiteralForEval(value->literal)); } predicate->literal_in_set_bound = true; @@ -440,7 +356,8 @@ static void BindLiteralInSet(const Query& query_ast, const Schema& schema, typed_values.reserve(predicate->values.size()); for (const ExprPtr& value : predicate->values) { - const std::optional typed_value = TryLiteralValueForColumnType(value->literal, schema.columns[source_index].type); + const std::optional typed_value = + TryParseLiteralValueAsInt128(value->literal, schema.columns[source_index].type); if (!typed_value.has_value()) { return; } @@ -473,29 +390,10 @@ static void BindLiteralLikePattern(const Query& query_ast, const Schema& schema, predicate->literal_like_pattern_bound = true; predicate->like_column_index = *projected_index; - predicate->like_pattern = LiteralTextForEval(predicate->right->literal); + predicate->like_pattern = NormalizeLiteralForEval(predicate->right->literal); predicate->like_negated = predicate->kind == PredicateKind::NotLike; } -static ComparisonKind FlipComparison(const ComparisonKind comparison) { - switch (comparison) { - case ComparisonKind::Equal: - return ComparisonKind::Equal; - case ComparisonKind::NotEqual: - return ComparisonKind::NotEqual; - case ComparisonKind::Less: - return ComparisonKind::Greater; - case ComparisonKind::LessOrEqual: - return ComparisonKind::GreaterOrEqual; - case ComparisonKind::Greater: - return ComparisonKind::Less; - case ComparisonKind::GreaterOrEqual: - return ComparisonKind::LessOrEqual; - } - - return comparison; -} - static void BindTypedLiteralComparison(const Query& query_ast, const Schema& schema, const std::vector& projection_indexes, const PredicatePtr& predicate) { if (!predicate || predicate->kind != PredicateKind::Comparison || !predicate->left || !predicate->right) { @@ -525,7 +423,7 @@ static void BindTypedLiteralComparison(const Query& query_ast, const Schema& sch } const ColumnType type = schema.columns[source_index].type; - const std::optional literal = TryLiteralValueForColumnType((*literal_expr)->literal, type); + const std::optional literal = TryParseLiteralValueAsInt128((*literal_expr)->literal, type); if (!literal.has_value()) { return; } @@ -564,26 +462,8 @@ static std::optional> TryLinearColumnExpr(const Exp return std::pair{expr->left->column, offset}; } -static bool IsNumericType(const ColumnType type) { - switch (type) { - case ColumnType::Boolean: - case ColumnType::Int16: - case ColumnType::Int32: - case ColumnType::Int64: - case ColumnType::Int128: - case ColumnType::Character: - return true; - case ColumnType::String: - case ColumnType::Date: - case ColumnType::Timestamp: - return false; - } - - return false; -} - static bool SupportsDirectTypedAggInput(const AggFuncDefinition& function, const ColumnType type) { - if (IsNumericType(type)) { + if (IsNumericColumnType(type)) { return true; } const std::string name = ToUpperAscii(function.canonical_name); diff --git a/src/common/planner_utils.cpp b/src/executor/query_utils.cpp similarity index 67% rename from src/common/planner_utils.cpp rename to src/executor/query_utils.cpp index 5a30b3e..710db9f 100644 --- a/src/common/planner_utils.cpp +++ b/src/executor/query_utils.cpp @@ -1,4 +1,4 @@ -#include "common/planner_utils.h" +#include "executor/query_utils.h" #include @@ -77,6 +77,21 @@ std::string NormalizeLiteral(const QueryLiteral& literal, const ColumnType type) } } +std::string NormalizeLiteralForEval(const QueryLiteral& literal) { + if (literal.kind != LiteralKind::String) { + return literal.text; + } + + return UnescapeSqlString(literal.text); +} + +void NormalizeRegexReplacement(std::string& replacement) { + for (size_t pos = 0; (pos = replacement.find("\\1", pos)) != std::string::npos;) { + replacement.replace(pos, 2, "$1"); + pos += 2; + } +} + bool SameColumnName(const std::string_view lhs, const std::string_view rhs) { return ToLowerAscii(lhs) == ToLowerAscii(rhs); } @@ -109,6 +124,39 @@ bool SameOutputName(const SelectItemSpec& lhs, const SelectItemSpec& rhs) { return SameColumnName(lhs.output_name, rhs.output_name); } +bool SameExpr(const ExprPtr& lhs, const ExprPtr& rhs) { + if (!lhs || !rhs || lhs->kind != rhs->kind) { + return false; + } + + switch (lhs->kind) { + case ExprKind::Column: + return SameColumnRef(lhs->column, rhs->column); + case ExprKind::Literal: + return lhs->literal.text == rhs->literal.text && lhs->literal.kind == rhs->literal.kind; + case ExprKind::Star: + return true; + case ExprKind::Binary: + return lhs->binary_op == rhs->binary_op && SameExpr(lhs->left, rhs->left) && + SameExpr(lhs->right, rhs->right); + case ExprKind::Function: + if (!SameColumnName(lhs->function_name, rhs->function_name) || + lhs->arguments.size() != rhs->arguments.size()) { + return false; + } + for (size_t i = 0; i < lhs->arguments.size(); ++i) { + if (!SameExpr(lhs->arguments[i], rhs->arguments[i])) { + return false; + } + } + return true; + case ExprKind::Case: + return lhs->output_name == rhs->output_name; + } + + return false; +} + std::vector StarColumnIndexes(const Schema& schema) { std::vector clickbench_subset; @@ -134,3 +182,12 @@ std::vector StarColumnIndexes(const Schema& schema) { return all; } + +std::optional TryFindBatchColumn(const Schema& schema, const std::string_view name) { + for (size_t i = 0; i < schema.columns.size(); ++i) { + if (SameColumnName(schema.columns[i].name, name)) { + return i; + } + } + return std::nullopt; +} diff --git a/src/executor/typed_value_utils.cpp b/src/executor/typed_value_utils.cpp new file mode 100644 index 0000000..6e6d243 --- /dev/null +++ b/src/executor/typed_value_utils.cpp @@ -0,0 +1,107 @@ +#include "executor/typed_value_utils.h" + +#include + +#include "common/error.h" +#include "common/parsing.h" +#include "executor/query_utils.h" + +bool IsNumericColumnType(const ColumnType type) { + switch (type) { + case ColumnType::Boolean: + case ColumnType::Int16: + case ColumnType::Int32: + case ColumnType::Int64: + case ColumnType::Int128: + case ColumnType::Character: + return true; + case ColumnType::String: + case ColumnType::Date: + case ColumnType::Timestamp: + return false; + } + + return false; +} + +Int128 ParseColumnValueAsInt128(const ColumnType type, const std::string_view value) { + switch (type) { + case ColumnType::Boolean: + return ParseBoolean(value) ? 1 : 0; + case ColumnType::Int16: + return ParseInt16(value); + case ColumnType::Int32: + return ParseInt32(value); + case ColumnType::Int64: + return ParseInt64(value); + case ColumnType::Int128: + return ParseInt128(value); + case ColumnType::Date: + return ParseDate(value); + case ColumnType::Timestamp: + return ParseTimestamp(value); + case ColumnType::Character: + return ParseCharacter(value); + case ColumnType::String: + break; + } + + throw Error::Unsupported("executor", "unsupported typed value conversion"); +} + +std::optional TryParseLiteralValueAsInt128(const QueryLiteral& literal, const ColumnType type) { + try { + if (literal.kind == LiteralKind::Numeric) { + return ParseInt128(literal.text); + } + + if (literal.kind != LiteralKind::String) { + return std::nullopt; + } + + return ParseColumnValueAsInt128(type, NormalizeLiteralForEval(literal)); + } catch (const Error&) { + return std::nullopt; + } +} + +std::string FormatInt128Value(const ColumnType type, const Int128 value) { + switch (type) { + case ColumnType::Boolean: + return BooleanToString(value != 0); + case ColumnType::Int16: + case ColumnType::Int32: + case ColumnType::Int64: + case ColumnType::Int128: + return Int128ToString(value); + case ColumnType::Date: + return DateToString(static_cast(value)); + case ColumnType::Timestamp: + return TimestampToString(static_cast(value)); + case ColumnType::Character: + return std::string(1, static_cast(value)); + case ColumnType::String: + break; + } + + throw Error::Unsupported("executor", "unsupported typed value formatting"); +} + +ComparisonKind FlipComparison(const ComparisonKind comparison) { + switch (comparison) { + case ComparisonKind::Equal: + return ComparisonKind::Equal; + case ComparisonKind::NotEqual: + return ComparisonKind::NotEqual; + case ComparisonKind::Less: + return ComparisonKind::Greater; + case ComparisonKind::LessOrEqual: + return ComparisonKind::GreaterOrEqual; + case ComparisonKind::Greater: + return ComparisonKind::Less; + case ComparisonKind::GreaterOrEqual: + return ComparisonKind::LessOrEqual; + } + + return comparison; +} diff --git a/src/model/column_string.cpp b/src/model/column_string.cpp index e2b0e55..82ce4b1 100644 --- a/src/model/column_string.cpp +++ b/src/model/column_string.cpp @@ -5,7 +5,7 @@ #include #include "common/error.h" -#include "common/operator_utils.h" +#include "common/string_pattern_utils.h" #include "io/stream.h" constexpr char EncodedValueSeparator = ':'; From 1c83f0490fead033d01199ad48cf14e443a90a8f Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 21:34:00 +0300 Subject: [PATCH 10/13] fix: batch optimizations --- src/executor/operators_aggregate.cpp | 117 +++++++++++++++++++-------- src/executor/operators_runtime.cpp | 85 +++++++++++-------- src/executor/query_planner.cpp | 17 +--- 3 files changed, 135 insertions(+), 84 deletions(-) diff --git a/src/executor/operators_aggregate.cpp b/src/executor/operators_aggregate.cpp index 1d3173b..02ed743 100644 --- a/src/executor/operators_aggregate.cpp +++ b/src/executor/operators_aggregate.cpp @@ -31,6 +31,8 @@ Schema BuildAggregateOutputSchema(const std::vector& bindings) { return schema; } +std::optional TryEvalTypedGroupKeyInt(const ExprPtr& expr, const Batch& batch, size_t row); + bool UsesRowAggInput(const PlannedAgg& aggregate) { return !aggregate.direct_numeric_argument && aggregate.argument_kind != AggArgumentKind::Column; } @@ -43,6 +45,13 @@ void ConsumeAggRow(const PlannedAgg& aggregate, const Batch& batch, const size_t } if (aggregate.argument_kind == AggArgumentKind::Column) { + if (aggregate.input_type != ColumnType::String) { + if (const auto typed_value = TryEvalTypedGroupKeyInt(aggregate.argument, batch, row); + typed_value.has_value()) { + state.ConsumeInt128(*typed_value); + return; + } + } state.ConsumeValue(EvalExpr(aggregate.argument, batch, row)); return; } @@ -304,11 +313,13 @@ class AggOperator final : public Operator { returned_ = true; while (auto batch = child_->Next()) { - std::vector selected_rows; - SelectRowsMatchingPredicate(filter_, *batch, selected_rows); - - for (auto& binding : bindings_) { - ConsumeAggRows(binding.aggregate, *batch, selected_rows, *binding.state); + for (size_t row = 0; row < batch->RowsCount(); ++row) { + if (!EvaluatePredicate(filter_, *batch, row)) { + continue; + } + for (auto& binding : bindings_) { + ConsumeAggRow(binding.aggregate, *batch, row, *binding.state); + } } } @@ -356,10 +367,11 @@ class GroupAggOperator final : public Operator { returned_ = true; while (auto batch = child_->Next()) { - std::vector selected_rows; - SelectRowsMatchingPredicate(filter_, *batch, selected_rows); + for (size_t row = 0; row < batch->RowsCount(); ++row) { + if (!EvaluatePredicate(filter_, *batch, row)) { + continue; + } - for (const size_t row : selected_rows) { TypedGroupKey key = group_key_materializer_.Materialize(*batch, row, string_arena_); size_t group_index = 0; @@ -371,6 +383,9 @@ class GroupAggOperator final : public Operator { groups_.push_back(GroupState{ .key_values = std::move(key.values), .states = CreateStates(), + .aggregate_values = {}, + .aggregate_int_values = {}, + .aggregate_values_finalized = false, }); } else { group_index = it->second; @@ -384,27 +399,13 @@ class GroupAggOperator final : public Operator { } const Schema schema = BuildSelectOutputSchema(select_items_); - std::vector output_order = BuildOutputOrder(); - Batch result(schema, output_order.size()); + const std::vector output_order = BuildOutputOrder(); + Batch result(schema, groups_.size()); for (const size_t group_index : output_order) { - AppendGroupToBatch(groups_[group_index], result); - } - - if (having_) { - std::vector selected_rows; - selected_rows.reserve(result.RowsCount()); - - for (size_t row = 0; row < result.RowsCount(); ++row) { - if (EvaluatePredicate(having_, result, row)) { - selected_rows.push_back(row); - } + if (GroupMatchesHaving(groups_[group_index], schema)) { + AppendGroupToBatch(groups_[group_index], result); } - - Batch filtered(schema, selected_rows.size()); - filtered.AppendRowsSelectedFromBatch(result, selected_rows); - - return filtered; } return result; @@ -414,6 +415,9 @@ class GroupAggOperator final : public Operator { struct GroupState { std::vector key_values; std::vector> states; + mutable std::vector aggregate_values; + mutable std::vector aggregate_int_values; + mutable bool aggregate_values_finalized = false; }; struct GroupAggBinding { @@ -431,17 +435,49 @@ class GroupAggOperator final : public Operator { return states; } + void FinalizeAggregateValues(const GroupState& group) const { + if (group.aggregate_values_finalized) { + return; + } + + group.aggregate_values.reserve(group.states.size()); + group.aggregate_int_values.reserve(group.states.size()); + + for (size_t i = 0; i < group.states.size(); ++i) { + std::string value = group.states[i]->Finalize(); + const ColumnType type = AggregateOutputType(bindings_[i].aggregate); + group.aggregate_int_values.push_back(type == ColumnType::String ? 0 + : ParseColumnValueAsInt128(type, value)); + group.aggregate_values.push_back(std::move(value)); + } + + group.aggregate_values_finalized = true; + } + void AppendGroupToBatch(const GroupState& group, const Batch& batch) const { + FinalizeAggregateValues(group); + for (size_t column = 0; column < select_items_.size(); ++column) { const PlannedSelectItem& item = select_items_[column]; if (item.kind == SelectItemKind::GroupKey) { batch.AppendValueFromString(column, FormatGroupKeyValue(group.key_values[item.index])); } else { - batch.AppendValueFromString(column, group.states[item.index]->Finalize()); + batch.AppendValueFromString(column, group.aggregate_values[item.index]); } } } + bool GroupMatchesHaving(const GroupState& group, const Schema& schema) const { + if (!having_) { + return true; + } + + const Batch row(schema, 1); + AppendGroupToBatch(group, row); + + return EvaluatePredicate(having_, row, 0); + } + std::vector BuildOutputOrder() const { std::vector order; order.reserve(groups_.size()); @@ -525,10 +561,11 @@ class GroupAggTopKOperator final : public Operator { returned_ = true; while (auto batch = child_->Next()) { - std::vector selected_rows; - SelectRowsMatchingPredicate(filter_, *batch, selected_rows); + for (size_t row = 0; row < batch->RowsCount(); ++row) { + if (!EvaluatePredicate(filter_, *batch, row)) { + continue; + } - for (const size_t row : selected_rows) { TypedGroupKey key = group_key_materializer_.Materialize(*batch, row, string_arena_); size_t group_index = 0; @@ -541,6 +578,7 @@ class GroupAggTopKOperator final : public Operator { .key_values = std::move(key.values), .states = CreateStates(), .aggregate_values = {}, + .aggregate_int_values = {}, .aggregate_values_finalized = false, }); } else { @@ -570,6 +608,7 @@ class GroupAggTopKOperator final : public Operator { std::vector key_values; std::vector> states; mutable std::vector aggregate_values; + mutable std::vector aggregate_int_values; mutable bool aggregate_values_finalized = false; }; @@ -588,15 +627,22 @@ class GroupAggTopKOperator final : public Operator { return states; } - static void FinalizeAggregateValues(const GroupState& group) { + void FinalizeAggregateValues(const GroupState& group) const { if (group.aggregate_values_finalized) { return; } group.aggregate_values.reserve(group.states.size()); - for (const auto& state : group.states) { - group.aggregate_values.push_back(state->Finalize()); + group.aggregate_int_values.reserve(group.states.size()); + + for (size_t i = 0; i < group.states.size(); ++i) { + std::string value = group.states[i]->Finalize(); + const ColumnType type = AggregateOutputType(bindings_[i].aggregate); + group.aggregate_int_values.push_back(type == ColumnType::String ? 0 + : ParseColumnValueAsInt128(type, value)); + group.aggregate_values.push_back(std::move(value)); } + group.aggregate_values_finalized = true; } @@ -613,8 +659,9 @@ class GroupAggTopKOperator final : public Operator { } else { FinalizeAggregateValues(lhs); FinalizeAggregateValues(rhs); - comparison = - CompareValues(item.output_type, lhs.aggregate_values[item.index], rhs.aggregate_values[item.index]); + comparison = item.output_type == ColumnType::String + ? lhs.aggregate_values[item.index] <=> rhs.aggregate_values[item.index] + : lhs.aggregate_int_values[item.index] <=> rhs.aggregate_int_values[item.index]; } if (comparison != 0) { diff --git a/src/executor/operators_runtime.cpp b/src/executor/operators_runtime.cpp index 841e794..c2ef071 100644 --- a/src/executor/operators_runtime.cpp +++ b/src/executor/operators_runtime.cpp @@ -248,19 +248,35 @@ ColumnType InferComparisonType(const std::string& lhs, const std::string& rhs) { return ColumnType::String; } -struct SortedRow { - std::vector values; +struct RowRef { + size_t batch_index = 0; + size_t row = 0; size_t ordinal = 0; }; +std::strong_ordering CompareColumnRows(const Column& lhs_column, const size_t lhs_row, const Column& rhs_column, + const size_t rhs_row, const ColumnType type) { + if (type != ColumnType::String) { + return lhs_column.ValueAsInt128(lhs_row) <=> rhs_column.ValueAsInt128(rhs_row); + } + + return lhs_column.ValueAsString(lhs_row) <=> rhs_column.ValueAsString(rhs_row); +} + class RowOrdering { public: - explicit RowOrdering(std::vector order_by) : order_by_(std::move(order_by)) {} + RowOrdering(std::vector order_by, const std::vector* batches) + : order_by_(std::move(order_by)), batches_(batches) {} + + bool operator()(const RowRef& lhs, const RowRef& rhs) const { + const Batch& lhs_batch = batches_->at(lhs.batch_index); + const Batch& rhs_batch = batches_->at(rhs.batch_index); - bool operator()(const SortedRow& lhs, const SortedRow& rhs) const { for (const auto& order : order_by_) { - const std::strong_ordering comparison = CompareValues( - order.value_type, lhs.values[order.result_column_index], rhs.values[order.result_column_index]); + const Column& lhs_column = lhs_batch.ColumnAt(order.result_column_index); + const Column& rhs_column = rhs_batch.ColumnAt(order.result_column_index); + const std::strong_ordering comparison = + CompareColumnRows(lhs_column, lhs.row, rhs_column, rhs.row, order.value_type); if (comparison != 0) { return order.descending ? comparison > 0 : comparison < 0; } @@ -271,25 +287,16 @@ class RowOrdering { private: std::vector order_by_; + const std::vector* batches_ = nullptr; }; -std::vector MaterializeBatchRow(const Batch& batch, const size_t row) { - std::vector values; - values.reserve(batch.ColumnsCount()); - - for (size_t column_index = 0; column_index < batch.ColumnsCount(); ++column_index) { - values.push_back(batch.ColumnAt(column_index).ValueAsString(row)); - } - - return values; -} - -Batch BuildBatchFromSortedRows(const Schema& schema, const std::vector& rows) { +Batch BuildBatchFromRowRefs(const Schema& schema, const std::vector& batches, const std::vector& rows) { Batch result(schema, rows.size()); for (const auto& row : rows) { - for (size_t i = 0; i < row.values.size(); ++i) { - result.AppendValueFromString(i, row.values[i]); + const Batch& source = batches[row.batch_index]; + for (size_t column = 0; column < source.ColumnsCount(); ++column) { + result.AppendValueFromColumn(column, source.ColumnAt(column), row.row); } } @@ -409,7 +416,7 @@ class ProjectionOperator final : public Operator { class OrderByOperator final : public Operator { public: OrderByOperator(std::unique_ptr child, std::vector order_by) - : child_(std::move(child)), ordering_(std::move(order_by)) {} + : child_(std::move(child)), ordering_(std::move(order_by), &batches_) {} std::optional Next() override { if (returned_) { @@ -419,7 +426,7 @@ class OrderByOperator final : public Operator { returned_ = true; std::optional schema; - std::vector rows; + std::vector rows; size_t ordinal = 0; while (auto batch = child_->Next()) { @@ -427,9 +434,14 @@ class OrderByOperator final : public Operator { schema = batch->GetSchema(); } - for (size_t row = 0; row < batch->RowsCount(); ++row) { - rows.push_back(SortedRow{ - .values = MaterializeBatchRow(*batch, row), + const size_t batch_index = batches_.size(); + const size_t rows_count = batch->RowsCount(); + batches_.push_back(std::move(*batch)); + + for (size_t row = 0; row < rows_count; ++row) { + rows.push_back(RowRef{ + .batch_index = batch_index, + .row = row, .ordinal = ordinal++, }); } @@ -439,13 +451,14 @@ class OrderByOperator final : public Operator { return std::nullopt; } - std::sort(rows.begin(), rows.end(), ordering_); + std::ranges::sort(rows, ordering_); - return BuildBatchFromSortedRows(*schema, rows); + return BuildBatchFromRowRefs(*schema, batches_, rows); } private: std::unique_ptr child_; + std::vector batches_; RowOrdering ordering_; bool returned_ = false; }; @@ -453,7 +466,7 @@ class OrderByOperator final : public Operator { class TopKOperator final : public Operator { public: TopKOperator(std::unique_ptr child, std::vector order_by, const size_t limit) - : child_(std::move(child)), ordering_(std::move(order_by)), limit_(limit) {} + : child_(std::move(child)), ordering_(std::move(order_by), &batches_), limit_(limit) {} std::optional Next() override { if (returned_ || limit_ == 0) { @@ -463,7 +476,7 @@ class TopKOperator final : public Operator { returned_ = true; std::optional schema; - std::vector rows; + std::vector rows; rows.reserve(limit_); size_t ordinal = 0; @@ -473,9 +486,14 @@ class TopKOperator final : public Operator { schema = batch->GetSchema(); } - for (size_t row = 0; row < batch->RowsCount(); ++row) { - SortedRow candidate{ - .values = MaterializeBatchRow(*batch, row), + const size_t batch_index = batches_.size(); + const size_t rows_count = batch->RowsCount(); + batches_.push_back(std::move(*batch)); + + for (size_t row = 0; row < rows_count; ++row) { + RowRef candidate{ + .batch_index = batch_index, + .row = row, .ordinal = ordinal++, }; @@ -499,11 +517,12 @@ class TopKOperator final : public Operator { std::ranges::sort(rows, ordering_); - return BuildBatchFromSortedRows(*schema, rows); + return BuildBatchFromRowRefs(*schema, batches_, rows); } private: std::unique_ptr child_; + std::vector batches_; RowOrdering ordering_; size_t limit_; bool returned_ = false; diff --git a/src/executor/query_planner.cpp b/src/executor/query_planner.cpp index 8fd4819..33d6261 100644 --- a/src/executor/query_planner.cpp +++ b/src/executor/query_planner.cpp @@ -631,23 +631,8 @@ PlannedQuery PlanQuery(const Query& query, const std::unordered_map effective_order_by = query.order_by; - for (const auto& item : query.select_items) { - if (item.expression && item.expression->kind != ExprKind::Star) { - const bool already_ordered = std::ranges::any_of(effective_order_by, [&](const OrderBySpec& order) { - return SameExpr(order.item.expression, item.expression); - }); - if (!already_ordered) { - effective_order_by.push_back(OrderBySpec{ - .item = item, - .descending = false, - }); - } - } - } - - for (const auto& order_item : effective_order_by) { + for (const auto& order_item : query.order_by) { size_t column_index = 0; const auto selected = FindSelectItem(query, order_item.item); const ExprPtr order_expression = From cbaf9e33101add7642d09575f2f0c7f7a1782495 Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 21:34:12 +0300 Subject: [PATCH 11/13] upd: update dashboard --- README.md | 124 +++++++++++------------ benchmarks/results/readme_benchmarks.csv | 86 ++++++++-------- 2 files changed, 105 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 2b700be..7220807 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## Benchmark Dashboard -`generated 2026-05-28 15:08:08` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) +`generated 2026-05-28 21:28:51` · `queries 43/43` · `warm runs 4` · [`csv`](benchmarks/results/readme_benchmarks.csv) @@ -16,14 +16,14 @@ - - - - + + + + - - + +
queries ok43 / 43
queries failed0
median warm time229.34 ms
average warm time230.90 ms
p95 warm time446.96 ms
cold/warm delta6.3%
median warm time220.81 ms
average warm time231.63 ms
p95 warm time476.62 ms
cold/warm delta5.3%
total output size20.3 KB
max output size6.0 KB
fastest queryQ06 · 4.26 ms
slowest queryQ28 · 979.97 ms
fastest queryQ06 · 4.12 ms
slowest queryQ28 · 969.54 ms
@@ -40,10 +40,10 @@ roundtrip csv size743.0 MB compression ratio6.71x columnar / csv14.9% -csv -> columnar5.67s -columnar -> csv9.59s -convert throughput141.7 MB/s -roundtrip throughput12.5 MB/s +csv -> columnar5.81s +columnar -> csv9.84s +convert throughput138.1 MB/s +roundtrip throughput12.2 MB/s @@ -56,60 +56,60 @@ | slot 1 | slot 2 | slot 3 | slot 4 | slot 5 | slot 6 | | ---: | ---: | ---: | ---: | ---: | ---: | -| Q00 🟩 `4.87` | Q01 🟩 `7.24` | Q02 🟩 `18.79` | Q03 🟩 `14.52` | Q04 🟩 `167.49` | Q05 🟩 `132.28` | -| Q06 🟩 `4.26` | Q07 🟩 `7.09` | Q08 🟩 `204.97` | Q09 🟦 `217.93` | Q10 🟥 `268.97` | Q11 🟥 `269.99` | -| Q12 🟦 `229.34` | Q13 🟦 `246.09` | Q14 🟦 `232.91` | Q15 🟩 `75.99` | Q16 🟩 `129.28` | Q17 🟩 `152.80` | -| Q18 🟥 `374.64` | Q19 🟩 `5.12` | Q20 🟩 `145.69` | Q21 🟥 `277.66` | Q22 🟥 `374.66` | Q23 🟦 `244.56` | -| Q24 🟥 `268.40` | Q25 🟦 `219.57` | Q26 🟥 `270.06` | Q27 🟥 `420.86` | Q28 🟥 `979.97` | Q29 🟥 `391.93` | -| Q30 🟦 `237.88` | Q31 🟥 `266.14` | Q32 🟥 `776.24` | Q33 🟥 `405.27` | Q34 🟥 `422.68` | Q35 🟩 `155.09` | -| Q36 🟥 `342.61` | Q37 🟦 `243.88` | Q38 🟩 `74.44` | Q39 🟥 `449.66` | Q40 🟩 `93.72` | Q41 🟩 `34.09` | -| Q42 🟩 `69.12` | | | | | | +| Q00 🟩 `7.14` | Q01 🟩 `12.85` | Q02 🟩 `21.68` | Q03 🟩 `14.30` | Q04 🟩 `147.02` | Q05 🟩 `128.76` | +| Q06 🟩 `4.12` | Q07 🟩 `11.69` | Q08 🟩 `189.51` | Q09 🟦 `208.53` | Q10 🟥 `267.84` | Q11 🟥 `267.06` | +| Q12 🟦 `225.37` | Q13 🟥 `253.71` | Q14 🟦 `227.55` | Q15 🟩 `74.16` | Q16 🟩 `116.97` | Q17 🟩 `146.00` | +| Q18 🟥 `349.51` | Q19 🟩 `4.39` | Q20 🟥 `252.69` | Q21 🟥 `277.31` | Q22 🟥 `364.71` | Q23 🟦 `240.62` | +| Q24 🟦 `220.81` | Q25 🟦 `219.69` | Q26 🟦 `220.03` | Q27 🟥 `450.51` | Q28 🟥 `969.54` | Q29 🟥 `427.54` | +| Q30 🟦 `235.10` | Q31 🟥 `262.43` | Q32 🟥 `881.20` | Q33 🟥 `349.73` | Q34 🟥 `420.41` | Q35 🟩 `149.57` | +| Q36 🟥 `339.52` | Q37 🟥 `247.46` | Q38 🟩 `73.93` | Q39 🟥 `479.52` | Q40 🟩 `95.10` | Q41 🟩 `34.88` | +| Q42 🟩 `69.69` | | | | | | ### Query Table | Query | Output CSV | First run, ms | Warm avg, ms | Warm median, ms | Warm min, ms | Warm max, ms | Status | | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | -| [Q00](benchmarks/queries/query_0.sql) | 7 B | 9.49 | 4.96 | 4.87 | 4.59 | 5.51 | ok | -| [Q01](benchmarks/queries/query_1.sql) | 6 B | 15.88 | 7.12 | 7.24 | 6.19 | 7.81 | ok | -| [Q02](benchmarks/queries/query_2.sql) | 18 B | 28.03 | 19.08 | 18.79 | 17.92 | 20.83 | ok | -| [Q03](benchmarks/queries/query_3.sql) | 20 B | 27.65 | 14.48 | 14.52 | 14.06 | 14.81 | ok | -| [Q04](benchmarks/queries/query_4.sql) | 6 B | 156.02 | 190.53 | 167.49 | 153.12 | 274.03 | ok | -| [Q05](benchmarks/queries/query_5.sql) | 6 B | 142.67 | 132.55 | 132.28 | 132.05 | 133.58 | ok | -| [Q06](benchmarks/queries/query_6.sql) | 22 B | 4.78 | 4.27 | 4.26 | 4.12 | 4.43 | ok | -| [Q07](benchmarks/queries/query_7.sql) | 31 B | 7.78 | 7.08 | 7.09 | 6.68 | 7.45 | ok | -| [Q08](benchmarks/queries/query_8.sql) | 81 B | 204.57 | 205.39 | 204.97 | 201.88 | 209.73 | ok | -| [Q09](benchmarks/queries/query_9.sql) | 237 B | 217.28 | 218.49 | 217.93 | 215.12 | 222.97 | ok | -| [Q10](benchmarks/queries/query_10.sql) | 102 B | 281.97 | 269.05 | 268.97 | 268.52 | 269.74 | ok | -| [Q11](benchmarks/queries/query_11.sql) | 116 B | 273.22 | 270.25 | 269.99 | 269.17 | 271.86 | ok | -| [Q12](benchmarks/queries/query_12.sql) | 387 B | 227.97 | 230.01 | 229.34 | 227.64 | 233.73 | ok | -| [Q13](benchmarks/queries/query_13.sql) | 350 B | 249.96 | 246.34 | 246.09 | 244.41 | 248.77 | ok | -| [Q14](benchmarks/queries/query_14.sql) | 400 B | 232.70 | 233.59 | 232.91 | 231.02 | 237.51 | ok | -| [Q15](benchmarks/queries/query_15.sql) | 236 B | 76.95 | 75.84 | 75.99 | 74.37 | 77.04 | ok | -| [Q16](benchmarks/queries/query_16.sql) | 244 B | 130.83 | 129.57 | 129.28 | 129.09 | 130.63 | ok | -| [Q17](benchmarks/queries/query_17.sql) | 414 B | 153.61 | 153.56 | 152.80 | 152.21 | 156.42 | ok | -| [Q18](benchmarks/queries/query_18.sql) | 264 B | 445.38 | 374.33 | 374.64 | 372.72 | 375.32 | ok | -| [Q19](benchmarks/queries/query_19.sql) | 0 B | 5.02 | 5.16 | 5.12 | 4.54 | 5.86 | ok | -| [Q20](benchmarks/queries/query_20.sql) | 3 B | 194.65 | 145.74 | 145.69 | 145.27 | 146.31 | ok | -| [Q21](benchmarks/queries/query_21.sql) | 162 B | 278.26 | 277.79 | 277.66 | 277.15 | 278.69 | ok | -| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 426.01 | 375.71 | 374.66 | 374.01 | 379.49 | ok | -| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 285.01 | 244.43 | 244.56 | 242.75 | 245.86 | ok | -| [Q24](benchmarks/queries/query_24.sql) | 658 B | 268.59 | 268.79 | 268.40 | 267.75 | 270.62 | ok | -| [Q25](benchmarks/queries/query_25.sql) | 382 B | 222.70 | 219.63 | 219.57 | 219.47 | 219.92 | ok | -| [Q26](benchmarks/queries/query_26.sql) | 658 B | 268.76 | 270.05 | 270.06 | 269.16 | 270.92 | ok | -| [Q27](benchmarks/queries/query_27.sql) | 26 B | 425.51 | 420.92 | 420.86 | 416.18 | 425.80 | ok | -| [Q28](benchmarks/queries/query_28.sql) | 556 B | 979.03 | 1016.62 | 979.97 | 974.72 | 1131.81 | ok | -| [Q29](benchmarks/queries/query_29.sql) | 990 B | 395.85 | 393.43 | 391.93 | 390.74 | 399.12 | ok | -| [Q30](benchmarks/queries/query_30.sql) | 244 B | 250.48 | 238.09 | 237.88 | 236.56 | 240.02 | ok | -| [Q31](benchmarks/queries/query_31.sql) | 398 B | 280.27 | 266.13 | 266.14 | 265.27 | 266.99 | ok | -| [Q32](benchmarks/queries/query_32.sql) | 376 B | 1030.95 | 777.39 | 776.24 | 769.61 | 787.48 | ok | -| [Q33](benchmarks/queries/query_33.sql) | 464 B | 403.13 | 404.75 | 405.27 | 402.76 | 405.69 | ok | -| [Q34](benchmarks/queries/query_34.sql) | 484 B | 453.77 | 422.89 | 422.68 | 419.71 | 426.50 | ok | -| [Q35](benchmarks/queries/query_35.sql) | 493 B | 156.60 | 155.02 | 155.09 | 153.26 | 156.64 | ok | -| [Q36](benchmarks/queries/query_36.sql) | 569 B | 347.88 | 342.61 | 342.61 | 342.31 | 342.92 | ok | -| [Q37](benchmarks/queries/query_37.sql) | 550 B | 244.48 | 243.77 | 243.88 | 242.73 | 244.58 | ok | -| [Q38](benchmarks/queries/query_38.sql) | 480 B | 80.37 | 74.58 | 74.44 | 74.34 | 75.09 | ok | -| [Q39](benchmarks/queries/query_39.sql) | 835 B | 453.18 | 450.37 | 449.66 | 447.42 | 454.74 | ok | -| [Q40](benchmarks/queries/query_40.sql) | 356 B | 111.18 | 93.89 | 93.72 | 93.59 | 94.54 | ok | -| [Q41](benchmarks/queries/query_41.sql) | 132 B | 37.92 | 34.08 | 34.09 | 33.77 | 34.37 | ok | -| [Q42](benchmarks/queries/query_42.sql) | 240 B | 68.61 | 69.07 | 69.12 | 68.65 | 69.40 | ok | +| [Q00](benchmarks/queries/query_0.sql) | 7 B | 372.47 | 7.12 | 7.14 | 5.32 | 8.89 | ok | +| [Q01](benchmarks/queries/query_1.sql) | 6 B | 16.18 | 13.17 | 12.85 | 11.37 | 15.61 | ok | +| [Q02](benchmarks/queries/query_2.sql) | 18 B | 28.44 | 21.85 | 21.68 | 21.58 | 22.45 | ok | +| [Q03](benchmarks/queries/query_3.sql) | 20 B | 14.56 | 14.35 | 14.30 | 14.24 | 14.55 | ok | +| [Q04](benchmarks/queries/query_4.sql) | 6 B | 151.94 | 148.55 | 147.02 | 146.46 | 153.70 | ok | +| [Q05](benchmarks/queries/query_5.sql) | 6 B | 131.23 | 128.78 | 128.76 | 128.60 | 129.00 | ok | +| [Q06](benchmarks/queries/query_6.sql) | 22 B | 4.63 | 4.11 | 4.12 | 4.00 | 4.22 | ok | +| [Q07](benchmarks/queries/query_7.sql) | 31 B | 11.54 | 11.68 | 11.69 | 11.61 | 11.71 | ok | +| [Q08](benchmarks/queries/query_8.sql) | 81 B | 189.62 | 189.70 | 189.51 | 189.29 | 190.49 | ok | +| [Q09](benchmarks/queries/query_9.sql) | 237 B | 209.27 | 209.04 | 208.53 | 207.81 | 211.30 | ok | +| [Q10](benchmarks/queries/query_10.sql) | 102 B | 273.63 | 268.25 | 267.84 | 264.68 | 272.65 | ok | +| [Q11](benchmarks/queries/query_11.sql) | 116 B | 267.74 | 267.06 | 267.06 | 266.30 | 267.81 | ok | +| [Q12](benchmarks/queries/query_12.sql) | 387 B | 225.38 | 225.50 | 225.37 | 225.15 | 226.12 | ok | +| [Q13](benchmarks/queries/query_13.sql) | 350 B | 255.52 | 253.82 | 253.71 | 253.17 | 254.72 | ok | +| [Q14](benchmarks/queries/query_14.sql) | 400 B | 227.71 | 227.58 | 227.55 | 227.02 | 228.18 | ok | +| [Q15](benchmarks/queries/query_15.sql) | 236 B | 73.92 | 74.08 | 74.16 | 73.23 | 74.78 | ok | +| [Q16](benchmarks/queries/query_16.sql) | 244 B | 116.27 | 117.15 | 116.97 | 116.74 | 117.90 | ok | +| [Q17](benchmarks/queries/query_17.sql) | 414 B | 146.45 | 145.98 | 146.00 | 145.32 | 146.60 | ok | +| [Q18](benchmarks/queries/query_18.sql) | 264 B | 371.65 | 354.57 | 349.51 | 345.73 | 373.53 | ok | +| [Q19](benchmarks/queries/query_19.sql) | 0 B | 5.11 | 4.40 | 4.39 | 4.35 | 4.45 | ok | +| [Q20](benchmarks/queries/query_20.sql) | 3 B | 253.57 | 252.47 | 252.69 | 251.74 | 252.77 | ok | +| [Q21](benchmarks/queries/query_21.sql) | 162 B | 275.26 | 277.07 | 277.31 | 274.21 | 279.46 | ok | +| [Q22](benchmarks/queries/query_22.sql) | 6.0 KB | 364.65 | 364.26 | 364.71 | 362.10 | 365.51 | ok | +| [Q23](benchmarks/queries/query_23.sql) | 2.6 KB | 247.58 | 240.46 | 240.62 | 239.89 | 240.72 | ok | +| [Q24](benchmarks/queries/query_24.sql) | 672 B | 221.33 | 220.77 | 220.81 | 220.51 | 220.95 | ok | +| [Q25](benchmarks/queries/query_25.sql) | 382 B | 219.30 | 219.65 | 219.69 | 218.74 | 220.49 | ok | +| [Q26](benchmarks/queries/query_26.sql) | 658 B | 221.11 | 220.32 | 220.03 | 219.98 | 221.26 | ok | +| [Q27](benchmarks/queries/query_27.sql) | 26 B | 448.24 | 450.32 | 450.51 | 448.06 | 452.19 | ok | +| [Q28](benchmarks/queries/query_28.sql) | 556 B | 972.11 | 969.67 | 969.54 | 964.82 | 974.79 | ok | +| [Q29](benchmarks/queries/query_29.sql) | 990 B | 436.39 | 427.32 | 427.54 | 426.26 | 427.95 | ok | +| [Q30](benchmarks/queries/query_30.sql) | 244 B | 237.16 | 235.05 | 235.10 | 234.09 | 235.90 | ok | +| [Q31](benchmarks/queries/query_31.sql) | 398 B | 264.83 | 262.59 | 262.43 | 261.71 | 263.79 | ok | +| [Q32](benchmarks/queries/query_32.sql) | 376 B | 1089.10 | 866.39 | 881.20 | 808.51 | 894.64 | ok | +| [Q33](benchmarks/queries/query_33.sql) | 464 B | 354.03 | 349.81 | 349.73 | 349.37 | 350.43 | ok | +| [Q34](benchmarks/queries/query_34.sql) | 484 B | 369.46 | 435.58 | 420.41 | 370.29 | 531.22 | ok | +| [Q35](benchmarks/queries/query_35.sql) | 493 B | 151.95 | 149.11 | 149.57 | 142.89 | 154.43 | ok | +| [Q36](benchmarks/queries/query_36.sql) | 569 B | 331.43 | 345.67 | 339.52 | 333.21 | 370.41 | ok | +| [Q37](benchmarks/queries/query_37.sql) | 550 B | 249.16 | 247.76 | 247.46 | 246.64 | 249.47 | ok | +| [Q38](benchmarks/queries/query_38.sql) | 480 B | 79.44 | 74.20 | 73.93 | 73.32 | 75.60 | ok | +| [Q39](benchmarks/queries/query_39.sql) | 835 B | 400.94 | 541.20 | 479.52 | 391.72 | 814.03 | ok | +| [Q40](benchmarks/queries/query_40.sql) | 356 B | 103.53 | 95.10 | 95.10 | 94.63 | 95.58 | ok | +| [Q41](benchmarks/queries/query_41.sql) | 132 B | 39.71 | 34.90 | 34.88 | 34.50 | 35.36 | ok | +| [Q42](benchmarks/queries/query_42.sql) | 240 B | 68.37 | 69.46 | 69.69 | 68.04 | 70.42 | ok | diff --git a/benchmarks/results/readme_benchmarks.csv b/benchmarks/results/readme_benchmarks.csv index e62935a..673aaa1 100644 --- a/benchmarks/results/readme_benchmarks.csv +++ b/benchmarks/results/readme_benchmarks.csv @@ -1,44 +1,44 @@ query_id,query_file,sql,first_run_ms,warm_avg_ms,warm_median_ms,warm_min_ms,warm_max_ms,output_csv_bytes,status,error -0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,9.49,4.96,4.87,4.59,5.51,7,ok, -1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,15.88,7.12,7.24,6.19,7.81,6,ok, -2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",28.03,19.08,18.79,17.92,20.83,18,ok, -3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,27.65,14.48,14.52,14.06,14.81,20,ok, -4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,156.02,190.53,167.49,153.12,274.03,6,ok, -5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,142.67,132.55,132.28,132.05,133.58,6,ok, -6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",4.78,4.27,4.26,4.12,4.43,22,ok, -7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",7.78,7.08,7.09,6.68,7.45,31,ok, -8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",204.57,205.39,204.97,201.88,209.73,81,ok, -9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",217.28,218.49,217.93,215.12,222.97,237,ok, -10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",281.97,269.05,268.97,268.52,269.74,102,ok, -11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",273.22,270.25,269.99,269.17,271.86,116,ok, -12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",227.97,230.01,229.34,227.64,233.73,387,ok, -13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",249.96,246.34,246.09,244.41,248.77,350,ok, -14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",232.70,233.59,232.91,231.02,237.51,400,ok, -15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",76.95,75.84,75.99,74.37,77.04,236,ok, -16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",130.83,129.57,129.28,129.09,130.63,244,ok, -17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",153.61,153.56,152.80,152.21,156.42,414,ok, -18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",445.38,374.33,374.64,372.72,375.32,264,ok, -19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,5.02,5.16,5.12,4.54,5.86,0,ok, -20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,194.65,145.74,145.69,145.27,146.31,3,ok, -21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",278.26,277.79,277.66,277.15,278.69,162,ok, -22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",426.01,375.71,374.66,374.01,379.49,6188,ok, -23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,285.01,244.43,244.56,242.75,245.86,2633,ok, -24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,268.59,268.79,268.40,267.75,270.62,658,ok, -25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,222.70,219.63,219.57,219.47,219.92,382,ok, -26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",268.76,270.05,270.06,269.16,270.92,658,ok, -27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",425.51,420.92,420.86,416.18,425.80,26,ok, -28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",979.03,1016.62,979.97,974.72,1131.81,556,ok, -29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",395.85,393.43,391.93,390.74,399.12,990,ok, -30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",250.48,238.09,237.88,236.56,240.02,244,ok, -31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",280.27,266.13,266.14,265.27,266.99,398,ok, -32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",1030.95,777.39,776.24,769.61,787.48,376,ok, -33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",403.13,404.75,405.27,402.76,405.69,464,ok, -34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",453.77,422.89,422.68,419.71,426.50,484,ok, -35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",156.60,155.02,155.09,153.26,156.64,493,ok, -36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",347.88,342.61,342.61,342.31,342.92,569,ok, -37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",244.48,243.77,243.88,242.73,244.58,550,ok, -38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",80.37,74.58,74.44,74.34,75.09,480,ok, -39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",453.18,450.37,449.66,447.42,454.74,835,ok, -40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",111.18,93.89,93.72,93.59,94.54,356,ok, -41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",37.92,34.08,34.09,33.77,34.37,132,ok, -42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",68.61,69.07,69.12,68.65,69.40,240,ok, +0,benchmarks/queries/query_0.sql,SELECT COUNT(*) FROM hits;,372.47,7.12,7.14,5.32,8.89,7,ok, +1,benchmarks/queries/query_1.sql,SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;,16.18,13.17,12.85,11.37,15.61,6,ok, +2,benchmarks/queries/query_2.sql,"SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;",28.44,21.85,21.68,21.58,22.45,18,ok, +3,benchmarks/queries/query_3.sql,SELECT AVG(UserID) FROM hits;,14.56,14.35,14.30,14.24,14.55,20,ok, +4,benchmarks/queries/query_4.sql,SELECT COUNT(DISTINCT UserID) FROM hits;,151.94,148.55,147.02,146.46,153.70,6,ok, +5,benchmarks/queries/query_5.sql,SELECT COUNT(DISTINCT SearchPhrase) FROM hits;,131.23,128.78,128.76,128.60,129.00,6,ok, +6,benchmarks/queries/query_6.sql,"SELECT MIN(EventDate), MAX(EventDate) FROM hits;",4.63,4.11,4.12,4.00,4.22,22,ok, +7,benchmarks/queries/query_7.sql,"SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;",11.54,11.68,11.69,11.61,11.71,31,ok, +8,benchmarks/queries/query_8.sql,"SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;",189.62,189.70,189.51,189.29,190.49,81,ok, +9,benchmarks/queries/query_9.sql,"SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;",209.27,209.04,208.53,207.81,211.30,237,ok, +10,benchmarks/queries/query_10.sql,"SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;",273.63,268.25,267.84,264.68,272.65,102,ok, +11,benchmarks/queries/query_11.sql,"SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;",267.74,267.06,267.06,266.30,267.81,116,ok, +12,benchmarks/queries/query_12.sql,"SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",225.38,225.50,225.37,225.15,226.12,387,ok, +13,benchmarks/queries/query_13.sql,"SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;",255.52,253.82,253.71,253.17,254.72,350,ok, +14,benchmarks/queries/query_14.sql,"SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;",227.71,227.58,227.55,227.02,228.18,400,ok, +15,benchmarks/queries/query_15.sql,"SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;",73.92,74.08,74.16,73.23,74.78,236,ok, +16,benchmarks/queries/query_16.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",116.27,117.15,116.97,116.74,117.90,244,ok, +17,benchmarks/queries/query_17.sql,"SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;",146.45,145.98,146.00,145.32,146.60,414,ok, +18,benchmarks/queries/query_18.sql,"SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;",371.65,354.57,349.51,345.73,373.53,264,ok, +19,benchmarks/queries/query_19.sql,SELECT UserID FROM hits WHERE UserID = 435090932899640449;,5.11,4.40,4.39,4.35,4.45,0,ok, +20,benchmarks/queries/query_20.sql,SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';,253.57,252.47,252.69,251.74,252.77,3,ok, +21,benchmarks/queries/query_21.sql,"SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",275.26,277.07,277.31,274.21,279.46,162,ok, +22,benchmarks/queries/query_22.sql,"SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;",364.65,364.26,364.71,362.10,365.51,6188,ok, +23,benchmarks/queries/query_23.sql,SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;,247.58,240.46,240.62,239.89,240.72,2633,ok, +24,benchmarks/queries/query_24.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;,221.33,220.77,220.81,220.51,220.95,672,ok, +25,benchmarks/queries/query_25.sql,SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;,219.30,219.65,219.69,218.74,220.49,382,ok, +26,benchmarks/queries/query_26.sql,"SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;",221.11,220.32,220.03,219.98,221.26,658,ok, +27,benchmarks/queries/query_27.sql,"SELECT CounterID, AVG(STRLEN(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",448.24,450.32,450.51,448.06,452.19,26,ok, +28,benchmarks/queries/query_28.sql,"SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(STRLEN(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;",972.11,969.67,969.54,964.82,974.79,556,ok, +29,benchmarks/queries/query_29.sql,"SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;",436.39,427.32,427.54,426.26,427.95,990,ok, +30,benchmarks/queries/query_30.sql,"SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;",237.16,235.05,235.10,234.09,235.90,244,ok, +31,benchmarks/queries/query_31.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",264.83,262.59,262.43,261.71,263.79,398,ok, +32,benchmarks/queries/query_32.sql,"SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;",1089.10,866.39,881.20,808.51,894.64,376,ok, +33,benchmarks/queries/query_33.sql,"SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;",354.03,349.81,349.73,349.37,350.43,464,ok, +34,benchmarks/queries/query_34.sql,"SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;",369.46,435.58,420.41,370.29,531.22,484,ok, +35,benchmarks/queries/query_35.sql,"SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;",151.95,149.11,149.57,142.89,154.43,493,ok, +36,benchmarks/queries/query_36.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;",331.43,345.67,339.52,333.21,370.41,569,ok, +37,benchmarks/queries/query_37.sql,"SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;",249.16,247.76,247.46,246.64,249.47,550,ok, +38,benchmarks/queries/query_38.sql,"SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",79.44,74.20,73.93,73.32,75.60,480,ok, +39,benchmarks/queries/query_39.sql,"SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;",400.94,541.20,479.52,391.72,814.03,835,ok, +40,benchmarks/queries/query_40.sql,"SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;",103.53,95.10,95.10,94.63,95.58,356,ok, +41,benchmarks/queries/query_41.sql,"SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;",39.71,34.90,34.88,34.50,35.36,132,ok, +42,benchmarks/queries/query_42.sql,"SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;",68.37,69.46,69.69,68.04,70.42,240,ok, From e8dbad2d020b5e059fff860d373b548e6dfd804d Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 21:40:17 +0300 Subject: [PATCH 12/13] fix: refactoring --- include/executor/query.h | 14 +++++++++++++- include/executor/query_plan.h | 7 ++++++- include/io/csv_batch.h | 1 + include/model/metadata.h | 3 +++ src/executor/operators_aggregate.cpp | 17 +++++++++++++++++ src/executor/operators_runtime.cpp | 13 +++++++++++++ src/executor/operators_scan.cpp | 4 ++++ 7 files changed, 57 insertions(+), 2 deletions(-) diff --git a/include/executor/query.h b/include/executor/query.h index 6ed56ad..36ca0e3 100644 --- a/include/executor/query.h +++ b/include/executor/query.h @@ -42,6 +42,7 @@ enum class QueryValueKind { struct QueryValue { QueryValueKind kind = QueryValueKind::Literal; + ColumnRef column; QueryLiteral literal; }; @@ -76,6 +77,7 @@ struct ExprSpec { ExprKind kind = ExprKind::Literal; ColumnRef column; + bool column_index_bound = false; size_t column_index = 0; @@ -109,8 +111,8 @@ struct PredicateSpec { PredicateKind kind = PredicateKind::Comparison; ExprPtr left; - ComparisonKind comparison = ComparisonKind::Equal; ExprPtr right; + ComparisonKind comparison = ComparisonKind::Equal; std::vector values; @@ -118,24 +120,29 @@ struct PredicateSpec { PredicatePtr rhs; bool typed_literal_comparison_bound = false; + size_t typed_column_index = 0; Int128 typed_literal_value = 0; ComparisonKind typed_comparison = ComparisonKind::Equal; bool metadata_typed_literal_comparison_bound = false; + size_t metadata_typed_column_index = 0; Int128 metadata_typed_literal_value = 0; ComparisonKind metadata_typed_comparison = ComparisonKind::Equal; bool literal_in_set_bound = false; + size_t in_column_index = 0; std::unordered_set literal_in_values; bool metadata_typed_in_set_bound = false; + size_t metadata_typed_in_column_index = 0; std::vector metadata_typed_in_values; bool literal_like_pattern_bound = false; + size_t like_column_index = 0; std::string like_pattern; bool like_negated = false; @@ -149,9 +156,12 @@ enum class AggArgumentKind { struct AggSpec { std::string function_name; bool distinct = false; + AggArgumentKind argument_kind = AggArgumentKind::Column; + ColumnRef column; ExprPtr argument; + std::string output_name; }; @@ -162,9 +172,11 @@ enum class SelectItemKind { struct SelectItemSpec { SelectItemKind kind = SelectItemKind::GroupKey; + ColumnRef column; ExprPtr expression; AggSpec aggregate; + std::string output_name; }; diff --git a/include/executor/query_plan.h b/include/executor/query_plan.h index fcebdcd..9fc7f32 100644 --- a/include/executor/query_plan.h +++ b/include/executor/query_plan.h @@ -40,13 +40,16 @@ struct PlannedGroupKey { struct PlannedSelectItem { SelectItemKind kind = SelectItemKind::GroupKey; size_t index = 0; + ColumnType output_type = ColumnType::String; std::string output_name; }; struct PlannedOrderBy { size_t result_column_index = 0; + bool descending = false; + ColumnType value_type = ColumnType::String; }; @@ -66,8 +69,9 @@ struct PlannedValue { struct PlannedFilter { PlannedValue left; - ComparisonKind comparison = ComparisonKind::Equal; PlannedValue right; + + ComparisonKind comparison = ComparisonKind::Equal; ColumnType comparison_type = ColumnType::String; }; @@ -89,6 +93,7 @@ struct PlannedQuery { std::vector select_items; std::vector plain_select_items; + std::vector order_by; std::vector plain_order_by; diff --git a/include/io/csv_batch.h b/include/io/csv_batch.h index 97dbab2..de63f9a 100644 --- a/include/io/csv_batch.h +++ b/include/io/csv_batch.h @@ -22,6 +22,7 @@ class CsvBatchReader final : public BatchReader { private: CsvReader csv_reader_; + Schema schema_; BatchSizing sizing_; diff --git a/include/model/metadata.h b/include/model/metadata.h index 2152d1c..9db52f7 100644 --- a/include/model/metadata.h +++ b/include/model/metadata.h @@ -10,10 +10,13 @@ struct ColumnChunkMetadata { uint64_t offset = 0; + uint64_t compressed_size = 0; uint64_t uncompressed_size = 0; Compression compression = Compression::None; + bool has_min_max = false; + Int128 min_value = 0; Int128 max_value = 0; }; diff --git a/src/executor/operators_aggregate.cpp b/src/executor/operators_aggregate.cpp index 02ed743..56c6620 100644 --- a/src/executor/operators_aggregate.cpp +++ b/src/executor/operators_aggregate.cpp @@ -73,11 +73,13 @@ void ConsumeAggRows(const PlannedAgg& aggregate, const Batch& batch, const std:: struct AggBinding { PlannedAgg aggregate; + std::unique_ptr state; }; struct GroupKeyComponent { ColumnType type = ColumnType::String; + Int128 int_value = 0; std::string_view string_value; }; @@ -414,9 +416,12 @@ class GroupAggOperator final : public Operator { private: struct GroupState { std::vector key_values; + std::vector> states; + mutable std::vector aggregate_values; mutable std::vector aggregate_int_values; + mutable bool aggregate_values_finalized = false; }; @@ -522,13 +527,17 @@ class GroupAggOperator final : public Operator { std::unique_ptr child_; GroupKeyMaterializer group_key_materializer_; + std::vector bindings_; std::vector select_items_; + PredicatePtr filter_; PredicatePtr having_; + StringArena string_arena_; std::unordered_map group_indexes_; std::vector groups_; + bool sort_by_group_keys_ = false; bool returned_ = false; }; @@ -606,9 +615,12 @@ class GroupAggTopKOperator final : public Operator { private: struct GroupState { std::vector key_values; + std::vector> states; + mutable std::vector aggregate_values; mutable std::vector aggregate_int_values; + mutable bool aggregate_values_finalized = false; }; @@ -713,14 +725,19 @@ class GroupAggTopKOperator final : public Operator { std::unique_ptr child_; GroupKeyMaterializer group_key_materializer_; + std::vector bindings_; std::vector select_items_; + PredicatePtr filter_; + std::vector order_by_; size_t limit_ = 0; + StringArena string_arena_; std::unordered_map group_indexes_; mutable std::vector groups_; + bool returned_ = false; }; diff --git a/src/executor/operators_runtime.cpp b/src/executor/operators_runtime.cpp index c2ef071..75ef34b 100644 --- a/src/executor/operators_runtime.cpp +++ b/src/executor/operators_runtime.cpp @@ -251,6 +251,7 @@ ColumnType InferComparisonType(const std::string& lhs, const std::string& rhs) { struct RowRef { size_t batch_index = 0; size_t row = 0; + size_t ordinal = 0; }; @@ -328,6 +329,7 @@ class FilterOperator final : public Operator { private: std::unique_ptr child_; + PredicatePtr filter_; }; @@ -358,7 +360,9 @@ class EnsureSchemaOperator final : public Operator { private: std::unique_ptr child_; + Schema schema_; + bool returned_any_ = false; bool returned_empty_ = false; }; @@ -406,10 +410,14 @@ class ProjectionOperator final : public Operator { private: std::unique_ptr child_; + std::vector items_; + Schema source_schema_; Schema schema_; + std::vector star_column_indexes_; + bool star_column_indexes_initialized_ = false; }; @@ -458,8 +466,10 @@ class OrderByOperator final : public Operator { private: std::unique_ptr child_; + std::vector batches_; RowOrdering ordering_; + bool returned_ = false; }; @@ -522,9 +532,12 @@ class TopKOperator final : public Operator { private: std::unique_ptr child_; + std::vector batches_; RowOrdering ordering_; + size_t limit_; + bool returned_ = false; }; diff --git a/src/executor/operators_scan.cpp b/src/executor/operators_scan.cpp index 39a026f..0e89bc3 100644 --- a/src/executor/operators_scan.cpp +++ b/src/executor/operators_scan.cpp @@ -37,6 +37,7 @@ class MetadataCountOperator final : public Operator { private: std::filesystem::path path_; std::string output_name_; + bool returned_ = false; }; @@ -91,6 +92,7 @@ class MetadataExtremaOperator final : public Operator { private: std::filesystem::path path_; std::vector aggregates_; + bool returned_ = false; }; @@ -211,7 +213,9 @@ class ScanOperator final : public Operator { ColumnarMetadata metadata_; std::vector projection_indexes_; Schema projected_schema_; + PredicatePtr filter_; + size_t next_group_ = 0; }; From 1e283834091ea0b9bd29e02b26a9b62f14359f5d Mon Sep 17 00:00:00 2001 From: ik4rd Date: Thu, 28 May 2026 21:45:51 +0300 Subject: [PATCH 13/13] fix: reduntant move --- src/executor/operators_runtime.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/executor/operators_runtime.cpp b/src/executor/operators_runtime.cpp index 75ef34b..2e35cbc 100644 --- a/src/executor/operators_runtime.cpp +++ b/src/executor/operators_runtime.cpp @@ -74,7 +74,7 @@ std::optional TryResolveAggregateValue(const ExprSpec& expr, const std::string EvalFunction(const ExprSpec& expr, const Batch& batch, const size_t row) { if (const auto aggregate_value = TryResolveAggregateValue(expr, batch, row); aggregate_value.has_value()) { - return std::move(*aggregate_value); + return *aggregate_value; } const std::string name = ToUpperAscii(expr.function_name);