From d0cb9dbab8aeec6f3817cf516e9f3eef0f0947da Mon Sep 17 00:00:00 2001 From: DinosL Date: Tue, 21 Oct 2025 12:50:39 +0200 Subject: [PATCH 01/27] added row_number virtual column --- .../catalog_entry/duck_table_entry.cpp | 3 + .../catalog_entry/table_catalog_entry.cpp | 7 ++ src/common/constants.cpp | 5 ++ src/function/table/table_scan.cpp | 48 ++++++++++++- .../catalog_entry/table_catalog_entry.hpp | 1 + src/include/duckdb/common/column_index.hpp | 3 + src/include/duckdb/common/constants.hpp | 2 + src/include/duckdb/storage/storage_index.hpp | 3 + .../duckdb/storage/table/row_group.hpp | 4 ++ .../storage/table/row_number_column_data.hpp | 30 ++++++++ src/storage/table/CMakeLists.txt | 1 + src/storage/table/row_group.cpp | 29 ++++++-- src/storage/table/row_number_column_data.cpp | 44 ++++++++++++ test/optimizer/row_number_virtual_column.test | 72 +++++++++++++++++++ 14 files changed, 244 insertions(+), 8 deletions(-) create mode 100644 src/include/duckdb/storage/table/row_number_column_data.hpp create mode 100644 src/storage/table/row_number_column_data.cpp create mode 100644 test/optimizer/row_number_virtual_column.test diff --git a/src/catalog/catalog_entry/duck_table_entry.cpp b/src/catalog/catalog_entry/duck_table_entry.cpp index 4f1866ee4969..216e28305353 100644 --- a/src/catalog/catalog_entry/duck_table_entry.cpp +++ b/src/catalog/catalog_entry/duck_table_entry.cpp @@ -130,6 +130,9 @@ unique_ptr DuckTableEntry::GetStatistics(ClientContext &context, if (column_id == COLUMN_IDENTIFIER_ROW_ID) { return nullptr; } + if (column_id == COLUMN_IDENTIFIER_ROW_NUMBER) { + return nullptr; + } auto &column = columns.GetColumn(LogicalIndex(column_id)); if (column.Generated()) { return nullptr; diff --git a/src/catalog/catalog_entry/table_catalog_entry.cpp b/src/catalog/catalog_entry/table_catalog_entry.cpp index 8582fa93c6ac..7c5a456d630e 100644 --- a/src/catalog/catalog_entry/table_catalog_entry.cpp +++ b/src/catalog/catalog_entry/table_catalog_entry.cpp @@ -348,6 +348,7 @@ bool TableCatalogEntry::HasPrimaryKey() const { virtual_column_map_t TableCatalogEntry::GetVirtualColumns() const { virtual_column_map_t virtual_columns; virtual_columns.insert(make_pair(COLUMN_IDENTIFIER_ROW_ID, TableColumn("rowid", LogicalType::ROW_TYPE))); + virtual_columns.insert(make_pair(COLUMN_IDENTIFIER_ROW_NUMBER, TableColumn("row_number", LogicalType::ROW_TYPE))); return virtual_columns; } @@ -357,4 +358,10 @@ vector TableCatalogEntry::GetRowIdColumns() const { return result; } +vector TableCatalogEntry::GetRowNumberColumns() const { + vector result; + result.push_back(COLUMN_IDENTIFIER_ROW_NUMBER); + return result; +} + } // namespace duckdb diff --git a/src/common/constants.cpp b/src/common/constants.cpp index 2bddd619532f..7e00aa9322e7 100644 --- a/src/common/constants.cpp +++ b/src/common/constants.cpp @@ -11,6 +11,7 @@ const row_t MAX_ROW_ID = 36028797018960000ULL; // 2^55 const row_t MAX_ROW_ID_LOCAL = 72057594037920000ULL; // 2^56 const column_t COLUMN_IDENTIFIER_ROW_ID = UINT64_C(18446744073709551615); const column_t COLUMN_IDENTIFIER_EMPTY = UINT64_C(18446744073709551614); +const column_t COLUMN_IDENTIFIER_ROW_NUMBER = UINT64_C(18446744073709551613); const column_t VIRTUAL_COLUMN_START = UINT64_C(9223372036854775808); // 2^63 const double PI = 3.141592653589793; @@ -58,6 +59,10 @@ bool IsRowIdColumnId(column_t column_id) { return column_id == COLUMN_IDENTIFIER_ROW_ID; } +bool IsRowNumberColumnId(column_t column_id) { + return column_id == COLUMN_IDENTIFIER_ROW_NUMBER; +} + bool IsVirtualColumn(column_t column_id) { return column_id >= VIRTUAL_COLUMN_START; } diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index fc24ec702e78..ad78c8767c18 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -40,6 +40,7 @@ struct TableScanLocalState : public LocalTableFunctionState { //! The DataChunk containing all read columns. //! This includes filter columns, which are immediately removed. DataChunk all_columns; + idx_t row_number_count; }; struct IndexScanLocalState : public LocalTableFunctionState { @@ -69,6 +70,10 @@ static StorageIndex GetStorageIndex(TableCatalogEntry &table, const ColumnIndex return StorageIndex(); } + if (column_id.IsRowNumberColumn()) { + return StorageIndex(COLUMN_IDENTIFIER_ROW_NUMBER); + } + // The index of the base ColumnIndex is equal to the physical column index in the table // for any child indices because the indices are already the physical indices. // Only the top-level can have generated columns. @@ -93,6 +98,10 @@ class TableScanGlobalState : public GlobalTableFunctionState { vector projection_ids; //! The types of all scanned columns. vector scanned_types; + //! row_number offsets for each row group + vector row_number_offsets; + //! row_number column index + idx_t row_number_col_index = DConstants::INVALID_INDEX; public: virtual unique_ptr InitLocalState(ExecutionContext &context, @@ -268,6 +277,7 @@ class DuckTableScanState : public TableScanGlobalState { void TableScanFunc(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) override { auto &l_state = data_p.local_state->Cast(); l_state.scan_state.options.force_fetch_row = ClientConfig::GetConfig(context).force_fetch_row; + auto &global_state = data_p.global_state->Cast(); do { if (context.interrupted) { @@ -284,6 +294,21 @@ class DuckTableScanState : public TableScanGlobalState { storage.Scan(tx, output, l_state.scan_state); } if (output.size() > 0) { + if (row_number_col_index != DConstants::INVALID_INDEX) { + auto &row_number_vec = output.data[row_number_col_index]; + row_number_vec.SetVectorType(VectorType::FLAT_VECTOR); + auto row_number_data = FlatVector::GetData(row_number_vec); + auto count = output.size(); + + idx_t row_group_index = l_state.scan_state.table_state.batch_index - 1; + D_ASSERT(row_group_index < global_state.row_number_offsets.size()); + idx_t base = global_state.row_number_offsets[row_group_index] + l_state.row_number_count; + + for (idx_t i = 0; i < count; i++) { + row_number_data[i] = static_cast(base + i + 1); + } + l_state.row_number_count += count; + } return; } @@ -291,6 +316,7 @@ class DuckTableScanState : public TableScanGlobalState { if (!next) { return; } + l_state.row_number_count = 0; } while (true); } @@ -332,7 +358,7 @@ static unique_ptr TableScanInitLocal(ExecutionContext & } unique_ptr DuckTableScanInitGlobal(ClientContext &context, TableFunctionInitInput &input, - DataTable &storage, const TableScanBindData &bind_data) { + DataTable &storage, const TableScanBindData &bind_data) { auto g_state = make_uniq(context, input.bind_data.get()); if (bind_data.order_options) { g_state->state.scan_state.reorderer = make_uniq(*bind_data.order_options); @@ -340,6 +366,24 @@ unique_ptr DuckTableScanInitGlobal(ClientContext &cont } storage.InitializeParallelScan(context, g_state->state); + for (idx_t i = 0; i < input.column_ids.size(); i++) { + if (input.column_ids[i] == COLUMN_IDENTIFIER_ROW_NUMBER) { + g_state->row_number_col_index = i; + break; + } + } + if (g_state->row_number_col_index != DConstants::INVALID_INDEX) { + idx_t current_offset = 0; + int64_t counter = 0; + auto row_groups = g_state->state.scan_state.collection; + auto row_group = row_groups->GetRowGroup(counter); + while (row_group) { + g_state->row_number_offsets.push_back(current_offset); + current_offset += row_group->GetCommittedRowCount(); + counter++; + row_group = row_groups->GetRowGroup(counter); + } + } if (!input.CanRemoveFilterColumns()) { return std::move(g_state); } @@ -348,7 +392,7 @@ unique_ptr DuckTableScanInitGlobal(ClientContext &cont auto &duck_table = bind_data.table.Cast(); const auto &columns = duck_table.GetColumns(); for (const auto &col_idx : input.column_indexes) { - if (col_idx.IsRowIdColumn()) { + if (col_idx.IsRowIdColumn() || col_idx.IsRowNumberColumn()) { g_state->scanned_types.emplace_back(LogicalType::ROW_TYPE); } else { g_state->scanned_types.push_back(columns.GetColumn(col_idx.ToLogical()).Type()); diff --git a/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp b/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp index 1e40319cf270..23ccf581f656 100644 --- a/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp @@ -128,6 +128,7 @@ class TableCatalogEntry : public StandardEntry { virtual virtual_column_map_t GetVirtualColumns() const; virtual vector GetRowIdColumns() const; + virtual vector GetRowNumberColumns() const; protected: //! A list of columns that are part of this table diff --git a/src/include/duckdb/common/column_index.hpp b/src/include/duckdb/common/column_index.hpp index b7bc3ae971f3..f473aa46051e 100644 --- a/src/include/duckdb/common/column_index.hpp +++ b/src/include/duckdb/common/column_index.hpp @@ -61,6 +61,9 @@ struct ColumnIndex { bool IsRowIdColumn() const { return index == COLUMN_IDENTIFIER_ROW_ID; } + bool IsRowNumberColumn() const { + return index == COLUMN_IDENTIFIER_ROW_NUMBER; + } bool IsEmptyColumn() const { return index == COLUMN_IDENTIFIER_EMPTY; } diff --git a/src/include/duckdb/common/constants.hpp b/src/include/duckdb/common/constants.hpp index e9f816a17619..e1101f5d5cde 100644 --- a/src/include/duckdb/common/constants.hpp +++ b/src/include/duckdb/common/constants.hpp @@ -40,6 +40,8 @@ DUCKDB_API bool IsInvalidCatalog(const string &str); //! Special value used to signify the ROW ID of a table DUCKDB_API extern const column_t COLUMN_IDENTIFIER_ROW_ID; +//! Special value used to signify the ROW_NUMBER of a table +DUCKDB_API extern const column_t COLUMN_IDENTIFIER_ROW_NUMBER; //! Special value used to signify an empty column (used for e.g. COUNT(*)) DUCKDB_API extern const column_t COLUMN_IDENTIFIER_EMPTY; DUCKDB_API extern const column_t VIRTUAL_COLUMN_START; diff --git a/src/include/duckdb/storage/storage_index.hpp b/src/include/duckdb/storage/storage_index.hpp index 38ba99976261..cc37e02a9c7c 100644 --- a/src/include/duckdb/storage/storage_index.hpp +++ b/src/include/duckdb/storage/storage_index.hpp @@ -61,6 +61,9 @@ struct StorageIndex { bool IsRowIdColumn() const { return index == DConstants::INVALID_INDEX; } + bool IsRowNumberColumn() const { + return index == COLUMN_IDENTIFIER_ROW_NUMBER; + } private: idx_t index; diff --git a/src/include/duckdb/storage/table/row_group.hpp b/src/include/duckdb/storage/table/row_group.hpp index 64234516843e..5bfd06242ea1 100644 --- a/src/include/duckdb/storage/table/row_group.hpp +++ b/src/include/duckdb/storage/table/row_group.hpp @@ -224,6 +224,7 @@ class RowGroup : public SegmentBase { idx_t GetColumnCount() const; vector> &GetColumns(); ColumnData &GetRowIdColumnData(); + ColumnData &GetRowNumberColumnData(); void SetCount(idx_t count); template @@ -235,6 +236,7 @@ class RowGroup : public SegmentBase { private: mutex row_group_lock; + mutex row_number_group_lock; vector column_pointers; unique_ptr[]> is_loaded; vector deletes_pointers; @@ -244,6 +246,8 @@ class RowGroup : public SegmentBase { atomic allocation_size; unique_ptr row_id_column_data; atomic row_id_is_loaded; + unique_ptr row_number_column_data; + atomic row_number_is_loaded; atomic has_changes; }; diff --git a/src/include/duckdb/storage/table/row_number_column_data.hpp b/src/include/duckdb/storage/table/row_number_column_data.hpp new file mode 100644 index 000000000000..5ffc86eafaea --- /dev/null +++ b/src/include/duckdb/storage/table/row_number_column_data.hpp @@ -0,0 +1,30 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/storage/table/row_number_column_data.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/storage/table/column_data.hpp" + +namespace duckdb { + +class RowNumberColumnData : public ColumnData { +public: + RowNumberColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t start_row); + +public: + void InitializeScan(ColumnScanState &state) override; + void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override; + idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, + idx_t scan_count) override; + idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates, + idx_t scan_count) override; + void ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result) override; + idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count, idx_t result_offset = 0) override; +}; + +} // namespace duckdb diff --git a/src/storage/table/CMakeLists.txt b/src/storage/table/CMakeLists.txt index 62dd5e3481ba..2a3da5ca2859 100644 --- a/src/storage/table/CMakeLists.txt +++ b/src/storage/table/CMakeLists.txt @@ -12,6 +12,7 @@ add_library_unity( update_segment.cpp persistent_table_data.cpp row_id_column_data.cpp + row_number_column_data.cpp row_group.cpp row_group_collection.cpp row_version_manager.cpp diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 8f6688e9bf44..8c2f7eeed396 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -21,19 +21,20 @@ #include "duckdb/transaction/duck_transaction.hpp" #include "duckdb/transaction/duck_transaction_manager.hpp" #include "duckdb/storage/table/row_id_column_data.hpp" +#include "duckdb/storage/table/row_number_column_data.hpp" #include "duckdb/main/settings.hpp" namespace duckdb { RowGroup::RowGroup(RowGroupCollection &collection_p, idx_t count) - : SegmentBase(count), collection(collection_p), version_info(nullptr), deletes_is_loaded(false), - allocation_size(0), row_id_is_loaded(false), has_changes(false) { + : SegmentBase(count), collection(collection_p), version_info(nullptr), + allocation_size(0), row_id_is_loaded(false), row_number_is_loaded(false), has_changes(false) { Verify(); } RowGroup::RowGroup(RowGroupCollection &collection_p, RowGroupPointer pointer) : SegmentBase(pointer.tuple_count), collection(collection_p), version_info(nullptr), - deletes_is_loaded(false), allocation_size(0), row_id_is_loaded(false), has_changes(false) { +allocation_size(0), row_id_is_loaded(false), row_number_is_loaded(false), has_changes(false) { // deserialize the columns if (pointer.data_pointers.size() != collection_p.GetTypes().size()) { throw IOException("Row group column count is unaligned with table column count. Corrupt file?"); @@ -53,7 +54,7 @@ RowGroup::RowGroup(RowGroupCollection &collection_p, RowGroupPointer pointer) RowGroup::RowGroup(RowGroupCollection &collection_p, PersistentRowGroupData &data) : SegmentBase(data.count), collection(collection_p), version_info(nullptr), deletes_is_loaded(false), - allocation_size(0), row_id_is_loaded(false), has_changes(false) { + allocation_size(0), row_id_is_loaded(false), row_number_is_loaded(false), has_changes(false) { auto &block_manager = GetBlockManager(); auto &info = GetTableInfo(); auto &types = collection.get().GetTypes(); @@ -119,6 +120,19 @@ ColumnData &RowGroup::GetRowIdColumnData() { return *row_id_column_data; } +ColumnData &RowGroup::GetRowNumberColumnData() { + if (row_number_is_loaded) { + return *row_number_column_data; + } + lock_guard l(row_number_group_lock); + if (!row_number_column_data) { + row_number_column_data = make_uniq(GetBlockManager(), GetTableInfo(), start); + row_number_column_data->count = count.load(); + row_number_is_loaded = true; + } + return *row_number_column_data; +} + ColumnData &RowGroup::GetColumn(const StorageIndex &c) { return GetColumn(c.GetPrimaryIndex()); } @@ -127,6 +141,9 @@ ColumnData &RowGroup::GetColumn(storage_t c) { if (c == COLUMN_IDENTIFIER_ROW_ID) { return GetRowIdColumnData(); } + if (c == COLUMN_IDENTIFIER_ROW_NUMBER) { + return GetRowNumberColumnData(); + } D_ASSERT(c < columns.size()); if (!is_loaded) { // not being lazy loaded @@ -246,7 +263,7 @@ void CollectionScanState::Initialize(const QueryContext &context, const vector RowGroup::RemoveColumn(RowGroupCollection &new_collection, D_ASSERT(removed_column < columns.size()); - auto row_group = make_uniq(new_collection, this->count); + auto row_group = make_uniq(new_collection, this->start, this->count); row_group->SetVersionInfo(GetOrCreateVersionInfoPtr()); // copy over all columns except for the removed one auto &cols = GetColumns(); diff --git a/src/storage/table/row_number_column_data.cpp b/src/storage/table/row_number_column_data.cpp new file mode 100644 index 000000000000..c2fb4efc64f4 --- /dev/null +++ b/src/storage/table/row_number_column_data.cpp @@ -0,0 +1,44 @@ +#include "duckdb/storage/table/scan_state.hpp" +#include "duckdb/storage/table/row_number_column_data.hpp" + +namespace duckdb { + +RowNumberColumnData::RowNumberColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t start_row) + : ColumnData(block_manager, info, COLUMN_IDENTIFIER_ROW_NUMBER, start_row, LogicalType(LogicalTypeId::BIGINT), + nullptr) { +} + +void RowNumberColumnData::InitializeScan(ColumnScanState &state) { + InitializeScanWithOffset(state, start); +} + +void RowNumberColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) { + state.row_index = row_idx; +} + +idx_t RowNumberColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, + idx_t scan_count) { + return ScanCommitted(vector_index, state, result, false, scan_count); +} + +idx_t RowNumberColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates, + idx_t scan_count) { + return ScanCount(state, result, scan_count, 0); +} + +void RowNumberColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, + Vector &result) { + D_ASSERT(this->start == row_group_start); + result.Sequence(UnsafeNumericCast(1 + offset_in_row_group), 1, count); +} + +idx_t RowNumberColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t count, idx_t result_offset) { + if (result_offset != 0) { + throw InternalException("RowNumberColumnData result_offset must be 0"); + } + ScanCommittedRange(start, state.row_index - start, count, result); + state.row_index += count; + return count; +} + +} // namespace duckdb diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test new file mode 100644 index 000000000000..e28a9dd2f842 --- /dev/null +++ b/test/optimizer/row_number_virtual_column.test @@ -0,0 +1,72 @@ +# name: test/optimizer/row_number_virtual_column.test +# description: Test row_number virtual column +# group: [optimizer] + +statement ok +pragma disable_optimizer; + +# statement ok +# pragma threads=2; + +statement ok +CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(500000); + +statement ok +delete from foo where a<10; + +query II +select a, row_number from foo limit 10; +---- +10 1 +11 2 +12 3 +13 4 +14 5 +15 6 +16 7 +17 8 +18 9 +19 10 + + +query II +select a, row_number from foo where a >= 100000 and a < 200000 limit 1; +---- +100000 99991 + + +statement ok +delete from foo where a>1500 and a < 150000; + +statement ok +delete from foo where a>1 and a < 1500; + +query II +select a, row_number from foo limit 10; +---- +1500 1 +150000 2 +150001 3 +150002 4 +150003 5 +150004 6 +150005 7 +150006 8 +150007 9 +150008 10 + +statement ok +CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(500000); + +statement ok +delete from foo where a>150 and a < 1500; + +# query IIII +# select a,rowid, row_number, row_number() over() as x from foo order by a desc limit 1; +# ---- +# 129999 129999 198651 198651 + +query III +select a,rowid, row_number from foo order by a desc limit 1; +---- +499999 499999 498651 From 85911252a2535d97f3435cc34c1121d9e4ca1a10 Mon Sep 17 00:00:00 2001 From: DinosL Date: Thu, 23 Oct 2025 16:46:19 +0200 Subject: [PATCH 02/27] added optimizer rule to rewrite window functions to emit row_numbers in parallel --- src/common/enum_util.cpp | 10 +- src/common/enums/metric_type.cpp | 6 + src/common/enums/optimizer_type.cpp | 1 + src/function/table/table_scan.cpp | 6 +- .../duckdb/common/enums/metric_type.hpp | 1 + .../duckdb/common/enums/optimizer_type.hpp | 1 + .../join_filter_pushdown_optimizer.hpp | 1 + .../duckdb/optimizer/window_rewriter.hpp | 28 ++++ src/optimizer/CMakeLists.txt | 3 +- .../join_filter_pushdown_optimizer.cpp | 43 ++++++ src/optimizer/optimizer.cpp | 7 + src/optimizer/window_rewriter.cpp | 144 ++++++++++++++++++ .../expression/bind_columnref_expression.cpp | 6 + test/optimizer/row_number_virtual_column.test | 120 ++++++++++----- 14 files changed, 329 insertions(+), 48 deletions(-) create mode 100644 src/include/duckdb/optimizer/window_rewriter.hpp create mode 100644 src/optimizer/window_rewriter.cpp diff --git a/src/common/enum_util.cpp b/src/common/enum_util.cpp index b6bdfecfd4f2..f81731d2e915 100644 --- a/src/common/enum_util.cpp +++ b/src/common/enum_util.cpp @@ -2925,18 +2925,19 @@ const StringUtil::EnumStringLiteral *GetMetricsTypeValues() { { static_cast(MetricsType::OPTIMIZER_CTE_INLINING), "OPTIMIZER_CTE_INLINING" }, { static_cast(MetricsType::OPTIMIZER_COMMON_SUBPLAN), "OPTIMIZER_COMMON_SUBPLAN" }, { static_cast(MetricsType::OPTIMIZER_JOIN_ELIMINATION), "OPTIMIZER_JOIN_ELIMINATION" } + { static_cast(MetricsType::OPTIMIZER_WINDOW_REWRITER), "OPTIMIZER_WINDOW_REWRITER" } }; return values; } template<> const char* EnumUtil::ToChars(MetricsType value) { - return StringUtil::EnumToString(GetMetricsTypeValues(), 64, "MetricsType", static_cast(value)); + return StringUtil::EnumToString(GetMetricsTypeValues(), 65, "MetricsType", static_cast(value)); } template<> MetricsType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetMetricsTypeValues(), 64, "MetricsType", value)); + return static_cast(StringUtil::StringToEnum(GetMetricsTypeValues(), 65, "MetricsType", value)); } const StringUtil::EnumStringLiteral *GetMultiFileColumnMappingModeValues() { @@ -3172,18 +3173,19 @@ const StringUtil::EnumStringLiteral *GetOptimizerTypeValues() { { static_cast(OptimizerType::CTE_INLINING), "CTE_INLINING" }, { static_cast(OptimizerType::COMMON_SUBPLAN), "COMMON_SUBPLAN" }, { static_cast(OptimizerType::JOIN_ELIMINATION), "JOIN_ELIMINATION" } + { static_cast(OptimizerType::WINDOW_REWRITER), "WINDOW_REWRITER" } }; return values; } template<> const char* EnumUtil::ToChars(OptimizerType value) { - return StringUtil::EnumToString(GetOptimizerTypeValues(), 32, "OptimizerType", static_cast(value)); + return StringUtil::EnumToString(GetOptimizerTypeValues(), 33, "OptimizerType", static_cast(value)); } template<> OptimizerType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetOptimizerTypeValues(), 32, "OptimizerType", value)); + return static_cast(StringUtil::StringToEnum(GetOptimizerTypeValues(), 33, "OptimizerType", value)); } const StringUtil::EnumStringLiteral *GetOrderByNullTypeValues() { diff --git a/src/common/enums/metric_type.cpp b/src/common/enums/metric_type.cpp index c883a363cb11..44f50ae963e8 100644 --- a/src/common/enums/metric_type.cpp +++ b/src/common/enums/metric_type.cpp @@ -44,6 +44,7 @@ profiler_settings_t MetricsUtils::GetOptimizerMetrics() { MetricsType::OPTIMIZER_CTE_INLINING, MetricsType::OPTIMIZER_COMMON_SUBPLAN, MetricsType::OPTIMIZER_JOIN_ELIMINATION, + MetricsType::OPTIMIZER_WINDOW_REWRITER, }; } @@ -124,6 +125,8 @@ MetricsType MetricsUtils::GetOptimizerMetricByType(OptimizerType type) { return MetricsType::OPTIMIZER_COMMON_SUBPLAN; case OptimizerType::JOIN_ELIMINATION: return MetricsType::OPTIMIZER_JOIN_ELIMINATION; + case OptimizerType::WINDOW_REWRITER: + return MetricsType::OPTIMIZER_WINDOW_REWRITER; default: throw InternalException("OptimizerType %s cannot be converted to a MetricsType", EnumUtil::ToString(type)); }; @@ -193,6 +196,8 @@ OptimizerType MetricsUtils::GetOptimizerTypeByMetric(MetricsType type) { return OptimizerType::COMMON_SUBPLAN; case MetricsType::OPTIMIZER_JOIN_ELIMINATION: return OptimizerType::JOIN_ELIMINATION; + case MetricsType::OPTIMIZER_WINDOW_REWRITER: + return OptimizerType::WINDOW_REWRITER; default: return OptimizerType::INVALID; }; @@ -231,6 +236,7 @@ bool MetricsUtils::IsOptimizerMetric(MetricsType type) { case MetricsType::OPTIMIZER_CTE_INLINING: case MetricsType::OPTIMIZER_COMMON_SUBPLAN: case MetricsType::OPTIMIZER_JOIN_ELIMINATION: + case MetricsType::OPTIMIZER_WINDOW_REWRITER: return true; default: return false; diff --git a/src/common/enums/optimizer_type.cpp b/src/common/enums/optimizer_type.cpp index 0e410bfa1238..a25d3a6ee580 100644 --- a/src/common/enums/optimizer_type.cpp +++ b/src/common/enums/optimizer_type.cpp @@ -44,6 +44,7 @@ static const DefaultOptimizerType internal_optimizer_types[] = { {"cte_inlining", OptimizerType::CTE_INLINING}, {"common_subplan", OptimizerType::COMMON_SUBPLAN}, {"join_elimination", OptimizerType::JOIN_ELIMINATION}, +{"window_rewriter", OptimizerType::WINDOW_REWRITER}, {nullptr, OptimizerType::INVALID}}; string OptimizerTypeToString(OptimizerType type) { diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index ad78c8767c18..c67bf1de65ef 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -102,6 +102,8 @@ class TableScanGlobalState : public GlobalTableFunctionState { vector row_number_offsets; //! row_number column index idx_t row_number_col_index = DConstants::INVALID_INDEX; + //! + mutex global_state_mutex; public: virtual unique_ptr InitLocalState(ExecutionContext &context, @@ -302,6 +304,7 @@ class DuckTableScanState : public TableScanGlobalState { idx_t row_group_index = l_state.scan_state.table_state.batch_index - 1; D_ASSERT(row_group_index < global_state.row_number_offsets.size()); + std::lock_guard lock(global_state_mutex); idx_t base = global_state.row_number_offsets[row_group_index] + l_state.row_number_count; for (idx_t i = 0; i < count; i++) { @@ -316,6 +319,7 @@ class DuckTableScanState : public TableScanGlobalState { if (!next) { return; } + // One thread is assigned more than one batches. When we are done with one batch, we reset the counter. l_state.row_number_count = 0; } while (true); } @@ -358,7 +362,7 @@ static unique_ptr TableScanInitLocal(ExecutionContext & } unique_ptr DuckTableScanInitGlobal(ClientContext &context, TableFunctionInitInput &input, - DataTable &storage, const TableScanBindData &bind_data) { + DataTable &storage, const TableScanBindData &bind_data) { auto g_state = make_uniq(context, input.bind_data.get()); if (bind_data.order_options) { g_state->state.scan_state.reorderer = make_uniq(*bind_data.order_options); diff --git a/src/include/duckdb/common/enums/metric_type.hpp b/src/include/duckdb/common/enums/metric_type.hpp index 182b8e11f5fe..0d4aeca4ee7c 100644 --- a/src/include/duckdb/common/enums/metric_type.hpp +++ b/src/include/duckdb/common/enums/metric_type.hpp @@ -84,6 +84,7 @@ enum class MetricsType : uint8_t { OPTIMIZER_CTE_INLINING, OPTIMIZER_COMMON_SUBPLAN, OPTIMIZER_JOIN_ELIMINATION, + OPTIMIZER_WINDOW_REWRITER, }; struct MetricsTypeHashFunction { diff --git a/src/include/duckdb/common/enums/optimizer_type.hpp b/src/include/duckdb/common/enums/optimizer_type.hpp index dbae1fac9a36..17fe75d6b7e9 100644 --- a/src/include/duckdb/common/enums/optimizer_type.hpp +++ b/src/include/duckdb/common/enums/optimizer_type.hpp @@ -46,6 +46,7 @@ enum class OptimizerType : uint32_t { CTE_INLINING, COMMON_SUBPLAN, JOIN_ELIMINATION + WINDOW_REWRITER, }; string OptimizerTypeToString(OptimizerType type); diff --git a/src/include/duckdb/optimizer/join_filter_pushdown_optimizer.hpp b/src/include/duckdb/optimizer/join_filter_pushdown_optimizer.hpp index aef9ba05576f..7d3fa349a2e6 100644 --- a/src/include/duckdb/optimizer/join_filter_pushdown_optimizer.hpp +++ b/src/include/duckdb/optimizer/join_filter_pushdown_optimizer.hpp @@ -26,6 +26,7 @@ class JoinFilterPushdownOptimizer : public LogicalOperatorVisitor { void VisitOperator(LogicalOperator &op) override; static void GetPushdownFilterTargets(LogicalOperator &op, vector columns, vector &targets); + bool CanOptimize(LogicalOperator &op); private: void GenerateJoinFilters(LogicalComparisonJoin &join); diff --git a/src/include/duckdb/optimizer/window_rewriter.hpp b/src/include/duckdb/optimizer/window_rewriter.hpp new file mode 100644 index 000000000000..fb08b8e64f57 --- /dev/null +++ b/src/include/duckdb/optimizer/window_rewriter.hpp @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/optimizer/window_rewriter.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once +#include "duckdb/common/unique_ptr.hpp" +#include "duckdb/planner/logical_operator.hpp" + +namespace duckdb { + +class WindowRewriter : public BaseColumnPruner { +public: + explicit WindowRewriter(Optimizer &optimizer); + unique_ptr Optimize(unique_ptr op); + unique_ptr OptimizeInternal(unique_ptr op, ColumnBindingReplacer &replacer); + bool CanOptimize(LogicalOperator &op); + unique_ptr RewriteGet(unique_ptr op, ColumnBindingReplacer &replacer); + +private: + Optimizer &optimizer; + bool lhs_window; +}; + +} // namespace duckdb diff --git a/src/optimizer/CMakeLists.txt b/src/optimizer/CMakeLists.txt index d9938a65ef3b..7afe57056374 100644 --- a/src/optimizer/CMakeLists.txt +++ b/src/optimizer/CMakeLists.txt @@ -40,7 +40,8 @@ add_library_unity( topn_window_elimination.cpp unnest_rewriter.cpp sampling_pushdown.cpp - sum_rewriter.cpp) + sum_rewriter.cpp + window_rewriter.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/optimizer/join_filter_pushdown_optimizer.cpp b/src/optimizer/join_filter_pushdown_optimizer.cpp index 02d6c2e45c45..b78914ec6a8a 100644 --- a/src/optimizer/join_filter_pushdown_optimizer.cpp +++ b/src/optimizer/join_filter_pushdown_optimizer.cpp @@ -256,9 +256,52 @@ void JoinFilterPushdownOptimizer::GenerateJoinFilters(LogicalComparisonJoin &joi join.filter_pushdown = std::move(pushdown_info); } +bool JoinFilterPushdownOptimizer::CanOptimize(LogicalOperator &op) { + // Check if any child scan outputs a ROW_NUMBER column. + switch (op.type) { + case LogicalOperatorType::LOGICAL_GET: { + auto &get = op.Cast(); + // Check the column IDs projected by the LogicalGet. If the scan outputs row_number column we skip pushdown + for (auto &col_id : get.GetColumnIds()) { + if (col_id.IsRowNumberColumn()) { + return false; + } + } + break; + } + case LogicalOperatorType::LOGICAL_PROJECTION: + case LogicalOperatorType::LOGICAL_FILTER: + case LogicalOperatorType::LOGICAL_ORDER_BY: + case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: + case LogicalOperatorType::LOGICAL_LIMIT: + case LogicalOperatorType::LOGICAL_WINDOW: + case LogicalOperatorType::LOGICAL_TOP_N: { + // Recurse into children + for (auto &child : op.children) { + if (!CanOptimize(*child)) { + return false; + } + } + break; + } + default: + break; + } + return true; +} + void JoinFilterPushdownOptimizer::VisitOperator(LogicalOperator &op) { if (op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { // comparison join - try to generate join filters (if possible) + + // If LHS is projecting row_numbers we skip this optimization. Row_numbers are assigned during scan, hence we + // can scan different things in each end of the join. + auto &join_op = op.Cast(); + auto &lhs = *join_op.children[0]; + if (!CanOptimize(lhs)) { + return; + } + GenerateJoinFilters(op.Cast()); } LogicalOperatorVisitor::VisitOperator(op); diff --git a/src/optimizer/optimizer.cpp b/src/optimizer/optimizer.cpp index d99121716bc0..37f8910a08f6 100644 --- a/src/optimizer/optimizer.cpp +++ b/src/optimizer/optimizer.cpp @@ -37,6 +37,7 @@ #include "duckdb/optimizer/unnest_rewriter.hpp" #include "duckdb/optimizer/late_materialization.hpp" #include "duckdb/optimizer/common_subplan_optimizer.hpp" +#include "duckdb/optimizer/window_rewriter.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/planner.hpp" @@ -252,6 +253,12 @@ void Optimizer::RunBuiltInOptimizers() { plan = sampling_pushdown.Optimize(std::move(plan)); }); + // Rewrite window functions to emit row_numbers in parallel + RunOptimizer(OptimizerType::WINDOW_REWRITER, [&]() { + WindowRewriter window_rewriter(*this); + plan = window_rewriter.Optimize(std::move(plan)); + }); + // transform ORDER BY + LIMIT to TopN RunOptimizer(OptimizerType::TOP_N, [&]() { TopN topn(context); diff --git a/src/optimizer/window_rewriter.cpp b/src/optimizer/window_rewriter.cpp new file mode 100644 index 000000000000..30380ea3c0c0 --- /dev/null +++ b/src/optimizer/window_rewriter.cpp @@ -0,0 +1,144 @@ +#include "duckdb/optimizer/window_rewriter.hpp" + +namespace duckdb { + +WindowRewriter::WindowRewriter(Optimizer &optimizer) : optimizer(optimizer), lhs_window(false) { +} + +bool WindowRewriter::CanOptimize(LogicalOperator &op) { + // FIXME Not the most elegant way + if (op.type == LogicalOperatorType::LOGICAL_JOIN || op.type == LogicalOperatorType::LOGICAL_ANY_JOIN || + op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op.type == LogicalOperatorType::LOGICAL_ASOF_JOIN || + op.type == LogicalOperatorType::LOGICAL_JOIN) { + auto &join_op = op.Cast(); + auto *child = join_op.children[0].get(); + while (child) { + if (child->type == LogicalOperatorType::LOGICAL_WINDOW) { + lhs_window = true; + break; + } + child = child->children[0].get(); + } + } + + // If the operator is a projection and its child is a window, check if optimization is possible + if (op.type == LogicalOperatorType::LOGICAL_PROJECTION && + op.children[0]->type == LogicalOperatorType::LOGICAL_WINDOW && !lhs_window) { + + auto *child = op.children[0].get(); + auto &window = child->Cast(); + + // Only support ROW_NUMBER() OVER () with no PARTITION or ORDER BY + if (window.expressions.size() != 1) { + return false; + } + + auto &expression = window.expressions[0]; + if (expression->type != ExpressionType::WINDOW_ROW_NUMBER) { + return false; + } + + return true; + } + + return false; +} + +unique_ptr WindowRewriter::Optimize(unique_ptr op) { + + ColumnBindingReplacer replacer; + op = OptimizeInternal(std::move(op), replacer); + + if (!replacer.replacement_bindings.empty()) { + replacer.VisitOperator(*op); + } + + return op; +} + +unique_ptr WindowRewriter::RewriteGet(unique_ptr op, + ColumnBindingReplacer &replacer) { + auto &proj = op->Cast(); + auto &window = proj.children[0]->Cast(); + auto &child = window.children[0]; + + auto &get = child->Cast(); + + // Extend child LogicalGet output with virtual row_number column + auto column_ids = get.GetColumnIds(); + // FIXME I think it is safe to add the row_number virtual column last, but check! + idx_t row_number_index = column_ids.size(); + column_ids.emplace_back(ColumnIndex(COLUMN_IDENTIFIER_ROW_NUMBER)); + get.SetColumnIds(std::move(column_ids)); + get.types.push_back(LogicalType::BIGINT); + get.projection_ids.push_back(row_number_index); + + const auto child_bindings = get.GetColumnBindings(); + const auto child_types = get.types; + auto proj_index = child_bindings[0].table_index; + + // Replace WINDOW + PROJECTION with new projection + const auto old_window_bindings = proj.GetColumnBindings(); + + vector> expressions; + expressions.reserve(row_number_index + 1); + + // Copy all original child columns + for (idx_t i = 0; i < row_number_index; ++i) { + expressions.push_back(make_uniq(child_types[i], child_bindings[i])); + } + + // Add virtual row_number reference + auto row_number_binding = ColumnBinding(proj_index, row_number_index); + expressions.push_back(make_uniq(LogicalType::BIGINT, row_number_binding)); + + // New projection gets its own table index + auto new_proj_index = optimizer.binder.GenerateTableIndex(); + auto new_projection = make_uniq(new_proj_index, std::move(expressions)); + new_projection->children.push_back(std::move(child)); + + // Update column binding replacer mapping + const auto new_projection_bindings = new_projection->GetColumnBindings(); + D_ASSERT(new_projection_bindings.size() == old_window_bindings.size()); + for (idx_t i = 0; i < old_window_bindings.size(); i++) { + replacer.replacement_bindings.emplace_back(old_window_bindings[i], new_projection_bindings[i]); + } + + replacer.stop_operator = new_projection.get(); + return std::move(new_projection); +} + +unique_ptr WindowRewriter::OptimizeInternal(unique_ptr op, + ColumnBindingReplacer &replacer) { + + if (CanOptimize(*op)) { + auto &proj = op->Cast(); + auto &window = proj.children[0]->Cast(); + + auto &child = window.children[0]; + + switch (child->type) { + case LogicalOperatorType::LOGICAL_JOIN: + case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: + case LogicalOperatorType::LOGICAL_ANY_JOIN: + case LogicalOperatorType::LOGICAL_ASOF_JOIN: + case LogicalOperatorType::LOGICAL_DELIM_JOIN: + // FIXME implement RewriteJoins + break; + case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: + // FIXME implement RewriteAggregate + break; + case LogicalOperatorType::LOGICAL_GET: + return RewriteGet(std::move(op), replacer); + default: + break; + } + } + + // Recurse into children + for (auto &child : op->children) { + child = OptimizeInternal(std::move(child), replacer); + } + return op; +} +} // namespace duckdb diff --git a/src/planner/binder/expression/bind_columnref_expression.cpp b/src/planner/binder/expression/bind_columnref_expression.cpp index 0c0d982d5bf4..7035c0351e7e 100644 --- a/src/planner/binder/expression/bind_columnref_expression.cpp +++ b/src/planner/binder/expression/bind_columnref_expression.cpp @@ -55,6 +55,12 @@ unique_ptr ExpressionBinder::GetSQLValueFunction(const string } unique_ptr ExpressionBinder::QualifyColumnName(const string &column_name, ErrorData &error) { + // We don't want users to be able to access row_number. Only through the window rewriter optimizer + if (column_name == "row_number") { + auto similar_bindings = binder.bind_context.GetSimilarBindings(column_name); + error = ErrorData(BinderException::ColumnNotFound(column_name, similar_bindings)); + return nullptr; + } auto using_binding = binder.bind_context.GetUsingBinding(column_name); if (using_binding) { // we are referencing a USING column diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index e28a9dd2f842..a67e453996f6 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -3,70 +3,106 @@ # group: [optimizer] statement ok -pragma disable_optimizer; - -# statement ok -# pragma threads=2; +pragma enable_verification; statement ok -CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(500000); +CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(1300000); + +# Check that row_number is not accessible +statement error +SELECT a, row_number FROM foo; +---- +Binder Error: Referenced column "row_number" not found in FROM clause statement ok -delete from foo where a<10; +delete from foo where a < 5000; -query II -select a, row_number from foo limit 10; +query III +select a,rowid, row_number() over() as x from foo order by a desc limit 1; ---- -10 1 -11 2 -12 3 -13 4 -14 5 -15 6 -16 7 -17 8 -18 9 -19 10 - +1299999 1299999 1295000 query II -select a, row_number from foo where a >= 100000 and a < 200000 limit 1; +explain select a,rowid, row_number() over() as x from foo order by a desc limit 1; ---- -100000 99991 +physical_plan :.*WINDOW.* + +# Test that if the window function is on the LHS we do not rewrite +statement ok +CREATE TABLE test (a INTEGER, b INTEGER); + +statement ok +INSERT INTO test VALUES (11, 1), (12, 2), (13, 3); statement ok -delete from foo where a>1500 and a < 150000; +CREATE TABLE test2 (b INTEGER, c INTEGER); statement ok -delete from foo where a>1 and a < 1500; +INSERT INTO test2 VALUES (1, 10), (1, 20), (2, 30); query II -select a, row_number from foo limit 10; +explain SELECT t.row_num, t.a, t2.c FROM ( SELECT a, b, ROW_NUMBER() OVER () AS row_num FROM test) t JOIN test2 t2 ON t.b = t2.b; ---- -1500 1 -150000 2 -150001 3 -150002 4 -150003 5 -150004 6 -150005 7 -150006 8 -150007 9 -150008 10 +physical_plan :.*WINDOW.* statement ok -CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(500000); +select setseed(0.1); statement ok -delete from foo where a>150 and a < 1500; +create or replace table test as select (random()*10)::INT a, (random()*5)::INT b from range(10); -# query IIII -# select a,rowid, row_number, row_number() over() as x from foo order by a desc limit 1; -# ---- -# 129999 129999 198651 198651 +statement ok +create or replace table test2 as select (random()*100)::INT c, (random()*5)::INT b from range(10); + +statement ok +pragma enable_optimizer + +# Try with a window function on LHS query III -select a,rowid, row_number from foo order by a desc limit 1; +SELECT t.row_num, t.a, t2.c FROM ( SELECT a, b, ROW_NUMBER() OVER () AS row_num FROM test) t JOIN test2 t2 ON t.b = t2.b limit 10; +---- +1 3 1 +2 5 16 +4 8 16 +5 8 1 +7 2 45 +8 7 24 +9 10 1 +10 1 45 +1 3 31 +5 8 31 + + + +# =================== IGNORE =================== +# BUILD_SIDE_PROBE_SIDE changes the window from RHS to LHS + +# Try with a window function on RHS + +query IIII +SELECT t.a, t.b, t2.row_num, t2.c FROM test t JOIN ( SELECT b, c, ROW_NUMBER() OVER () AS row_num FROM test2) t2 ON t.b = t2.b; ---- -499999 499999 498651 +7 1 1 85 +8 3 2 16 +7 1 3 67 +10 2 4 69 +10 2 5 36 +10 2 6 33 +1 4 7 45 +10 2 8 31 +7 1 9 24 +10 2 10 1 +5 3 2 16 +8 2 4 69 +8 2 5 36 +8 2 6 33 +2 4 7 45 +8 2 8 31 +8 2 10 1 +3 2 4 69 +3 2 5 36 +3 2 6 33 +3 2 8 31 +3 2 10 1 \ No newline at end of file From 1c5b5cd4d1dd45f6a18c8123e1a5315b61131a67 Mon Sep 17 00:00:00 2001 From: DinosL Date: Wed, 5 Nov 2025 17:00:23 +0100 Subject: [PATCH 03/27] check if operator has children --- src/optimizer/window_rewriter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/optimizer/window_rewriter.cpp b/src/optimizer/window_rewriter.cpp index 30380ea3c0c0..e80339570f7b 100644 --- a/src/optimizer/window_rewriter.cpp +++ b/src/optimizer/window_rewriter.cpp @@ -9,7 +9,7 @@ bool WindowRewriter::CanOptimize(LogicalOperator &op) { // FIXME Not the most elegant way if (op.type == LogicalOperatorType::LOGICAL_JOIN || op.type == LogicalOperatorType::LOGICAL_ANY_JOIN || op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op.type == LogicalOperatorType::LOGICAL_ASOF_JOIN || - op.type == LogicalOperatorType::LOGICAL_JOIN) { + op.type == LogicalOperatorType::LOGICAL_JOIN && !op.children.empty()) { auto &join_op = op.Cast(); auto *child = join_op.children[0].get(); while (child) { From b0008d55bb82e83b07bf8b8dd2d5cf45e2d12b3c Mon Sep 17 00:00:00 2001 From: DinosL Date: Fri, 7 Nov 2025 10:30:03 +0100 Subject: [PATCH 04/27] fixed related to failling tests --- src/common/enum_util.cpp | 4 +- src/function/table/read_duckdb.cpp | 4 +- .../duckdb/common/enums/optimizer_type.hpp | 2 +- .../storage/table/row_number_column_data.hpp | 8 +- src/optimizer/window_rewriter.cpp | 90 +++++++++++++------ src/storage/table/row_group.cpp | 11 +-- src/storage/table/row_number_column_data.cpp | 27 ++++-- test/optimizer/row_number_virtual_column.test | 49 ++-------- test/sql/window/test_streaming_window.test | 2 +- 9 files changed, 106 insertions(+), 91 deletions(-) diff --git a/src/common/enum_util.cpp b/src/common/enum_util.cpp index f81731d2e915..009398a471ab 100644 --- a/src/common/enum_util.cpp +++ b/src/common/enum_util.cpp @@ -2924,7 +2924,7 @@ const StringUtil::EnumStringLiteral *GetMetricsTypeValues() { { static_cast(MetricsType::OPTIMIZER_LATE_MATERIALIZATION), "OPTIMIZER_LATE_MATERIALIZATION" }, { static_cast(MetricsType::OPTIMIZER_CTE_INLINING), "OPTIMIZER_CTE_INLINING" }, { static_cast(MetricsType::OPTIMIZER_COMMON_SUBPLAN), "OPTIMIZER_COMMON_SUBPLAN" }, - { static_cast(MetricsType::OPTIMIZER_JOIN_ELIMINATION), "OPTIMIZER_JOIN_ELIMINATION" } + { static_cast(MetricsType::OPTIMIZER_JOIN_ELIMINATION), "OPTIMIZER_JOIN_ELIMINATION" }, { static_cast(MetricsType::OPTIMIZER_WINDOW_REWRITER), "OPTIMIZER_WINDOW_REWRITER" } }; return values; @@ -3172,7 +3172,7 @@ const StringUtil::EnumStringLiteral *GetOptimizerTypeValues() { { static_cast(OptimizerType::LATE_MATERIALIZATION), "LATE_MATERIALIZATION" }, { static_cast(OptimizerType::CTE_INLINING), "CTE_INLINING" }, { static_cast(OptimizerType::COMMON_SUBPLAN), "COMMON_SUBPLAN" }, - { static_cast(OptimizerType::JOIN_ELIMINATION), "JOIN_ELIMINATION" } + { static_cast(OptimizerType::JOIN_ELIMINATION), "JOIN_ELIMINATION" }, { static_cast(OptimizerType::WINDOW_REWRITER), "WINDOW_REWRITER" } }; return values; diff --git a/src/function/table/read_duckdb.cpp b/src/function/table/read_duckdb.cpp index 4f8cfac66977..6ee3b67f2ee0 100644 --- a/src/function/table/read_duckdb.cpp +++ b/src/function/table/read_duckdb.cpp @@ -363,7 +363,7 @@ double DuckDBReader::GetProgressInFile(ClientContext &context) { } void DuckDBReader::AddVirtualColumn(column_t virtual_column_id) { - if (virtual_column_id != COLUMN_IDENTIFIER_ROW_ID) { + if (virtual_column_id != COLUMN_IDENTIFIER_ROW_ID && virtual_column_id != COLUMN_IDENTIFIER_ROW_NUMBER) { throw InternalException("Unsupported virtual column id %d for duckdb reader", virtual_column_id); } } @@ -498,6 +498,7 @@ FileGlobInput DuckDBMultiFileInfo::GetGlobInput() { void DuckDBMultiFileInfo::GetVirtualColumns(ClientContext &, MultiFileBindData &, virtual_column_map_t &result) { result.insert(make_pair(COLUMN_IDENTIFIER_ROW_ID, TableColumn("rowid", LogicalType::BIGINT))); + result.insert(make_pair(COLUMN_IDENTIFIER_ROW_NUMBER, TableColumn("row_number", LogicalType::BIGINT))); } void ReadDuckDBAddNamedParameters(TableFunction &table_function) { @@ -511,6 +512,7 @@ static vector DuckDBGetRowIdColumns(ClientContext &, optional_ptr result; result.emplace_back(MultiFileReader::COLUMN_IDENTIFIER_FILE_INDEX); result.emplace_back(COLUMN_IDENTIFIER_ROW_ID); + result.emplace_back(COLUMN_IDENTIFIER_ROW_NUMBER); return result; } diff --git a/src/include/duckdb/common/enums/optimizer_type.hpp b/src/include/duckdb/common/enums/optimizer_type.hpp index 17fe75d6b7e9..e79c63c81631 100644 --- a/src/include/duckdb/common/enums/optimizer_type.hpp +++ b/src/include/duckdb/common/enums/optimizer_type.hpp @@ -45,7 +45,7 @@ enum class OptimizerType : uint32_t { LATE_MATERIALIZATION, CTE_INLINING, COMMON_SUBPLAN, - JOIN_ELIMINATION + JOIN_ELIMINATION, WINDOW_REWRITER, }; diff --git a/src/include/duckdb/storage/table/row_number_column_data.hpp b/src/include/duckdb/storage/table/row_number_column_data.hpp index 5ffc86eafaea..aeabff2a6ceb 100644 --- a/src/include/duckdb/storage/table/row_number_column_data.hpp +++ b/src/include/duckdb/storage/table/row_number_column_data.hpp @@ -14,17 +14,17 @@ namespace duckdb { class RowNumberColumnData : public ColumnData { public: - RowNumberColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t start_row); + RowNumberColumnData(BlockManager &block_manager, DataTableInfo &info); public: void InitializeScan(ColumnScanState &state) override; void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override; idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, - idx_t scan_count) override; + idx_t scan_count) override; idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates, - idx_t scan_count) override; + idx_t scan_count) override; void ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result) override; idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count, idx_t result_offset = 0) override; }; -} // namespace duckdb +} // namespace duckdb \ No newline at end of file diff --git a/src/optimizer/window_rewriter.cpp b/src/optimizer/window_rewriter.cpp index e80339570f7b..240b232571c8 100644 --- a/src/optimizer/window_rewriter.cpp +++ b/src/optimizer/window_rewriter.cpp @@ -6,34 +6,49 @@ WindowRewriter::WindowRewriter(Optimizer &optimizer) : optimizer(optimizer), lhs } bool WindowRewriter::CanOptimize(LogicalOperator &op) { - // FIXME Not the most elegant way + + // Check for window on LHS of join if (op.type == LogicalOperatorType::LOGICAL_JOIN || op.type == LogicalOperatorType::LOGICAL_ANY_JOIN || - op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op.type == LogicalOperatorType::LOGICAL_ASOF_JOIN || - op.type == LogicalOperatorType::LOGICAL_JOIN && !op.children.empty()) { + op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op.type == LogicalOperatorType::LOGICAL_ASOF_JOIN) { + auto &join_op = op.Cast(); + if (join_op.children.empty()) { + return false; + } + auto *child = join_op.children[0].get(); while (child) { if (child->type == LogicalOperatorType::LOGICAL_WINDOW) { lhs_window = true; break; } + if (child->children.empty()) + break; child = child->children[0].get(); } } // If the operator is a projection and its child is a window, check if optimization is possible - if (op.type == LogicalOperatorType::LOGICAL_PROJECTION && - op.children[0]->type == LogicalOperatorType::LOGICAL_WINDOW && !lhs_window) { + if (op.type == LogicalOperatorType::LOGICAL_PROJECTION && !lhs_window) { + if (op.children.empty() || !op.children[0]) { + return false; + } + if (op.children[0]->type != LogicalOperatorType::LOGICAL_WINDOW) { + return false; + } auto *child = op.children[0].get(); auto &window = child->Cast(); - // Only support ROW_NUMBER() OVER () with no PARTITION or ORDER BY if (window.expressions.size() != 1) { return false; } auto &expression = window.expressions[0]; + auto &window_expr = expression->Cast(); + if (!window_expr.partitions.empty() || !window_expr.orders.empty()) { + return false; + } if (expression->type != ExpressionType::WINDOW_ROW_NUMBER) { return false; } @@ -63,45 +78,68 @@ unique_ptr WindowRewriter::RewriteGet(unique_ptrCast(); + if (get.function.name != "seq_scan") { + return op; + } + + // Find where in the projection the row_number column appears + idx_t row_number_index = 0; + for (idx_t i = 0; i < proj.expressions.size(); i++) { + auto &col = proj.expressions.at(i); + auto &col_ref = col->Cast(); + if (col_ref.binding.table_index == window.window_index) { + row_number_index = i; + break; + } + } // Extend child LogicalGet output with virtual row_number column auto column_ids = get.GetColumnIds(); - // FIXME I think it is safe to add the row_number virtual column last, but check! - idx_t row_number_index = column_ids.size(); + auto types = get.types; + auto projection_ids = get.projection_ids; + column_ids.emplace_back(ColumnIndex(COLUMN_IDENTIFIER_ROW_NUMBER)); + types.push_back(LogicalType::BIGINT); + projection_ids.push_back(column_ids.size() - 1); + get.SetColumnIds(std::move(column_ids)); - get.types.push_back(LogicalType::BIGINT); - get.projection_ids.push_back(row_number_index); + get.types = std::move(types); + get.projection_ids = std::move(projection_ids); const auto child_bindings = get.GetColumnBindings(); const auto child_types = get.types; - auto proj_index = child_bindings[0].table_index; - // Replace WINDOW + PROJECTION with new projection - const auto old_window_bindings = proj.GetColumnBindings(); + // Replace WINDOW + PROJECTION with new PROJECTION + const auto old_projection_bindings = proj.GetColumnBindings(); vector> expressions; - expressions.reserve(row_number_index + 1); - - // Copy all original child columns - for (idx_t i = 0; i < row_number_index; ++i) { - expressions.push_back(make_uniq(child_types[i], child_bindings[i])); + expressions.reserve(child_bindings.size()); + + // Build projection expressions in correct order + for (idx_t i = 0; i < proj.expressions.size(); ++i) { + if (i == row_number_index) { + // Add virtual ROW_NUMBER() reference at this position + auto &row_number_binding = child_bindings.back(); + expressions.push_back(make_uniq(LogicalType::BIGINT, row_number_binding)); + } else { + // Copy the existing projection + auto &col_ref = proj.expressions[i]->Cast(); + auto binding_index = col_ref.binding.column_index; + expressions.push_back( + make_uniq(child_types[binding_index], child_bindings[binding_index])); + } } - // Add virtual row_number reference - auto row_number_binding = ColumnBinding(proj_index, row_number_index); - expressions.push_back(make_uniq(LogicalType::BIGINT, row_number_binding)); - - // New projection gets its own table index + // Create the new projection auto new_proj_index = optimizer.binder.GenerateTableIndex(); auto new_projection = make_uniq(new_proj_index, std::move(expressions)); new_projection->children.push_back(std::move(child)); // Update column binding replacer mapping const auto new_projection_bindings = new_projection->GetColumnBindings(); - D_ASSERT(new_projection_bindings.size() == old_window_bindings.size()); - for (idx_t i = 0; i < old_window_bindings.size(); i++) { - replacer.replacement_bindings.emplace_back(old_window_bindings[i], new_projection_bindings[i]); + D_ASSERT(new_projection_bindings.size() == old_projection_bindings.size()); + for (idx_t i = 0; i < old_projection_bindings.size(); i++) { + replacer.replacement_bindings.emplace_back(old_projection_bindings[i], new_projection_bindings[i]); } replacer.stop_operator = new_projection.get(); diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 8c2f7eeed396..229dc7e931c5 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -27,14 +27,15 @@ namespace duckdb { RowGroup::RowGroup(RowGroupCollection &collection_p, idx_t count) - : SegmentBase(count), collection(collection_p), version_info(nullptr), + : SegmentBase(count), collection(collection_p), version_info(nullptr), deletes_is_loaded(false), allocation_size(0), row_id_is_loaded(false), row_number_is_loaded(false), has_changes(false) { Verify(); } RowGroup::RowGroup(RowGroupCollection &collection_p, RowGroupPointer pointer) : SegmentBase(pointer.tuple_count), collection(collection_p), version_info(nullptr), -allocation_size(0), row_id_is_loaded(false), row_number_is_loaded(false), has_changes(false) { + deletes_is_loaded(false), allocation_size(0), row_id_is_loaded(false), row_number_is_loaded(false), + has_changes(false) { // deserialize the columns if (pointer.data_pointers.size() != collection_p.GetTypes().size()) { throw IOException("Row group column count is unaligned with table column count. Corrupt file?"); @@ -126,7 +127,7 @@ ColumnData &RowGroup::GetRowNumberColumnData() { } lock_guard l(row_number_group_lock); if (!row_number_column_data) { - row_number_column_data = make_uniq(GetBlockManager(), GetTableInfo(), start); + row_number_column_data = make_uniq(GetBlockManager(), GetTableInfo()); row_number_column_data->count = count.load(); row_number_is_loaded = true; } @@ -416,7 +417,7 @@ unique_ptr RowGroup::RemoveColumn(RowGroupCollection &new_collection, D_ASSERT(removed_column < columns.size()); - auto row_group = make_uniq(new_collection, this->start, this->count); + auto row_group = make_uniq(new_collection, this->count); row_group->SetVersionInfo(GetOrCreateVersionInfoPtr()); // copy over all columns except for the removed one auto &cols = GetColumns(); @@ -1410,4 +1411,4 @@ void VersionDeleteState::Flush() { count = 0; } -} // namespace duckdb +} // namespace duckdb \ No newline at end of file diff --git a/src/storage/table/row_number_column_data.cpp b/src/storage/table/row_number_column_data.cpp index c2fb4efc64f4..d86babd37b6b 100644 --- a/src/storage/table/row_number_column_data.cpp +++ b/src/storage/table/row_number_column_data.cpp @@ -3,17 +3,26 @@ namespace duckdb { -RowNumberColumnData::RowNumberColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t start_row) - : ColumnData(block_manager, info, COLUMN_IDENTIFIER_ROW_NUMBER, start_row, LogicalType(LogicalTypeId::BIGINT), - nullptr) { +RowNumberColumnData::RowNumberColumnData(BlockManager &block_manager, DataTableInfo &info) + : ColumnData(block_manager, info, COLUMN_IDENTIFIER_ROW_NUMBER, LogicalType(LogicalTypeId::BIGINT), + ColumnDataType::MAIN_TABLE, nullptr) { } void RowNumberColumnData::InitializeScan(ColumnScanState &state) { - InitializeScanWithOffset(state, start); + InitializeScanWithOffset(state, 0); } void RowNumberColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) { - state.row_index = row_idx; + if (row_idx > count) { + throw InternalException("row_idx in InitializeScanWithOffset out of range"); + } + state.current = nullptr; + state.segment_tree = nullptr; + state.offset_in_column = row_idx; + state.internal_index = state.offset_in_column; + state.initialized = true; + state.scan_state.reset(); + state.last_offset = 0; } idx_t RowNumberColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, @@ -28,17 +37,17 @@ idx_t RowNumberColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &st void RowNumberColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result) { - D_ASSERT(this->start == row_group_start); result.Sequence(UnsafeNumericCast(1 + offset_in_row_group), 1, count); } idx_t RowNumberColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t count, idx_t result_offset) { + auto row_start = state.parent->row_group->row_start; if (result_offset != 0) { throw InternalException("RowNumberColumnData result_offset must be 0"); } - ScanCommittedRange(start, state.row_index - start, count, result); - state.row_index += count; + ScanCommittedRange(row_start, state.offset_in_column, count, result); + state.offset_in_column += count; return count; } -} // namespace duckdb +} // namespace duckdb \ No newline at end of file diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index a67e453996f6..3a77358bfa00 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -55,54 +55,19 @@ create or replace table test as select (random()*10)::INT a, (random()*5)::INT b statement ok create or replace table test2 as select (random()*100)::INT c, (random()*5)::INT b from range(10); -statement ok -pragma enable_optimizer - # Try with a window function on LHS -query III -SELECT t.row_num, t.a, t2.c FROM ( SELECT a, b, ROW_NUMBER() OVER () AS row_num FROM test) t JOIN test2 t2 ON t.b = t2.b limit 10; +query II +explain SELECT t.row_num, t.a, t2.c FROM ( SELECT a, b, ROW_NUMBER() OVER () AS row_num FROM test) t JOIN test2 t2 ON t.b = t2.b limit 10; ---- -1 3 1 -2 5 16 -4 8 16 -5 8 1 -7 2 45 -8 7 24 -9 10 1 -10 1 45 -1 3 31 -5 8 31 - - +physical_plan :.*WINDOW.* # =================== IGNORE =================== # BUILD_SIDE_PROBE_SIDE changes the window from RHS to LHS # Try with a window function on RHS -query IIII -SELECT t.a, t.b, t2.row_num, t2.c FROM test t JOIN ( SELECT b, c, ROW_NUMBER() OVER () AS row_num FROM test2) t2 ON t.b = t2.b; ----- -7 1 1 85 -8 3 2 16 -7 1 3 67 -10 2 4 69 -10 2 5 36 -10 2 6 33 -1 4 7 45 -10 2 8 31 -7 1 9 24 -10 2 10 1 -5 3 2 16 -8 2 4 69 -8 2 5 36 -8 2 6 33 -2 4 7 45 -8 2 8 31 -8 2 10 1 -3 2 4 69 -3 2 5 36 -3 2 6 33 -3 2 8 31 -3 2 10 1 \ No newline at end of file +# query II +# explain SELECT t.a, t.b, t2.row_num, t2.c FROM test t JOIN ( SELECT b, c, ROW_NUMBER() OVER () AS row_num FROM test2) t2 ON t.b = t2.b; +# ---- +# physical_plan :.*WINDOW.* \ No newline at end of file diff --git a/test/sql/window/test_streaming_window.test b/test/sql/window/test_streaming_window.test index 5f2955b60c64..a818cd4ef3e2 100644 --- a/test/sql/window/test_streaming_window.test +++ b/test/sql/window/test_streaming_window.test @@ -54,7 +54,7 @@ physical_plan :.*STREAMING_WINDOW.* query TT explain select row_number() over (), i, j from integers ---- -physical_plan :.*STREAMING_WINDOW.* +physical_plan :.*STREAMING_WINDOW.* query TTT select row_number() over (), i, j from integers From 289054411073ca7dd87608594d9def62b9e70d2d Mon Sep 17 00:00:00 2001 From: DinosL Date: Mon, 10 Nov 2025 12:57:03 +0100 Subject: [PATCH 05/27] Reduce complexity - only rewrite the basic case of window+seq_scan for now --- src/common/enum_util.cpp | 9 +-- .../join_filter_pushdown_optimizer.hpp | 1 - .../duckdb/optimizer/window_rewriter.hpp | 8 +- .../join_filter_pushdown_optimizer.cpp | 48 ++--------- src/optimizer/window_rewriter.cpp | 80 +++++++------------ test/optimizer/row_number_virtual_column.test | 40 +++------- 6 files changed, 52 insertions(+), 134 deletions(-) diff --git a/src/common/enum_util.cpp b/src/common/enum_util.cpp index 009398a471ab..9d46bec6435d 100644 --- a/src/common/enum_util.cpp +++ b/src/common/enum_util.cpp @@ -2924,8 +2924,8 @@ const StringUtil::EnumStringLiteral *GetMetricsTypeValues() { { static_cast(MetricsType::OPTIMIZER_LATE_MATERIALIZATION), "OPTIMIZER_LATE_MATERIALIZATION" }, { static_cast(MetricsType::OPTIMIZER_CTE_INLINING), "OPTIMIZER_CTE_INLINING" }, { static_cast(MetricsType::OPTIMIZER_COMMON_SUBPLAN), "OPTIMIZER_COMMON_SUBPLAN" }, - { static_cast(MetricsType::OPTIMIZER_JOIN_ELIMINATION), "OPTIMIZER_JOIN_ELIMINATION" }, - { static_cast(MetricsType::OPTIMIZER_WINDOW_REWRITER), "OPTIMIZER_WINDOW_REWRITER" } + { static_cast(MetricsType::OPTIMIZER_WINDOW_REWRITER), "OPTIMIZER_WINDOW_REWRITER" }, + { static_cast(MetricsType::OPTIMIZER_JOIN_ELIMINATION), "OPTIMIZER_JOIN_ELIMINATION" } }; return values; } @@ -3172,8 +3172,8 @@ const StringUtil::EnumStringLiteral *GetOptimizerTypeValues() { { static_cast(OptimizerType::LATE_MATERIALIZATION), "LATE_MATERIALIZATION" }, { static_cast(OptimizerType::CTE_INLINING), "CTE_INLINING" }, { static_cast(OptimizerType::COMMON_SUBPLAN), "COMMON_SUBPLAN" }, - { static_cast(OptimizerType::JOIN_ELIMINATION), "JOIN_ELIMINATION" }, - { static_cast(OptimizerType::WINDOW_REWRITER), "WINDOW_REWRITER" } + { static_cast(OptimizerType::WINDOW_REWRITER), "WINDOW_REWRITER" }, + { static_cast(OptimizerType::JOIN_ELIMINATION), "JOIN_ELIMINATION" } }; return values; } @@ -5244,4 +5244,3 @@ WindowMergeSortStage EnumUtil::FromString(const char *valu } } - diff --git a/src/include/duckdb/optimizer/join_filter_pushdown_optimizer.hpp b/src/include/duckdb/optimizer/join_filter_pushdown_optimizer.hpp index 7d3fa349a2e6..aef9ba05576f 100644 --- a/src/include/duckdb/optimizer/join_filter_pushdown_optimizer.hpp +++ b/src/include/duckdb/optimizer/join_filter_pushdown_optimizer.hpp @@ -26,7 +26,6 @@ class JoinFilterPushdownOptimizer : public LogicalOperatorVisitor { void VisitOperator(LogicalOperator &op) override; static void GetPushdownFilterTargets(LogicalOperator &op, vector columns, vector &targets); - bool CanOptimize(LogicalOperator &op); private: void GenerateJoinFilters(LogicalComparisonJoin &join); diff --git a/src/include/duckdb/optimizer/window_rewriter.hpp b/src/include/duckdb/optimizer/window_rewriter.hpp index fb08b8e64f57..e82c881d3669 100644 --- a/src/include/duckdb/optimizer/window_rewriter.hpp +++ b/src/include/duckdb/optimizer/window_rewriter.hpp @@ -9,20 +9,22 @@ #pragma once #include "duckdb/common/unique_ptr.hpp" #include "duckdb/planner/logical_operator.hpp" +#include "duckdb/optimizer/column_binding_replacer.hpp" namespace duckdb { -class WindowRewriter : public BaseColumnPruner { +class Optimizer; + +class WindowRewriter { public: explicit WindowRewriter(Optimizer &optimizer); unique_ptr Optimize(unique_ptr op); unique_ptr OptimizeInternal(unique_ptr op, ColumnBindingReplacer &replacer); bool CanOptimize(LogicalOperator &op); - unique_ptr RewriteGet(unique_ptr op, ColumnBindingReplacer &replacer); + unique_ptr Rewrite(unique_ptr op, ColumnBindingReplacer &replacer); private: Optimizer &optimizer; - bool lhs_window; }; } // namespace duckdb diff --git a/src/optimizer/join_filter_pushdown_optimizer.cpp b/src/optimizer/join_filter_pushdown_optimizer.cpp index b78914ec6a8a..bc279c454d0e 100644 --- a/src/optimizer/join_filter_pushdown_optimizer.cpp +++ b/src/optimizer/join_filter_pushdown_optimizer.cpp @@ -93,6 +93,11 @@ void JoinFilterPushdownOptimizer::GetPushdownFilterTargets(LogicalOperator &op, case LogicalOperatorType::LOGICAL_GET: { // found LogicalGet auto &get = probe_child.Cast(); + for (auto &col_id : get.GetColumnIds()) { + if (col_id.IsRowNumberColumn()) { + return; + } + } if (!get.function.filter_pushdown) { // filter pushdown is not supported - no need to consider this node return; @@ -256,52 +261,9 @@ void JoinFilterPushdownOptimizer::GenerateJoinFilters(LogicalComparisonJoin &joi join.filter_pushdown = std::move(pushdown_info); } -bool JoinFilterPushdownOptimizer::CanOptimize(LogicalOperator &op) { - // Check if any child scan outputs a ROW_NUMBER column. - switch (op.type) { - case LogicalOperatorType::LOGICAL_GET: { - auto &get = op.Cast(); - // Check the column IDs projected by the LogicalGet. If the scan outputs row_number column we skip pushdown - for (auto &col_id : get.GetColumnIds()) { - if (col_id.IsRowNumberColumn()) { - return false; - } - } - break; - } - case LogicalOperatorType::LOGICAL_PROJECTION: - case LogicalOperatorType::LOGICAL_FILTER: - case LogicalOperatorType::LOGICAL_ORDER_BY: - case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: - case LogicalOperatorType::LOGICAL_LIMIT: - case LogicalOperatorType::LOGICAL_WINDOW: - case LogicalOperatorType::LOGICAL_TOP_N: { - // Recurse into children - for (auto &child : op.children) { - if (!CanOptimize(*child)) { - return false; - } - } - break; - } - default: - break; - } - return true; -} - void JoinFilterPushdownOptimizer::VisitOperator(LogicalOperator &op) { if (op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { // comparison join - try to generate join filters (if possible) - - // If LHS is projecting row_numbers we skip this optimization. Row_numbers are assigned during scan, hence we - // can scan different things in each end of the join. - auto &join_op = op.Cast(); - auto &lhs = *join_op.children[0]; - if (!CanOptimize(lhs)) { - return; - } - GenerateJoinFilters(op.Cast()); } LogicalOperatorVisitor::VisitOperator(op); diff --git a/src/optimizer/window_rewriter.cpp b/src/optimizer/window_rewriter.cpp index 240b232571c8..853bfd32c99b 100644 --- a/src/optimizer/window_rewriter.cpp +++ b/src/optimizer/window_rewriter.cpp @@ -1,35 +1,19 @@ #include "duckdb/optimizer/window_rewriter.hpp" +#include "duckdb/planner/operator/logical_get.hpp" +#include "duckdb/planner/expression/bound_window_expression.hpp" +#include "duckdb/optimizer/optimizer.hpp" +#include "duckdb/optimizer/column_binding_replacer.hpp" +#include "duckdb/planner/binder.hpp" + namespace duckdb { -WindowRewriter::WindowRewriter(Optimizer &optimizer) : optimizer(optimizer), lhs_window(false) { +WindowRewriter::WindowRewriter(Optimizer &optimizer) : optimizer(optimizer) { } bool WindowRewriter::CanOptimize(LogicalOperator &op) { - - // Check for window on LHS of join - if (op.type == LogicalOperatorType::LOGICAL_JOIN || op.type == LogicalOperatorType::LOGICAL_ANY_JOIN || - op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op.type == LogicalOperatorType::LOGICAL_ASOF_JOIN) { - - auto &join_op = op.Cast(); - if (join_op.children.empty()) { - return false; - } - - auto *child = join_op.children[0].get(); - while (child) { - if (child->type == LogicalOperatorType::LOGICAL_WINDOW) { - lhs_window = true; - break; - } - if (child->children.empty()) - break; - child = child->children[0].get(); - } - } - // If the operator is a projection and its child is a window, check if optimization is possible - if (op.type == LogicalOperatorType::LOGICAL_PROJECTION && !lhs_window) { + if (op.type == LogicalOperatorType::LOGICAL_PROJECTION) { if (op.children.empty() || !op.children[0]) { return false; } @@ -46,6 +30,8 @@ bool WindowRewriter::CanOptimize(LogicalOperator &op) { auto &expression = window.expressions[0]; auto &window_expr = expression->Cast(); + + // Try to optimize simple window functions, without partitions or ordering if (!window_expr.partitions.empty() || !window_expr.orders.empty()) { return false; } @@ -53,6 +39,18 @@ bool WindowRewriter::CanOptimize(LogicalOperator &op) { return false; } + // Should be followed by a get + auto &window_ch = window.children[0]; + if (window_ch->type != LogicalOperatorType::LOGICAL_GET) { + return false; + } + + // and can only be a seq_scan + auto &get = window_ch->Cast(); + if (get.function.name != "seq_scan") { + return false; + } + return true; } @@ -71,21 +69,20 @@ unique_ptr WindowRewriter::Optimize(unique_ptr return op; } -unique_ptr WindowRewriter::RewriteGet(unique_ptr op, - ColumnBindingReplacer &replacer) { +unique_ptr WindowRewriter::Rewrite(unique_ptr op, ColumnBindingReplacer &replacer) { + D_ASSERT(op->type == LogicalOperatorType::LOGICAL_PROJECTION); auto &proj = op->Cast(); + D_ASSERT(proj.children[0]->type == LogicalOperatorType::LOGICAL_WINDOW); auto &window = proj.children[0]->Cast(); auto &child = window.children[0]; - + D_ASSERT(child->type == LogicalOperatorType::LOGICAL_GET); auto &get = child->Cast(); - if (get.function.name != "seq_scan") { - return op; - } // Find where in the projection the row_number column appears idx_t row_number_index = 0; for (idx_t i = 0; i < proj.expressions.size(); i++) { auto &col = proj.expressions.at(i); + D_ASSERT(col->type == ExpressionType::BOUND_COLUMN_REF); auto &col_ref = col->Cast(); if (col_ref.binding.table_index == window.window_index) { row_number_index = i; @@ -123,6 +120,7 @@ unique_ptr WindowRewriter::RewriteGet(unique_ptr(LogicalType::BIGINT, row_number_binding)); } else { // Copy the existing projection + D_ASSERT(proj.expressions[i]->type == ExpressionType::BOUND_COLUMN_REF); auto &col_ref = proj.expressions[i]->Cast(); auto binding_index = col_ref.binding.column_index; expressions.push_back( @@ -150,27 +148,7 @@ unique_ptr WindowRewriter::OptimizeInternal(unique_ptrCast(); - auto &window = proj.children[0]->Cast(); - - auto &child = window.children[0]; - - switch (child->type) { - case LogicalOperatorType::LOGICAL_JOIN: - case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: - case LogicalOperatorType::LOGICAL_ANY_JOIN: - case LogicalOperatorType::LOGICAL_ASOF_JOIN: - case LogicalOperatorType::LOGICAL_DELIM_JOIN: - // FIXME implement RewriteJoins - break; - case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: - // FIXME implement RewriteAggregate - break; - case LogicalOperatorType::LOGICAL_GET: - return RewriteGet(std::move(op), replacer); - default: - break; - } + return Rewrite(std::move(op), replacer); } // Recurse into children diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index 3a77358bfa00..35b38e4ec14e 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -14,6 +14,13 @@ SELECT a, row_number FROM foo; ---- Binder Error: Referenced column "row_number" not found in FROM clause +query I +SELECT a as row_number FROM foo limit 3; +---- +0 +1 +2 + statement ok delete from foo where a < 5000; @@ -27,25 +34,6 @@ explain select a,rowid, row_number() over() as x from foo order by a desc limit ---- physical_plan :.*WINDOW.* -# Test that if the window function is on the LHS we do not rewrite - -statement ok -CREATE TABLE test (a INTEGER, b INTEGER); - -statement ok -INSERT INTO test VALUES (11, 1), (12, 2), (13, 3); - -statement ok -CREATE TABLE test2 (b INTEGER, c INTEGER); - -statement ok -INSERT INTO test2 VALUES (1, 10), (1, 20), (2, 30); - -query II -explain SELECT t.row_num, t.a, t2.c FROM ( SELECT a, b, ROW_NUMBER() OVER () AS row_num FROM test) t JOIN test2 t2 ON t.b = t2.b; ----- -physical_plan :.*WINDOW.* - statement ok select setseed(0.1); @@ -55,19 +43,9 @@ create or replace table test as select (random()*10)::INT a, (random()*5)::INT b statement ok create or replace table test2 as select (random()*100)::INT c, (random()*5)::INT b from range(10); -# Try with a window function on LHS +# Try with a window function and a Join query II explain SELECT t.row_num, t.a, t2.c FROM ( SELECT a, b, ROW_NUMBER() OVER () AS row_num FROM test) t JOIN test2 t2 ON t.b = t2.b limit 10; ---- -physical_plan :.*WINDOW.* - -# =================== IGNORE =================== -# BUILD_SIDE_PROBE_SIDE changes the window from RHS to LHS - -# Try with a window function on RHS - -# query II -# explain SELECT t.a, t.b, t2.row_num, t2.c FROM test t JOIN ( SELECT b, c, ROW_NUMBER() OVER () AS row_num FROM test2) t2 ON t.b = t2.b; -# ---- -# physical_plan :.*WINDOW.* \ No newline at end of file +physical_plan :.*WINDOW.* \ No newline at end of file From 626b4d693ceba657d6794ac7ffd44d00fc4dc8a9 Mon Sep 17 00:00:00 2001 From: DinosL Date: Mon, 10 Nov 2025 15:11:18 +0100 Subject: [PATCH 06/27] row_number not accessible as a column --- src/include/duckdb/planner/table_binding.hpp | 4 ++++ src/optimizer/window_rewriter.cpp | 18 ++++++++---------- .../expression/bind_columnref_expression.cpp | 17 ++++++++++++++--- src/storage/table/row_group.cpp | 6 ++++-- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/include/duckdb/planner/table_binding.hpp b/src/include/duckdb/planner/table_binding.hpp index 836f52c417df..700408d6b664 100644 --- a/src/include/duckdb/planner/table_binding.hpp +++ b/src/include/duckdb/planner/table_binding.hpp @@ -57,6 +57,10 @@ struct Binding { static BindingAlias GetAlias(const string &explicit_alias, const StandardEntry &entry); static BindingAlias GetAlias(const string &explicit_alias, optional_ptr entry); + const case_insensitive_map_t &GetNameMap() const { + return name_map; + } + public: template TARGET &Cast() { diff --git a/src/optimizer/window_rewriter.cpp b/src/optimizer/window_rewriter.cpp index 853bfd32c99b..6ba5bc8c81d5 100644 --- a/src/optimizer/window_rewriter.cpp +++ b/src/optimizer/window_rewriter.cpp @@ -5,6 +5,8 @@ #include "duckdb/optimizer/optimizer.hpp" #include "duckdb/optimizer/column_binding_replacer.hpp" #include "duckdb/planner/binder.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" +#include "duckdb/planner/operator/logical_window.hpp" namespace duckdb { @@ -58,7 +60,6 @@ bool WindowRewriter::CanOptimize(LogicalOperator &op) { } unique_ptr WindowRewriter::Optimize(unique_ptr op) { - ColumnBindingReplacer replacer; op = OptimizeInternal(std::move(op), replacer); @@ -82,7 +83,8 @@ unique_ptr WindowRewriter::Rewrite(unique_ptr idx_t row_number_index = 0; for (idx_t i = 0; i < proj.expressions.size(); i++) { auto &col = proj.expressions.at(i); - D_ASSERT(col->type == ExpressionType::BOUND_COLUMN_REF); + if (col->type != ExpressionType::BOUND_COLUMN_REF) + continue; auto &col_ref = col->Cast(); if (col_ref.binding.table_index == window.window_index) { row_number_index = i; @@ -95,7 +97,7 @@ unique_ptr WindowRewriter::Rewrite(unique_ptr auto types = get.types; auto projection_ids = get.projection_ids; - column_ids.emplace_back(ColumnIndex(COLUMN_IDENTIFIER_ROW_NUMBER)); + column_ids.emplace_back(COLUMN_IDENTIFIER_ROW_NUMBER); types.push_back(LogicalType::BIGINT); projection_ids.push_back(column_ids.size() - 1); @@ -119,12 +121,9 @@ unique_ptr WindowRewriter::Rewrite(unique_ptr auto &row_number_binding = child_bindings.back(); expressions.push_back(make_uniq(LogicalType::BIGINT, row_number_binding)); } else { - // Copy the existing projection - D_ASSERT(proj.expressions[i]->type == ExpressionType::BOUND_COLUMN_REF); - auto &col_ref = proj.expressions[i]->Cast(); - auto binding_index = col_ref.binding.column_index; - expressions.push_back( - make_uniq(child_types[binding_index], child_bindings[binding_index])); + // Copy the rest + auto &expr = proj.expressions[i]; + expressions.push_back(expr->Copy()); } } @@ -146,7 +145,6 @@ unique_ptr WindowRewriter::Rewrite(unique_ptr unique_ptr WindowRewriter::OptimizeInternal(unique_ptr op, ColumnBindingReplacer &replacer) { - if (CanOptimize(*op)) { return Rewrite(std::move(op), replacer); } diff --git a/src/planner/binder/expression/bind_columnref_expression.cpp b/src/planner/binder/expression/bind_columnref_expression.cpp index 7035c0351e7e..8576e370f6f4 100644 --- a/src/planner/binder/expression/bind_columnref_expression.cpp +++ b/src/planner/binder/expression/bind_columnref_expression.cpp @@ -57,10 +57,21 @@ unique_ptr ExpressionBinder::GetSQLValueFunction(const string unique_ptr ExpressionBinder::QualifyColumnName(const string &column_name, ErrorData &error) { // We don't want users to be able to access row_number. Only through the window rewriter optimizer if (column_name == "row_number") { - auto similar_bindings = binder.bind_context.GetSimilarBindings(column_name); - error = ErrorData(BinderException::ColumnNotFound(column_name, similar_bindings)); - return nullptr; + auto tt = binder.bind_context.GetMatchingBinding(column_name); + auto test = tt->Cast(); + for (auto name : test.GetNameMap()) { + if (name.first == "row_number" && name.second == COLUMN_IDENTIFIER_ROW_NUMBER) { + auto similar_bindings = binder.bind_context.GetSimilarBindings(column_name); + error = ErrorData(BinderException::ColumnNotFound(column_name, similar_bindings)); + return nullptr; + } + } } + // if (column_name == "row_number") { + // auto similar_bindings = binder.bind_context.GetSimilarBindings(column_name); + // error = ErrorData(BinderException::ColumnNotFound(column_name, similar_bindings)); + // return nullptr; + // } auto using_binding = binder.bind_context.GetUsingBinding(column_name); if (using_binding) { // we are referencing a USING column diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 229dc7e931c5..146176904dd9 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -54,8 +54,9 @@ RowGroup::RowGroup(RowGroupCollection &collection_p, RowGroupPointer pointer) } RowGroup::RowGroup(RowGroupCollection &collection_p, PersistentRowGroupData &data) - : SegmentBase(data.count), collection(collection_p), version_info(nullptr), deletes_is_loaded(false), - allocation_size(0), row_id_is_loaded(false), row_number_is_loaded(false), has_changes(false) { + : SegmentBase(data.start, data.count), collection(collection_p), version_info(nullptr), + deletes_is_loaded(false), allocation_size(0), row_id_is_loaded(false), row_number_is_loaded(false), + has_changes(false) { auto &block_manager = GetBlockManager(); auto &info = GetTableInfo(); auto &types = collection.get().GetTypes(); @@ -264,6 +265,7 @@ void CollectionScanState::Initialize(const QueryContext &context, const vector Date: Fri, 14 Nov 2025 16:16:18 +0100 Subject: [PATCH 07/27] Simplify code and fix issues identified in failing test cases adapt RowNumberColumnData to new ColumnData --- benchmark/row_number.benchmark | 9 ++ src/common/enum_util.cpp | 1 + src/function/table/table_scan.cpp | 3 +- .../duckdb/optimizer/window_rewriter.hpp | 8 +- src/optimizer/optimizer.cpp | 2 +- src/optimizer/window_rewriter.cpp | 103 ++++++------------ .../expression/bind_columnref_expression.cpp | 12 +- src/storage/table/row_group.cpp | 6 +- test/optimizer/row_number_virtual_column.test | 17 ++- 9 files changed, 65 insertions(+), 96 deletions(-) create mode 100644 benchmark/row_number.benchmark diff --git a/benchmark/row_number.benchmark b/benchmark/row_number.benchmark new file mode 100644 index 000000000000..6d7ef6fc13d3 --- /dev/null +++ b/benchmark/row_number.benchmark @@ -0,0 +1,9 @@ +# name: benchmark/row_number.benchmark +# description: Check if the time to generate row_numbers after rewriting is way worse than a select * without rewriting. The reason is that we traverse the row groups to get the offset before actually handling the row groups and that can be slow +# group: [benchmark] + +load +CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(1300000000); + +run +SELECT row_number() over() from foo; diff --git a/src/common/enum_util.cpp b/src/common/enum_util.cpp index 9d46bec6435d..2a2b899e490d 100644 --- a/src/common/enum_util.cpp +++ b/src/common/enum_util.cpp @@ -5244,3 +5244,4 @@ WindowMergeSortStage EnumUtil::FromString(const char *valu } } + diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index c67bf1de65ef..3f7328228167 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -102,7 +102,7 @@ class TableScanGlobalState : public GlobalTableFunctionState { vector row_number_offsets; //! row_number column index idx_t row_number_col_index = DConstants::INVALID_INDEX; - //! + //! Synchronize changes to the global scan state. mutex global_state_mutex; public: @@ -303,7 +303,6 @@ class DuckTableScanState : public TableScanGlobalState { auto count = output.size(); idx_t row_group_index = l_state.scan_state.table_state.batch_index - 1; - D_ASSERT(row_group_index < global_state.row_number_offsets.size()); std::lock_guard lock(global_state_mutex); idx_t base = global_state.row_number_offsets[row_group_index] + l_state.row_number_count; diff --git a/src/include/duckdb/optimizer/window_rewriter.hpp b/src/include/duckdb/optimizer/window_rewriter.hpp index e82c881d3669..9ac129df1a90 100644 --- a/src/include/duckdb/optimizer/window_rewriter.hpp +++ b/src/include/duckdb/optimizer/window_rewriter.hpp @@ -17,14 +17,10 @@ class Optimizer; class WindowRewriter { public: - explicit WindowRewriter(Optimizer &optimizer); unique_ptr Optimize(unique_ptr op); - unique_ptr OptimizeInternal(unique_ptr op, ColumnBindingReplacer &replacer); + unique_ptr RewritePlan(unique_ptr op, ColumnBindingReplacer &replacer); bool CanOptimize(LogicalOperator &op); - unique_ptr Rewrite(unique_ptr op, ColumnBindingReplacer &replacer); - -private: - Optimizer &optimizer; + unique_ptr RewriteGet(unique_ptr op, ColumnBindingReplacer &replacer); }; } // namespace duckdb diff --git a/src/optimizer/optimizer.cpp b/src/optimizer/optimizer.cpp index 37f8910a08f6..066533195876 100644 --- a/src/optimizer/optimizer.cpp +++ b/src/optimizer/optimizer.cpp @@ -255,7 +255,7 @@ void Optimizer::RunBuiltInOptimizers() { // Rewrite window functions to emit row_numbers in parallel RunOptimizer(OptimizerType::WINDOW_REWRITER, [&]() { - WindowRewriter window_rewriter(*this); + WindowRewriter window_rewriter; plan = window_rewriter.Optimize(std::move(plan)); }); diff --git a/src/optimizer/window_rewriter.cpp b/src/optimizer/window_rewriter.cpp index 6ba5bc8c81d5..9a16050ded31 100644 --- a/src/optimizer/window_rewriter.cpp +++ b/src/optimizer/window_rewriter.cpp @@ -10,27 +10,14 @@ namespace duckdb { -WindowRewriter::WindowRewriter(Optimizer &optimizer) : optimizer(optimizer) { -} - bool WindowRewriter::CanOptimize(LogicalOperator &op) { - // If the operator is a projection and its child is a window, check if optimization is possible - if (op.type == LogicalOperatorType::LOGICAL_PROJECTION) { - if (op.children.empty() || !op.children[0]) { - return false; - } - if (op.children[0]->type != LogicalOperatorType::LOGICAL_WINDOW) { - return false; - } - - auto *child = op.children[0].get(); - auto &window = child->Cast(); - - if (window.expressions.size() != 1) { + // If the operator is a window function and its child is a get, check if optimization is possible + if (op.type == LogicalOperatorType::LOGICAL_WINDOW) { + if (op.expressions.size() != 1) { return false; } - auto &expression = window.expressions[0]; + auto &expression = op.expressions[0]; auto &window_expr = expression->Cast(); // Try to optimize simple window functions, without partitions or ordering @@ -42,7 +29,7 @@ bool WindowRewriter::CanOptimize(LogicalOperator &op) { } // Should be followed by a get - auto &window_ch = window.children[0]; + auto &window_ch = op.children[0]; if (window_ch->type != LogicalOperatorType::LOGICAL_GET) { return false; } @@ -61,42 +48,31 @@ bool WindowRewriter::CanOptimize(LogicalOperator &op) { unique_ptr WindowRewriter::Optimize(unique_ptr op) { ColumnBindingReplacer replacer; - op = OptimizeInternal(std::move(op), replacer); + LogicalOperator *root = op.get(); + op = RewritePlan(std::move(op), replacer); if (!replacer.replacement_bindings.empty()) { - replacer.VisitOperator(*op); + replacer.VisitOperator(*root); } return op; } -unique_ptr WindowRewriter::Rewrite(unique_ptr op, ColumnBindingReplacer &replacer) { - D_ASSERT(op->type == LogicalOperatorType::LOGICAL_PROJECTION); - auto &proj = op->Cast(); - D_ASSERT(proj.children[0]->type == LogicalOperatorType::LOGICAL_WINDOW); - auto &window = proj.children[0]->Cast(); +unique_ptr WindowRewriter::RewriteGet(unique_ptr op, + ColumnBindingReplacer &replacer) { + auto &window = op->Cast(); auto &child = window.children[0]; - D_ASSERT(child->type == LogicalOperatorType::LOGICAL_GET); auto &get = child->Cast(); - // Find where in the projection the row_number column appears - idx_t row_number_index = 0; - for (idx_t i = 0; i < proj.expressions.size(); i++) { - auto &col = proj.expressions.at(i); - if (col->type != ExpressionType::BOUND_COLUMN_REF) - continue; - auto &col_ref = col->Cast(); - if (col_ref.binding.table_index == window.window_index) { - row_number_index = i; - break; - } - } - // Extend child LogicalGet output with virtual row_number column auto column_ids = get.GetColumnIds(); auto types = get.types; auto projection_ids = get.projection_ids; + if (projection_ids.size() == 0) { + column_ids.clear(); + types.clear(); + } column_ids.emplace_back(COLUMN_IDENTIFIER_ROW_NUMBER); types.push_back(LogicalType::BIGINT); projection_ids.push_back(column_ids.size() - 1); @@ -106,52 +82,35 @@ unique_ptr WindowRewriter::Rewrite(unique_ptr get.projection_ids = std::move(projection_ids); const auto child_bindings = get.GetColumnBindings(); - const auto child_types = get.types; - // Replace WINDOW + PROJECTION with new PROJECTION - const auto old_projection_bindings = proj.GetColumnBindings(); + // Remove WINDOW and update bindings + const auto old_window_bindings = window.GetColumnBindings(); - vector> expressions; - expressions.reserve(child_bindings.size()); - - // Build projection expressions in correct order - for (idx_t i = 0; i < proj.expressions.size(); ++i) { - if (i == row_number_index) { - // Add virtual ROW_NUMBER() reference at this position - auto &row_number_binding = child_bindings.back(); - expressions.push_back(make_uniq(LogicalType::BIGINT, row_number_binding)); + for (idx_t i = 0; i < old_window_bindings.size(); i++) { + ColumnBinding target_binding; + if (i < child_bindings.size() - 1) { + // Map existing columns + target_binding = child_bindings[i]; } else { - // Copy the rest - auto &expr = proj.expressions[i]; - expressions.push_back(expr->Copy()); + // Map the virtual ROW_NUMBER column + target_binding = child_bindings.back(); } + replacer.replacement_bindings.emplace_back(old_window_bindings[i], target_binding); } - // Create the new projection - auto new_proj_index = optimizer.binder.GenerateTableIndex(); - auto new_projection = make_uniq(new_proj_index, std::move(expressions)); - new_projection->children.push_back(std::move(child)); - - // Update column binding replacer mapping - const auto new_projection_bindings = new_projection->GetColumnBindings(); - D_ASSERT(new_projection_bindings.size() == old_projection_bindings.size()); - for (idx_t i = 0; i < old_projection_bindings.size(); i++) { - replacer.replacement_bindings.emplace_back(old_projection_bindings[i], new_projection_bindings[i]); - } - - replacer.stop_operator = new_projection.get(); - return std::move(new_projection); + replacer.stop_operator = child.get(); + return std::move(window.children[0]); } -unique_ptr WindowRewriter::OptimizeInternal(unique_ptr op, - ColumnBindingReplacer &replacer) { +unique_ptr WindowRewriter::RewritePlan(unique_ptr op, + ColumnBindingReplacer &replacer) { if (CanOptimize(*op)) { - return Rewrite(std::move(op), replacer); + return RewriteGet(std::move(op), replacer); } // Recurse into children for (auto &child : op->children) { - child = OptimizeInternal(std::move(child), replacer); + child = RewritePlan(std::move(child), replacer); } return op; } diff --git a/src/planner/binder/expression/bind_columnref_expression.cpp b/src/planner/binder/expression/bind_columnref_expression.cpp index 8576e370f6f4..f701312d2368 100644 --- a/src/planner/binder/expression/bind_columnref_expression.cpp +++ b/src/planner/binder/expression/bind_columnref_expression.cpp @@ -57,9 +57,10 @@ unique_ptr ExpressionBinder::GetSQLValueFunction(const string unique_ptr ExpressionBinder::QualifyColumnName(const string &column_name, ErrorData &error) { // We don't want users to be able to access row_number. Only through the window rewriter optimizer if (column_name == "row_number") { - auto tt = binder.bind_context.GetMatchingBinding(column_name); - auto test = tt->Cast(); - for (auto name : test.GetNameMap()) { + auto matching_binding = binder.bind_context.GetMatchingBinding(column_name); + auto table_binding = matching_binding->Cast(); + + for (const auto &name : table_binding.GetNameMap()) { if (name.first == "row_number" && name.second == COLUMN_IDENTIFIER_ROW_NUMBER) { auto similar_bindings = binder.bind_context.GetSimilarBindings(column_name); error = ErrorData(BinderException::ColumnNotFound(column_name, similar_bindings)); @@ -67,11 +68,6 @@ unique_ptr ExpressionBinder::QualifyColumnName(const string &c } } } - // if (column_name == "row_number") { - // auto similar_bindings = binder.bind_context.GetSimilarBindings(column_name); - // error = ErrorData(BinderException::ColumnNotFound(column_name, similar_bindings)); - // return nullptr; - // } auto using_binding = binder.bind_context.GetUsingBinding(column_name); if (using_binding) { // we are referencing a USING column diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 146176904dd9..229dc7e931c5 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -54,9 +54,8 @@ RowGroup::RowGroup(RowGroupCollection &collection_p, RowGroupPointer pointer) } RowGroup::RowGroup(RowGroupCollection &collection_p, PersistentRowGroupData &data) - : SegmentBase(data.start, data.count), collection(collection_p), version_info(nullptr), - deletes_is_loaded(false), allocation_size(0), row_id_is_loaded(false), row_number_is_loaded(false), - has_changes(false) { + : SegmentBase(data.count), collection(collection_p), version_info(nullptr), deletes_is_loaded(false), + allocation_size(0), row_id_is_loaded(false), row_number_is_loaded(false), has_changes(false) { auto &block_manager = GetBlockManager(); auto &info = GetTableInfo(); auto &types = collection.get().GetTypes(); @@ -265,7 +264,6 @@ void CollectionScanState::Initialize(const QueryContext &context, const vector:.*WINDOW.* \ No newline at end of file +physical_plan :.*WINDOW.* + + +statement ok +create table t1 as select range::FLOAT a, range b, range c, range d from range (3); + +query I +SELECT CASE WHEN row_number() OVER () = 2 THEN 2222::BIGINT ELSE 1111::BIGINT END FROM t1; +---- +1111 +2222 +1111 \ No newline at end of file From f5c55a49f6bbce9fe87faee1d2b7a863363c05bf Mon Sep 17 00:00:00 2001 From: DinosL Date: Tue, 18 Nov 2025 19:01:35 +0100 Subject: [PATCH 08/27] Improvements and more tests --- src/common/enum_util.cpp | 8 +- .../duckdb/optimizer/window_rewriter.hpp | 2 + src/optimizer/optimizer.cpp | 12 +-- src/optimizer/window_rewriter.cpp | 92 +++++++++---------- test/optimizer/row_number_virtual_column.test | 54 +++++++++-- 5 files changed, 102 insertions(+), 66 deletions(-) diff --git a/src/common/enum_util.cpp b/src/common/enum_util.cpp index 2a2b899e490d..009398a471ab 100644 --- a/src/common/enum_util.cpp +++ b/src/common/enum_util.cpp @@ -2924,8 +2924,8 @@ const StringUtil::EnumStringLiteral *GetMetricsTypeValues() { { static_cast(MetricsType::OPTIMIZER_LATE_MATERIALIZATION), "OPTIMIZER_LATE_MATERIALIZATION" }, { static_cast(MetricsType::OPTIMIZER_CTE_INLINING), "OPTIMIZER_CTE_INLINING" }, { static_cast(MetricsType::OPTIMIZER_COMMON_SUBPLAN), "OPTIMIZER_COMMON_SUBPLAN" }, - { static_cast(MetricsType::OPTIMIZER_WINDOW_REWRITER), "OPTIMIZER_WINDOW_REWRITER" }, - { static_cast(MetricsType::OPTIMIZER_JOIN_ELIMINATION), "OPTIMIZER_JOIN_ELIMINATION" } + { static_cast(MetricsType::OPTIMIZER_JOIN_ELIMINATION), "OPTIMIZER_JOIN_ELIMINATION" }, + { static_cast(MetricsType::OPTIMIZER_WINDOW_REWRITER), "OPTIMIZER_WINDOW_REWRITER" } }; return values; } @@ -3172,8 +3172,8 @@ const StringUtil::EnumStringLiteral *GetOptimizerTypeValues() { { static_cast(OptimizerType::LATE_MATERIALIZATION), "LATE_MATERIALIZATION" }, { static_cast(OptimizerType::CTE_INLINING), "CTE_INLINING" }, { static_cast(OptimizerType::COMMON_SUBPLAN), "COMMON_SUBPLAN" }, - { static_cast(OptimizerType::WINDOW_REWRITER), "WINDOW_REWRITER" }, - { static_cast(OptimizerType::JOIN_ELIMINATION), "JOIN_ELIMINATION" } + { static_cast(OptimizerType::JOIN_ELIMINATION), "JOIN_ELIMINATION" }, + { static_cast(OptimizerType::WINDOW_REWRITER), "WINDOW_REWRITER" } }; return values; } diff --git a/src/include/duckdb/optimizer/window_rewriter.hpp b/src/include/duckdb/optimizer/window_rewriter.hpp index 9ac129df1a90..465fd1a408e9 100644 --- a/src/include/duckdb/optimizer/window_rewriter.hpp +++ b/src/include/duckdb/optimizer/window_rewriter.hpp @@ -21,6 +21,8 @@ class WindowRewriter { unique_ptr RewritePlan(unique_ptr op, ColumnBindingReplacer &replacer); bool CanOptimize(LogicalOperator &op); unique_ptr RewriteGet(unique_ptr op, ColumnBindingReplacer &replacer); + + ColumnBindingReplacer replacer; }; } // namespace duckdb diff --git a/src/optimizer/optimizer.cpp b/src/optimizer/optimizer.cpp index 066533195876..0117a08e6b8f 100644 --- a/src/optimizer/optimizer.cpp +++ b/src/optimizer/optimizer.cpp @@ -210,6 +210,12 @@ void Optimizer::RunBuiltInOptimizers() { unused.VisitOperator(*plan); }); + // Rewrite window functions to emit row_numbers in parallel + RunOptimizer(OptimizerType::WINDOW_REWRITER, [&]() { + WindowRewriter window_rewriter; + plan = window_rewriter.Optimize(std::move(plan)); + }); + // Remove duplicate groups from aggregates RunOptimizer(OptimizerType::DUPLICATE_GROUPS, [&]() { RemoveDuplicateGroups remove; @@ -253,12 +259,6 @@ void Optimizer::RunBuiltInOptimizers() { plan = sampling_pushdown.Optimize(std::move(plan)); }); - // Rewrite window functions to emit row_numbers in parallel - RunOptimizer(OptimizerType::WINDOW_REWRITER, [&]() { - WindowRewriter window_rewriter; - plan = window_rewriter.Optimize(std::move(plan)); - }); - // transform ORDER BY + LIMIT to TopN RunOptimizer(OptimizerType::TOP_N, [&]() { TopN topn(context); diff --git a/src/optimizer/window_rewriter.cpp b/src/optimizer/window_rewriter.cpp index 9a16050ded31..f6dca4be9dd8 100644 --- a/src/optimizer/window_rewriter.cpp +++ b/src/optimizer/window_rewriter.cpp @@ -12,42 +12,41 @@ namespace duckdb { bool WindowRewriter::CanOptimize(LogicalOperator &op) { // If the operator is a window function and its child is a get, check if optimization is possible - if (op.type == LogicalOperatorType::LOGICAL_WINDOW) { - if (op.expressions.size() != 1) { - return false; - } - - auto &expression = op.expressions[0]; - auto &window_expr = expression->Cast(); - - // Try to optimize simple window functions, without partitions or ordering - if (!window_expr.partitions.empty() || !window_expr.orders.empty()) { - return false; - } - if (expression->type != ExpressionType::WINDOW_ROW_NUMBER) { - return false; - } - - // Should be followed by a get - auto &window_ch = op.children[0]; - if (window_ch->type != LogicalOperatorType::LOGICAL_GET) { - return false; - } - - // and can only be a seq_scan - auto &get = window_ch->Cast(); - if (get.function.name != "seq_scan") { - return false; - } - - return true; + if (op.type != LogicalOperatorType::LOGICAL_WINDOW) { + return false; } - return false; + if (op.expressions.size() != 1) { + return false; + } + + auto &expression = op.expressions[0]; + auto &window_expr = expression->Cast(); + + // Try to optimize simple window functions, without partitions or ordering + if (!window_expr.partitions.empty() || !window_expr.orders.empty()) { + return false; + } + if (expression->type != ExpressionType::WINDOW_ROW_NUMBER) { + return false; + } + + // Should be followed by a get + auto &window_ch = op.children[0]; + if (window_ch->type != LogicalOperatorType::LOGICAL_GET) { + return false; + } + + // and can only be a seq_scan + auto &get = window_ch->Cast(); + if (get.function.name != "seq_scan") { + return false; + } + + return true; } unique_ptr WindowRewriter::Optimize(unique_ptr op) { - ColumnBindingReplacer replacer; LogicalOperator *root = op.get(); op = RewritePlan(std::move(op), replacer); @@ -66,37 +65,30 @@ unique_ptr WindowRewriter::RewriteGet(unique_ptr:.*WINDOW.* \ No newline at end of file From 8bf06304392411586b09d096c4f7d8586c25c260 Mon Sep 17 00:00:00 2001 From: DinosL Date: Wed, 19 Nov 2025 00:40:33 +0100 Subject: [PATCH 09/27] format fix --- src/common/enums/optimizer_type.cpp | 2 +- src/include/duckdb/storage/table/row_number_column_data.hpp | 6 +++--- src/storage/table/row_group.cpp | 2 +- src/storage/table/row_number_column_data.cpp | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/common/enums/optimizer_type.cpp b/src/common/enums/optimizer_type.cpp index a25d3a6ee580..2bb795adca63 100644 --- a/src/common/enums/optimizer_type.cpp +++ b/src/common/enums/optimizer_type.cpp @@ -44,7 +44,7 @@ static const DefaultOptimizerType internal_optimizer_types[] = { {"cte_inlining", OptimizerType::CTE_INLINING}, {"common_subplan", OptimizerType::COMMON_SUBPLAN}, {"join_elimination", OptimizerType::JOIN_ELIMINATION}, -{"window_rewriter", OptimizerType::WINDOW_REWRITER}, + {"window_rewriter", OptimizerType::WINDOW_REWRITER}, {nullptr, OptimizerType::INVALID}}; string OptimizerTypeToString(OptimizerType type) { diff --git a/src/include/duckdb/storage/table/row_number_column_data.hpp b/src/include/duckdb/storage/table/row_number_column_data.hpp index aeabff2a6ceb..9eb8918fe212 100644 --- a/src/include/duckdb/storage/table/row_number_column_data.hpp +++ b/src/include/duckdb/storage/table/row_number_column_data.hpp @@ -20,11 +20,11 @@ class RowNumberColumnData : public ColumnData { void InitializeScan(ColumnScanState &state) override; void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override; idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, - idx_t scan_count) override; + idx_t scan_count) override; idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates, - idx_t scan_count) override; + idx_t scan_count) override; void ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result) override; idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count, idx_t result_offset = 0) override; }; -} // namespace duckdb \ No newline at end of file +} // namespace duckdb diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 229dc7e931c5..0d9d92751ab8 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -1411,4 +1411,4 @@ void VersionDeleteState::Flush() { count = 0; } -} // namespace duckdb \ No newline at end of file +} // namespace duckdb diff --git a/src/storage/table/row_number_column_data.cpp b/src/storage/table/row_number_column_data.cpp index d86babd37b6b..6ac494ae8cc9 100644 --- a/src/storage/table/row_number_column_data.cpp +++ b/src/storage/table/row_number_column_data.cpp @@ -50,4 +50,4 @@ idx_t RowNumberColumnData::ScanCount(ColumnScanState &state, Vector &result, idx return count; } -} // namespace duckdb \ No newline at end of file +} // namespace duckdb From ffde20c9339cd80eae23116bad17b3ccc7ec288f Mon Sep 17 00:00:00 2001 From: DinosL Date: Wed, 19 Nov 2025 00:45:15 +0100 Subject: [PATCH 10/27] tidy up --- src/common/enums/metric_type.cpp | 12 ++++++------ src/include/duckdb/common/enums/metric_type.hpp | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/common/enums/metric_type.cpp b/src/common/enums/metric_type.cpp index 44f50ae963e8..3612ba96a706 100644 --- a/src/common/enums/metric_type.cpp +++ b/src/common/enums/metric_type.cpp @@ -44,7 +44,7 @@ profiler_settings_t MetricsUtils::GetOptimizerMetrics() { MetricsType::OPTIMIZER_CTE_INLINING, MetricsType::OPTIMIZER_COMMON_SUBPLAN, MetricsType::OPTIMIZER_JOIN_ELIMINATION, - MetricsType::OPTIMIZER_WINDOW_REWRITER, + MetricsType::OPTIMIZER_WINDOW_REWRITER, }; } @@ -125,8 +125,8 @@ MetricsType MetricsUtils::GetOptimizerMetricByType(OptimizerType type) { return MetricsType::OPTIMIZER_COMMON_SUBPLAN; case OptimizerType::JOIN_ELIMINATION: return MetricsType::OPTIMIZER_JOIN_ELIMINATION; - case OptimizerType::WINDOW_REWRITER: - return MetricsType::OPTIMIZER_WINDOW_REWRITER; + case OptimizerType::WINDOW_REWRITER: + return MetricsType::OPTIMIZER_WINDOW_REWRITER; default: throw InternalException("OptimizerType %s cannot be converted to a MetricsType", EnumUtil::ToString(type)); }; @@ -196,8 +196,8 @@ OptimizerType MetricsUtils::GetOptimizerTypeByMetric(MetricsType type) { return OptimizerType::COMMON_SUBPLAN; case MetricsType::OPTIMIZER_JOIN_ELIMINATION: return OptimizerType::JOIN_ELIMINATION; - case MetricsType::OPTIMIZER_WINDOW_REWRITER: - return OptimizerType::WINDOW_REWRITER; + case MetricsType::OPTIMIZER_WINDOW_REWRITER: + return OptimizerType::WINDOW_REWRITER; default: return OptimizerType::INVALID; }; @@ -236,7 +236,7 @@ bool MetricsUtils::IsOptimizerMetric(MetricsType type) { case MetricsType::OPTIMIZER_CTE_INLINING: case MetricsType::OPTIMIZER_COMMON_SUBPLAN: case MetricsType::OPTIMIZER_JOIN_ELIMINATION: - case MetricsType::OPTIMIZER_WINDOW_REWRITER: + case MetricsType::OPTIMIZER_WINDOW_REWRITER: return true; default: return false; diff --git a/src/include/duckdb/common/enums/metric_type.hpp b/src/include/duckdb/common/enums/metric_type.hpp index 0d4aeca4ee7c..a42482413f38 100644 --- a/src/include/duckdb/common/enums/metric_type.hpp +++ b/src/include/duckdb/common/enums/metric_type.hpp @@ -84,7 +84,7 @@ enum class MetricsType : uint8_t { OPTIMIZER_CTE_INLINING, OPTIMIZER_COMMON_SUBPLAN, OPTIMIZER_JOIN_ELIMINATION, - OPTIMIZER_WINDOW_REWRITER, + OPTIMIZER_WINDOW_REWRITER, }; struct MetricsTypeHashFunction { From c9182afda1875769c7ec62eb2ee90eba489df63e Mon Sep 17 00:00:00 2001 From: DinosL Date: Wed, 19 Nov 2025 14:02:15 +0100 Subject: [PATCH 11/27] moved benchmark to the correct folder --- benchmark/{ => micro/window}/row_number.benchmark | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename benchmark/{ => micro/window}/row_number.benchmark (90%) diff --git a/benchmark/row_number.benchmark b/benchmark/micro/window/row_number.benchmark similarity index 90% rename from benchmark/row_number.benchmark rename to benchmark/micro/window/row_number.benchmark index 6d7ef6fc13d3..6846e8910b55 100644 --- a/benchmark/row_number.benchmark +++ b/benchmark/micro/window/row_number.benchmark @@ -6,4 +6,4 @@ load CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(1300000000); run -SELECT row_number() over() from foo; +SELECT a, row_number() over() from foo; From 215783d27a3dba7ef222d604cd8111160faac8a3 Mon Sep 17 00:00:00 2001 From: DinosL Date: Mon, 24 Nov 2025 11:18:40 +0100 Subject: [PATCH 12/27] address review comments: moved virtual column to DuckTableEntry, row_number data to row_group, and window_rewriter to the end of the list to prevent filter pushdown after rewriting. --- benchmark/micro/window/row_number.benchmark | 4 +- .../catalog_entry/duck_table_entry.cpp | 13 +++++ .../catalog_entry/table_catalog_entry.cpp | 7 --- src/function/table/table_scan.cpp | 5 +- .../catalog_entry/duck_table_entry.hpp | 4 ++ .../catalog_entry/table_catalog_entry.hpp | 1 - .../duckdb/function/table/table_scan.hpp | 1 + .../duckdb/optimizer/window_rewriter.hpp | 3 +- .../storage/table/row_number_column_data.hpp | 30 ----------- .../join_filter_pushdown_optimizer.cpp | 5 -- src/optimizer/optimizer.cpp | 12 ++--- src/optimizer/window_rewriter.cpp | 2 +- .../expression/bind_columnref_expression.cpp | 13 ----- src/planner/table_binding.cpp | 4 ++ src/storage/table/CMakeLists.txt | 1 - src/storage/table/row_group.cpp | 17 ------ src/storage/table/row_number_column_data.cpp | 53 ------------------- test/optimizer/row_number_virtual_column.test | 14 +++++ 18 files changed, 51 insertions(+), 138 deletions(-) delete mode 100644 src/include/duckdb/storage/table/row_number_column_data.hpp delete mode 100644 src/storage/table/row_number_column_data.cpp diff --git a/benchmark/micro/window/row_number.benchmark b/benchmark/micro/window/row_number.benchmark index 6846e8910b55..24a731158dee 100644 --- a/benchmark/micro/window/row_number.benchmark +++ b/benchmark/micro/window/row_number.benchmark @@ -1,6 +1,6 @@ -# name: benchmark/row_number.benchmark +# name: benchmark/micro/window/row_number.benchmark # description: Check if the time to generate row_numbers after rewriting is way worse than a select * without rewriting. The reason is that we traverse the row groups to get the offset before actually handling the row groups and that can be slow -# group: [benchmark] +# group: [window] load CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(1300000000); diff --git a/src/catalog/catalog_entry/duck_table_entry.cpp b/src/catalog/catalog_entry/duck_table_entry.cpp index 216e28305353..82fc1640dd32 100644 --- a/src/catalog/catalog_entry/duck_table_entry.cpp +++ b/src/catalog/catalog_entry/duck_table_entry.cpp @@ -39,6 +39,19 @@ IndexStorageInfo GetIndexInfo(const IndexConstraintType type, const bool v1_0_0_ return index_info; } +virtual_column_map_t DuckTableEntry::GetVirtualColumns() const { + virtual_column_map_t virtual_columns; + virtual_columns.insert(make_pair(COLUMN_IDENTIFIER_ROW_ID, TableColumn("rowid", LogicalType::ROW_TYPE))); + virtual_columns.insert(make_pair(COLUMN_IDENTIFIER_ROW_NUMBER, TableColumn("row_number", LogicalType::ROW_TYPE))); + return virtual_columns; +} + +vector DuckTableEntry::GetRowNumberColumns() const { + vector result; + result.push_back(COLUMN_IDENTIFIER_ROW_NUMBER); + return result; +} + DuckTableEntry::DuckTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, BoundCreateTableInfo &info, shared_ptr inherited_storage) : TableCatalogEntry(catalog, schema, info.Base()), storage(std::move(inherited_storage)), diff --git a/src/catalog/catalog_entry/table_catalog_entry.cpp b/src/catalog/catalog_entry/table_catalog_entry.cpp index 7c5a456d630e..8582fa93c6ac 100644 --- a/src/catalog/catalog_entry/table_catalog_entry.cpp +++ b/src/catalog/catalog_entry/table_catalog_entry.cpp @@ -348,7 +348,6 @@ bool TableCatalogEntry::HasPrimaryKey() const { virtual_column_map_t TableCatalogEntry::GetVirtualColumns() const { virtual_column_map_t virtual_columns; virtual_columns.insert(make_pair(COLUMN_IDENTIFIER_ROW_ID, TableColumn("rowid", LogicalType::ROW_TYPE))); - virtual_columns.insert(make_pair(COLUMN_IDENTIFIER_ROW_NUMBER, TableColumn("row_number", LogicalType::ROW_TYPE))); return virtual_columns; } @@ -358,10 +357,4 @@ vector TableCatalogEntry::GetRowIdColumns() const { return result; } -vector TableCatalogEntry::GetRowNumberColumns() const { - vector result; - result.push_back(COLUMN_IDENTIFIER_ROW_NUMBER); - return result; -} - } // namespace duckdb diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index 3f7328228167..fec891666a5d 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -71,7 +71,7 @@ static StorageIndex GetStorageIndex(TableCatalogEntry &table, const ColumnIndex } if (column_id.IsRowNumberColumn()) { - return StorageIndex(COLUMN_IDENTIFIER_ROW_NUMBER); + return StorageIndex(); } // The index of the base ColumnIndex is equal to the physical column index in the table @@ -257,6 +257,9 @@ class DuckTableScanState : public TableScanGlobalState { vector storage_ids; for (auto &col : input.column_indexes) { + if (col.IsRowNumberColumn()) { + continue; + } storage_ids.push_back(GetStorageIndex(bind_data.table, col)); } diff --git a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp index 7f206a43d792..711088934c98 100644 --- a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp @@ -55,6 +55,10 @@ class DuckTableEntry : public TableCatalogEntry { return true; } + //! Returns the virtual columns for this table + virtual virtual_column_map_t GetVirtualColumns() const override; + virtual vector GetRowNumberColumns() const; + private: unique_ptr RenameColumn(ClientContext &context, RenameColumnInfo &info); unique_ptr RenameField(ClientContext &context, RenameFieldInfo &info); diff --git a/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp b/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp index 23ccf581f656..1e40319cf270 100644 --- a/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp @@ -128,7 +128,6 @@ class TableCatalogEntry : public StandardEntry { virtual virtual_column_map_t GetVirtualColumns() const; virtual vector GetRowIdColumns() const; - virtual vector GetRowNumberColumns() const; protected: //! A list of columns that are part of this table diff --git a/src/include/duckdb/function/table/table_scan.hpp b/src/include/duckdb/function/table/table_scan.hpp index 22407fff5f5a..4435cc70e7c8 100644 --- a/src/include/duckdb/function/table/table_scan.hpp +++ b/src/include/duckdb/function/table/table_scan.hpp @@ -30,6 +30,7 @@ struct TableScanBindData : public TableFunctionData { bool is_create_index; //! In what order to scan the row groups unique_ptr order_options; + idx_t row_number_projection_index = DConstants::INVALID_INDEX; public: bool Equals(const FunctionData &other_p) const override { diff --git a/src/include/duckdb/optimizer/window_rewriter.hpp b/src/include/duckdb/optimizer/window_rewriter.hpp index 465fd1a408e9..9c7e361f5425 100644 --- a/src/include/duckdb/optimizer/window_rewriter.hpp +++ b/src/include/duckdb/optimizer/window_rewriter.hpp @@ -19,9 +19,10 @@ class WindowRewriter { public: unique_ptr Optimize(unique_ptr op); unique_ptr RewritePlan(unique_ptr op, ColumnBindingReplacer &replacer); + +private: bool CanOptimize(LogicalOperator &op); unique_ptr RewriteGet(unique_ptr op, ColumnBindingReplacer &replacer); - ColumnBindingReplacer replacer; }; diff --git a/src/include/duckdb/storage/table/row_number_column_data.hpp b/src/include/duckdb/storage/table/row_number_column_data.hpp deleted file mode 100644 index 9eb8918fe212..000000000000 --- a/src/include/duckdb/storage/table/row_number_column_data.hpp +++ /dev/null @@ -1,30 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/storage/table/row_number_column_data.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/storage/table/column_data.hpp" - -namespace duckdb { - -class RowNumberColumnData : public ColumnData { -public: - RowNumberColumnData(BlockManager &block_manager, DataTableInfo &info); - -public: - void InitializeScan(ColumnScanState &state) override; - void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override; - idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, - idx_t scan_count) override; - idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates, - idx_t scan_count) override; - void ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result) override; - idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count, idx_t result_offset = 0) override; -}; - -} // namespace duckdb diff --git a/src/optimizer/join_filter_pushdown_optimizer.cpp b/src/optimizer/join_filter_pushdown_optimizer.cpp index bc279c454d0e..02d6c2e45c45 100644 --- a/src/optimizer/join_filter_pushdown_optimizer.cpp +++ b/src/optimizer/join_filter_pushdown_optimizer.cpp @@ -93,11 +93,6 @@ void JoinFilterPushdownOptimizer::GetPushdownFilterTargets(LogicalOperator &op, case LogicalOperatorType::LOGICAL_GET: { // found LogicalGet auto &get = probe_child.Cast(); - for (auto &col_id : get.GetColumnIds()) { - if (col_id.IsRowNumberColumn()) { - return; - } - } if (!get.function.filter_pushdown) { // filter pushdown is not supported - no need to consider this node return; diff --git a/src/optimizer/optimizer.cpp b/src/optimizer/optimizer.cpp index 0117a08e6b8f..3115e0bfd158 100644 --- a/src/optimizer/optimizer.cpp +++ b/src/optimizer/optimizer.cpp @@ -210,12 +210,6 @@ void Optimizer::RunBuiltInOptimizers() { unused.VisitOperator(*plan); }); - // Rewrite window functions to emit row_numbers in parallel - RunOptimizer(OptimizerType::WINDOW_REWRITER, [&]() { - WindowRewriter window_rewriter; - plan = window_rewriter.Optimize(std::move(plan)); - }); - // Remove duplicate groups from aggregates RunOptimizer(OptimizerType::DUPLICATE_GROUPS, [&]() { RemoveDuplicateGroups remove; @@ -308,6 +302,12 @@ void Optimizer::RunBuiltInOptimizers() { JoinFilterPushdownOptimizer join_filter_pushdown(*this); join_filter_pushdown.VisitOperator(*plan); }); + + // Rewrite window functions to emit row_numbers in parallel + RunOptimizer(OptimizerType::WINDOW_REWRITER, [&]() { + WindowRewriter window_rewriter; + plan = window_rewriter.Optimize(std::move(plan)); + }); } unique_ptr Optimizer::Optimize(unique_ptr plan_p) { diff --git a/src/optimizer/window_rewriter.cpp b/src/optimizer/window_rewriter.cpp index f6dca4be9dd8..dc69036d9ecd 100644 --- a/src/optimizer/window_rewriter.cpp +++ b/src/optimizer/window_rewriter.cpp @@ -39,7 +39,7 @@ bool WindowRewriter::CanOptimize(LogicalOperator &op) { // and can only be a seq_scan auto &get = window_ch->Cast(); - if (get.function.name != "seq_scan") { + if (get.virtual_columns.find(COLUMN_IDENTIFIER_ROW_NUMBER) == get.virtual_columns.end()) { return false; } diff --git a/src/planner/binder/expression/bind_columnref_expression.cpp b/src/planner/binder/expression/bind_columnref_expression.cpp index f701312d2368..0c0d982d5bf4 100644 --- a/src/planner/binder/expression/bind_columnref_expression.cpp +++ b/src/planner/binder/expression/bind_columnref_expression.cpp @@ -55,19 +55,6 @@ unique_ptr ExpressionBinder::GetSQLValueFunction(const string } unique_ptr ExpressionBinder::QualifyColumnName(const string &column_name, ErrorData &error) { - // We don't want users to be able to access row_number. Only through the window rewriter optimizer - if (column_name == "row_number") { - auto matching_binding = binder.bind_context.GetMatchingBinding(column_name); - auto table_binding = matching_binding->Cast(); - - for (const auto &name : table_binding.GetNameMap()) { - if (name.first == "row_number" && name.second == COLUMN_IDENTIFIER_ROW_NUMBER) { - auto similar_bindings = binder.bind_context.GetSimilarBindings(column_name); - error = ErrorData(BinderException::ColumnNotFound(column_name, similar_bindings)); - return nullptr; - } - } - } auto using_binding = binder.bind_context.GetUsingBinding(column_name); if (using_binding) { // we are referencing a USING column diff --git a/src/planner/table_binding.cpp b/src/planner/table_binding.cpp index c55d0be822b4..3820e9948cb7 100644 --- a/src/planner/table_binding.cpp +++ b/src/planner/table_binding.cpp @@ -161,6 +161,10 @@ TableBinding::TableBinding(const string &alias, vector types_p, vec // the empty column cannot be queried by the user continue; } + if (idx == COLUMN_IDENTIFIER_ROW_NUMBER) { + // the row_number column cannot be queried by the user + continue; + } if (name_map.find(name) == name_map.end()) { name_map[name] = idx; } diff --git a/src/storage/table/CMakeLists.txt b/src/storage/table/CMakeLists.txt index 2a3da5ca2859..62dd5e3481ba 100644 --- a/src/storage/table/CMakeLists.txt +++ b/src/storage/table/CMakeLists.txt @@ -12,7 +12,6 @@ add_library_unity( update_segment.cpp persistent_table_data.cpp row_id_column_data.cpp - row_number_column_data.cpp row_group.cpp row_group_collection.cpp row_version_manager.cpp diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 0d9d92751ab8..7f820917a93d 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -21,7 +21,6 @@ #include "duckdb/transaction/duck_transaction.hpp" #include "duckdb/transaction/duck_transaction_manager.hpp" #include "duckdb/storage/table/row_id_column_data.hpp" -#include "duckdb/storage/table/row_number_column_data.hpp" #include "duckdb/main/settings.hpp" namespace duckdb { @@ -121,19 +120,6 @@ ColumnData &RowGroup::GetRowIdColumnData() { return *row_id_column_data; } -ColumnData &RowGroup::GetRowNumberColumnData() { - if (row_number_is_loaded) { - return *row_number_column_data; - } - lock_guard l(row_number_group_lock); - if (!row_number_column_data) { - row_number_column_data = make_uniq(GetBlockManager(), GetTableInfo()); - row_number_column_data->count = count.load(); - row_number_is_loaded = true; - } - return *row_number_column_data; -} - ColumnData &RowGroup::GetColumn(const StorageIndex &c) { return GetColumn(c.GetPrimaryIndex()); } @@ -142,9 +128,6 @@ ColumnData &RowGroup::GetColumn(storage_t c) { if (c == COLUMN_IDENTIFIER_ROW_ID) { return GetRowIdColumnData(); } - if (c == COLUMN_IDENTIFIER_ROW_NUMBER) { - return GetRowNumberColumnData(); - } D_ASSERT(c < columns.size()); if (!is_loaded) { // not being lazy loaded diff --git a/src/storage/table/row_number_column_data.cpp b/src/storage/table/row_number_column_data.cpp deleted file mode 100644 index 6ac494ae8cc9..000000000000 --- a/src/storage/table/row_number_column_data.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include "duckdb/storage/table/scan_state.hpp" -#include "duckdb/storage/table/row_number_column_data.hpp" - -namespace duckdb { - -RowNumberColumnData::RowNumberColumnData(BlockManager &block_manager, DataTableInfo &info) - : ColumnData(block_manager, info, COLUMN_IDENTIFIER_ROW_NUMBER, LogicalType(LogicalTypeId::BIGINT), - ColumnDataType::MAIN_TABLE, nullptr) { -} - -void RowNumberColumnData::InitializeScan(ColumnScanState &state) { - InitializeScanWithOffset(state, 0); -} - -void RowNumberColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) { - if (row_idx > count) { - throw InternalException("row_idx in InitializeScanWithOffset out of range"); - } - state.current = nullptr; - state.segment_tree = nullptr; - state.offset_in_column = row_idx; - state.internal_index = state.offset_in_column; - state.initialized = true; - state.scan_state.reset(); - state.last_offset = 0; -} - -idx_t RowNumberColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, - idx_t scan_count) { - return ScanCommitted(vector_index, state, result, false, scan_count); -} - -idx_t RowNumberColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates, - idx_t scan_count) { - return ScanCount(state, result, scan_count, 0); -} - -void RowNumberColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, - Vector &result) { - result.Sequence(UnsafeNumericCast(1 + offset_in_row_group), 1, count); -} - -idx_t RowNumberColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t count, idx_t result_offset) { - auto row_start = state.parent->row_group->row_start; - if (result_offset != 0) { - throw InternalException("RowNumberColumnData result_offset must be 0"); - } - ScanCommittedRange(row_start, state.offset_in_column, count, result); - state.offset_in_column += count; - return count; -} - -} // namespace duckdb diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index 85a2fd3bf4b2..ad4d2cd442fd 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -101,4 +101,18 @@ query II EXPLAIN SELECT f.row_num AS foo_row, b.row_num AS bar_row, f.a, b.b FROM (SELECT row_number() OVER () AS row_num, a FROM foo) f JOIN (SELECT row_number() OVER () AS row_num, b FROM bar) b ON f.row_num = b.row_num; ---- +physical_plan :.*WINDOW.* + +statement ok +pragma disable_optimizer; + +statement ok +create table tbl as select range a from range(3); + +statement ok +pragma enable_optimizer; + +query II +EXPLAIN SELECT row_number + 1 FROM (SELECT row_number() OVER () AS row_number FROM tbl); +---- physical_plan :.*WINDOW.* \ No newline at end of file From ea82ce9e823c913e0b3009ac4d8d93d9447108bc Mon Sep 17 00:00:00 2001 From: DinosL Date: Tue, 25 Nov 2025 09:57:35 +0100 Subject: [PATCH 13/27] merge --- src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp index 711088934c98..1a36f11451c6 100644 --- a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp @@ -56,7 +56,7 @@ class DuckTableEntry : public TableCatalogEntry { } //! Returns the virtual columns for this table - virtual virtual_column_map_t GetVirtualColumns() const override; + virtual_column_map_t GetVirtualColumns() const override; virtual vector GetRowNumberColumns() const; private: From b340db196204020238ed60a01884a4ca53956a80 Mon Sep 17 00:00:00 2001 From: DinosL Date: Tue, 25 Nov 2025 10:01:39 +0100 Subject: [PATCH 14/27] improve row group prefix sum calculation for parallel row_group generation --- src/function/table/table_scan.cpp | 15 ++------------ .../storage/table/row_group_collection.hpp | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index 3da46106ff71..e41f880b1522 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -282,7 +282,6 @@ class DuckTableScanState : public TableScanGlobalState { void TableScanFunc(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) override { auto &l_state = data_p.local_state->Cast(); l_state.scan_state.options.force_fetch_row = ClientConfig::GetConfig(context).force_fetch_row; - auto &global_state = data_p.global_state->Cast(); do { if (context.interrupted) { @@ -306,8 +305,7 @@ class DuckTableScanState : public TableScanGlobalState { auto count = output.size(); idx_t row_group_index = l_state.scan_state.table_state.batch_index - 1; - std::lock_guard lock(global_state_mutex); - idx_t base = global_state.row_number_offsets[row_group_index] + l_state.row_number_count; + idx_t base = state.scan_state.collection->GetPrefixSum(row_group_index) + l_state.row_number_count; for (idx_t i = 0; i < count; i++) { row_number_data[i] = static_cast(base + i + 1); @@ -379,16 +377,7 @@ unique_ptr DuckTableScanInitGlobal(ClientContext &cont } } if (g_state->row_number_col_index != DConstants::INVALID_INDEX) { - idx_t current_offset = 0; - int64_t counter = 0; - auto row_groups = g_state->state.scan_state.collection; - auto row_group = row_groups->GetRowGroup(counter); - while (row_group) { - g_state->row_number_offsets.push_back(current_offset); - current_offset += row_group->GetCommittedRowCount(); - counter++; - row_group = row_groups->GetRowGroup(counter); - } + g_state->state.scan_state.collection->PrecomputePrefixSums(); } if (!input.CanRemoveFilterColumns()) { return std::move(g_state); diff --git a/src/include/duckdb/storage/table/row_group_collection.hpp b/src/include/duckdb/storage/table/row_group_collection.hpp index 7d32eb19ad46..1a55169ff23f 100644 --- a/src/include/duckdb/storage/table/row_group_collection.hpp +++ b/src/include/duckdb/storage/table/row_group_collection.hpp @@ -8,6 +8,7 @@ #pragma once +#include "duckdb/storage/table/row_group_segment_tree.hpp" #include "duckdb/storage/table/row_group.hpp" #include "duckdb/storage/table/segment_tree.hpp" #include "duckdb/storage/statistics/column_statistics.hpp" @@ -153,6 +154,25 @@ class RowGroupCollection { } void SetAppendRequiresNewRowGroup(); + std::vector prefix_sums; + + void PrecomputePrefixSums() { + idx_t cumulative = 0; + prefix_sums.resize(owned_row_groups->GetSegmentCount()); + + for (idx_t i = 0; i < owned_row_groups->GetSegmentCount(); ++i) { + cumulative += owned_row_groups.get()->GetSegment(i)->GetNode().GetCommittedRowCount(); + prefix_sums[i] = cumulative; + } + } + + idx_t GetPrefixSum(idx_t index) const { + if (index == 0) { + return 0; // No rows before the first row group + } + return prefix_sums[index - 1]; + } + private: optional_ptr> NextUpdateRowGroup(RowGroupSegmentTree &row_groups, row_t *ids, idx_t &pos, idx_t count) const; From 0bd09e6ebd289588b40818b0346bf520bf230b49 Mon Sep 17 00:00:00 2001 From: DinosL Date: Wed, 26 Nov 2025 10:30:11 +0100 Subject: [PATCH 15/27] remove unused field --- src/function/table/table_scan.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index e41f880b1522..d1b332e30060 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -98,8 +98,6 @@ class TableScanGlobalState : public GlobalTableFunctionState { vector projection_ids; //! The types of all scanned columns. vector scanned_types; - //! row_number offsets for each row group - vector row_number_offsets; //! row_number column index idx_t row_number_col_index = DConstants::INVALID_INDEX; //! Synchronize changes to the global scan state. From 4b6c2354f9e3e0d4caca698215a1424488d8a8e7 Mon Sep 17 00:00:00 2001 From: DinosL Date: Wed, 26 Nov 2025 12:21:59 +0100 Subject: [PATCH 16/27] tidy --- src/common/enum_util.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/enum_util.cpp b/src/common/enum_util.cpp index 1a15be56f1d1..af62338967e1 100644 --- a/src/common/enum_util.cpp +++ b/src/common/enum_util.cpp @@ -2935,12 +2935,12 @@ const StringUtil::EnumStringLiteral *GetMetricsTypeValues() { template<> const char* EnumUtil::ToChars(MetricsType value) { - return StringUtil::EnumToString(GetMetricsTypeValues(), 66, "MetricsType", static_cast(value)); + return StringUtil::EnumToString(GetMetricsTypeValues(), 67, "MetricsType", static_cast(value)); } template<> MetricsType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetMetricsTypeValues(), 66, "MetricsType", value)); + return static_cast(StringUtil::StringToEnum(GetMetricsTypeValues(), 67, "MetricsType", value)); } const StringUtil::EnumStringLiteral *GetMultiFileColumnMappingModeValues() { @@ -3184,12 +3184,12 @@ const StringUtil::EnumStringLiteral *GetOptimizerTypeValues() { template<> const char* EnumUtil::ToChars(OptimizerType value) { - return StringUtil::EnumToString(GetOptimizerTypeValues(), 33, "OptimizerType", static_cast(value)); + return StringUtil::EnumToString(GetOptimizerTypeValues(), 34, "OptimizerType", static_cast(value)); } template<> OptimizerType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetOptimizerTypeValues(), 33, "OptimizerType", value)); + return static_cast(StringUtil::StringToEnum(GetOptimizerTypeValues(), 34, "OptimizerType", value)); } const StringUtil::EnumStringLiteral *GetOrderByNullTypeValues() { From 88530c95a5c07a16c16fa1fa0127c194f599e74e Mon Sep 17 00:00:00 2001 From: DinosL Date: Tue, 2 Dec 2025 17:26:07 +0100 Subject: [PATCH 17/27] .. --- src/function/table/table_scan.cpp | 18 +- .../duckdb/storage/table/chunk_info.hpp | 9 + .../duckdb/storage/table/row_group.hpp | 1 + .../storage/table/row_group_collection.hpp | 19 -- .../storage/table/row_version_manager.hpp | 1 + .../duckdb/storage/table/scan_state.hpp | 6 + src/storage/table/chunk_info.cpp | 118 +++++++- src/storage/table/row_group.cpp | 8 + src/storage/table/row_group_collection.cpp | 30 ++ src/storage/table/row_version_manager.cpp | 22 ++ src/storage/table/scan_state.cpp | 4 +- test/optimizer/row_number_virtual_column.test | 273 +++++++++++------- 12 files changed, 373 insertions(+), 136 deletions(-) diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index 7fdae3aaf62b..5a216d2fca60 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -100,7 +100,7 @@ class TableScanGlobalState : public GlobalTableFunctionState { //! The types of all scanned columns. vector scanned_types; //! row_number column index - idx_t row_number_col_index = DConstants::INVALID_INDEX; + optional_idx row_number_col_index; //! Synchronize changes to the global scan state. mutex global_state_mutex; @@ -301,6 +301,11 @@ class DuckTableScanState : public TableScanGlobalState { l_state->scan_state.Initialize(std::move(storage_ids), context.client, input.filters, input.sample_options); + auto &db = bind_data.table.catalog; + auto &txn = DuckTransaction::Get(context.client, db); + l_state->scan_state.local_state.transaction = txn; + l_state->scan_state.table_state.transaction = txn; + storage.NextParallelScan(context.client, state, l_state->scan_state); if (input.CanRemoveFilterColumns()) { l_state->all_columns.Initialize(context.client, scanned_types); @@ -329,14 +334,13 @@ class DuckTableScanState : public TableScanGlobalState { storage.Scan(tx, output, l_state.scan_state); } if (output.size() > 0) { - if (row_number_col_index != DConstants::INVALID_INDEX) { - auto &row_number_vec = output.data[row_number_col_index]; + if (row_number_col_index.IsValid()) { + auto &row_number_vec = output.data[row_number_col_index.GetIndex()]; row_number_vec.SetVectorType(VectorType::FLAT_VECTOR); auto row_number_data = FlatVector::GetData(row_number_vec); auto count = output.size(); - idx_t row_group_index = l_state.scan_state.table_state.batch_index - 1; - idx_t base = state.scan_state.collection->GetPrefixSum(row_group_index) + l_state.row_number_count; + idx_t base = l_state.scan_state.table_state.base_row_number + l_state.row_number_count; for (idx_t i = 0; i < count; i++) { row_number_data[i] = static_cast(base + i + 1); @@ -404,12 +408,10 @@ unique_ptr DuckTableScanInitGlobal(ClientContext &cont for (idx_t i = 0; i < input.column_ids.size(); i++) { if (input.column_ids[i] == COLUMN_IDENTIFIER_ROW_NUMBER) { g_state->row_number_col_index = i; + g_state->state.scan_state.emit_row_numbers = true; break; } } - if (g_state->row_number_col_index != DConstants::INVALID_INDEX) { - g_state->state.scan_state.collection->PrecomputePrefixSums(); - } if (!input.CanRemoveFilterColumns()) { return std::move(g_state); } diff --git a/src/include/duckdb/storage/table/chunk_info.hpp b/src/include/duckdb/storage/table/chunk_info.hpp index 507386fb8d82..cc3a0cf76676 100644 --- a/src/include/duckdb/storage/table/chunk_info.hpp +++ b/src/include/duckdb/storage/table/chunk_info.hpp @@ -47,6 +47,8 @@ class ChunkInfo { virtual bool Fetch(TransactionData transaction, row_t row) = 0; virtual void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) = 0; virtual idx_t GetCommittedDeletedCount(idx_t max_count) const = 0; + virtual idx_t GetCommittedDeletedCount(transaction_t min_start_time, transaction_t transaction_id, + idx_t max_count) = 0; virtual bool Cleanup(transaction_t lowest_transaction) const; virtual bool HasDeletes() const = 0; @@ -89,6 +91,7 @@ class ChunkConstantInfo : public ChunkInfo { bool Fetch(TransactionData transaction, row_t row) override; void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override; idx_t GetCommittedDeletedCount(idx_t max_count) const override; + idx_t GetCommittedDeletedCount(transaction_t min_start_id, transaction_t min_transaction_id, idx_t max_count) override; bool Cleanup(transaction_t lowest_transaction) const override; bool HasDeletes() const override; @@ -100,6 +103,8 @@ class ChunkConstantInfo : public ChunkInfo { template idx_t TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector, idx_t max_count) const; + template + idx_t TemplatedGetCommittedDeleteCount(transaction_t start_time, transaction_t transaction_id, idx_t max_count) const; }; class ChunkVectorInfo : public ChunkInfo { @@ -120,6 +125,8 @@ class ChunkVectorInfo : public ChunkInfo { void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override; bool Cleanup(transaction_t lowest_transaction) const override; idx_t GetCommittedDeletedCount(idx_t max_count) const override; + idx_t GetCommittedDeletedCount(transaction_t min_start_time, transaction_t transaction_id, + idx_t max_count) override; void Append(idx_t start, idx_t end, transaction_t commit_id); @@ -143,6 +150,8 @@ class ChunkVectorInfo : public ChunkInfo { template idx_t TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector, idx_t max_count) const; + template + idx_t TemplatedGetCommittedDeleteCount(transaction_t start_time, transaction_t transaction_id, idx_t max_count) const; IndexPointer GetInsertedPointer() const; IndexPointer GetDeletedPointer() const; diff --git a/src/include/duckdb/storage/table/row_group.hpp b/src/include/duckdb/storage/table/row_group.hpp index d1af4e5b8a1e..0a5874adfedc 100644 --- a/src/include/duckdb/storage/table/row_group.hpp +++ b/src/include/duckdb/storage/table/row_group.hpp @@ -169,6 +169,7 @@ class RowGroup : public SegmentBase { RowGroupWriteData WriteToDisk(RowGroupWriteInfo &info) const; //! Returns the number of committed rows (count - committed deletes) idx_t GetCommittedRowCount(); + idx_t GetCommittedRowCount(transaction_t min_start_time, transaction_t transaction_id); RowGroupWriteData WriteToDisk(RowGroupWriter &writer); RowGroupPointer Checkpoint(RowGroupWriteData write_data, RowGroupWriter &writer, TableStatistics &global_stats, idx_t row_group_start); diff --git a/src/include/duckdb/storage/table/row_group_collection.hpp b/src/include/duckdb/storage/table/row_group_collection.hpp index 1a300b2826c9..039c1a754d1e 100644 --- a/src/include/duckdb/storage/table/row_group_collection.hpp +++ b/src/include/duckdb/storage/table/row_group_collection.hpp @@ -155,25 +155,6 @@ class RowGroupCollection { } void SetAppendRequiresNewRowGroup(); - std::vector prefix_sums; - - void PrecomputePrefixSums() { - idx_t cumulative = 0; - prefix_sums.resize(owned_row_groups->GetSegmentCount()); - - for (idx_t i = 0; i < owned_row_groups->GetSegmentCount(); ++i) { - cumulative += owned_row_groups.get()->GetSegment(i)->GetNode().GetCommittedRowCount(); - prefix_sums[i] = cumulative; - } - } - - idx_t GetPrefixSum(idx_t index) const { - if (index == 0) { - return 0; // No rows before the first row group - } - return prefix_sums[index - 1]; - } - private: optional_ptr> NextUpdateRowGroup(RowGroupSegmentTree &row_groups, row_t *ids, idx_t &pos, idx_t count) const; diff --git a/src/include/duckdb/storage/table/row_version_manager.hpp b/src/include/duckdb/storage/table/row_version_manager.hpp index 3969d567f9de..a33b8ca3e223 100644 --- a/src/include/duckdb/storage/table/row_version_manager.hpp +++ b/src/include/duckdb/storage/table/row_version_manager.hpp @@ -29,6 +29,7 @@ class RowVersionManager { return allocator; } idx_t GetCommittedDeletedCount(idx_t count); + idx_t GetCommittedDeletedCount(transaction_t min_start_time, transaction_t transaction_id, idx_t max_count); idx_t GetSelVector(TransactionData transaction, idx_t vector_idx, SelectionVector &sel_vector, idx_t max_count); idx_t GetCommittedSelVector(transaction_t start_time, transaction_t transaction_id, idx_t vector_idx, diff --git a/src/include/duckdb/storage/table/scan_state.hpp b/src/include/duckdb/storage/table/scan_state.hpp index 3b17cfb68fbc..52c6c8213507 100644 --- a/src/include/duckdb/storage/table/scan_state.hpp +++ b/src/include/duckdb/storage/table/scan_state.hpp @@ -20,6 +20,7 @@ #include "duckdb/parser/parsed_data/sample_options.hpp" #include "duckdb/storage/storage_index.hpp" #include "duckdb/planner/table_filter_state.hpp" +#include "fmt/format.h" namespace duckdb { class AdaptiveFilter; @@ -213,6 +214,8 @@ class CollectionScanState { idx_t batch_index; //! The valid selection SelectionVector valid_sel; + idx_t base_row_number; + TransactionData transaction; RandomEngine random; @@ -304,6 +307,9 @@ struct ParallelCollectionScanState { idx_t vector_index; idx_t max_row; idx_t batch_index; + optional_idx base_row_number; + bool emit_row_numbers; + // TransactionData transaction_data; atomic processed_rows; mutex lock; diff --git a/src/storage/table/chunk_info.cpp b/src/storage/table/chunk_info.cpp index 3f502ee329b2..71bf30b36128 100644 --- a/src/storage/table/chunk_info.cpp +++ b/src/storage/table/chunk_info.cpp @@ -7,6 +7,9 @@ #include "duckdb/common/serializer/memory_stream.hpp" #include "duckdb/transaction/delete_info.hpp" #include "duckdb/execution/index/fixed_size_allocator.hpp" +#include "fmt/format.h" + +#include namespace duckdb { @@ -73,6 +76,15 @@ idx_t ChunkConstantInfo::TemplatedGetSelVector(transaction_t start_time, transac return 0; } +template +idx_t ChunkConstantInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time, transaction_t transaction_id, idx_t max_count) const { + if (OP::UseInsertedVersion(start_time, transaction_id, insert_id) && + OP::UseDeletedVersion(start_time, transaction_id, delete_id)) { + return max_count; + } + return 0; +} + idx_t ChunkConstantInfo::GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) const { return TemplatedGetSelVector(transaction.start_time, transaction.transaction_id, sel_vector, max_count); @@ -83,6 +95,10 @@ idx_t ChunkConstantInfo::GetCommittedSelVector(transaction_t min_start_id, trans return TemplatedGetSelVector(min_start_id, min_transaction_id, sel_vector, max_count); } +idx_t ChunkConstantInfo::GetCommittedDeletedCount(transaction_t min_start_id, transaction_t min_transaction_id, idx_t max_count) { + return TemplatedGetCommittedDeleteCount(min_start_id, min_transaction_id, max_count); +} + bool ChunkConstantInfo::Fetch(TransactionData transaction, row_t row) { return UseVersion(transaction, insert_id) && !UseVersion(transaction, delete_id); } @@ -101,6 +117,12 @@ idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) const { return delete_id < TRANSACTION_ID_START ? max_count : 0; } +//FIXME Find out when we use that and change accordingly +// idx_t ChunkConstantInfo::GetCommittedDeletedCount(transaction_t min_start_time, transaction_t transaction_id, +// idx_t max_count) const { +// return delete_id < TRANSACTION_ID_START ? max_count : 0; +// } + bool ChunkConstantInfo::Cleanup(transaction_t lowest_transaction) const { if (delete_id != NOT_DELETED_ID) { // the chunk info is labeled as deleted - we need to keep it around @@ -164,7 +186,8 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti auto deleted = segment.GetPtr(); for (idx_t i = 0; i < max_count; i++) { if (OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { - sel_vector.set_index(count++, i); + // sel_vector.set_index(count++, i); + count++; } } return count; @@ -177,7 +200,8 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti idx_t count = 0; for (idx_t i = 0; i < max_count; i++) { if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i])) { - sel_vector.set_index(count++, i); + // sel_vector.set_index(count++, i); + count++; } } return count; @@ -193,7 +217,8 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti for (idx_t i = 0; i < max_count; i++) { if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i]) && OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { - sel_vector.set_index(count++, i); + // sel_vector.set_index(count++, i); + count++; } } return count; @@ -409,6 +434,93 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) const { return delete_count; } +template +idx_t ChunkVectorInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time, transaction_t transaction_id, idx_t max_count) const { + if (HasConstantInsertionId()) { + if (!AnyDeleted()) { + // all tuples have the same inserted id: and no tuples were deleted + if (OP::UseInsertedVersion(start_time, transaction_id, ConstantInsertId())) { + return max_count; + } else { + return 0; + } + } + if (!OP::UseInsertedVersion(start_time, transaction_id, ConstantInsertId())) { + return 0; + } + // have to check deleted flag + idx_t count = 0; + auto segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = segment.GetPtr(); + for (idx_t i = 0; i < max_count; i++) { + if (OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { + // sel_vector.set_index(count++, i); + count++; + } + } + return count; + } + if (!AnyDeleted()) { + // have to check inserted flag + auto insert_segment = allocator.GetHandle(GetInsertedPointer()); + auto inserted = insert_segment.GetPtr(); + + idx_t count = 0; + for (idx_t i = 0; i < max_count; i++) { + if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i])) { + // sel_vector.set_index(count++, i); + count++; + } + } + return count; + } + + idx_t count = 0; + // have to check both flags + auto insert_segment = allocator.GetHandle(GetInsertedPointer()); + auto inserted = insert_segment.GetPtr(); + + auto delete_segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = delete_segment.GetPtr(); + for (idx_t i = 0; i < max_count; i++) { + if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i]) && + OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { + // sel_vector.set_index(count++, i); + count++; + } + } + return count; +} + +idx_t ChunkVectorInfo::GetCommittedDeletedCount(transaction_t start_time, transaction_t transaction_id, + idx_t max_count) { + + if (!AnyDeleted()) { + if (HasConstantInsertionId()) { + if (ConstantInsertId() < start_time || ConstantInsertId() == transaction_id){ + return max_count; + } + } + return 0; + } + + + auto segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = segment.GetPtr(); + + idx_t delete_count = 0; + for (idx_t i = 0; i < max_count; i++) { + // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started + if (!(deleted[i] < start_time || deleted[i] == transaction_id) ) { + delete_count++; + } + } + return delete_count; + + // return TemplatedGetCommittedDeleteCount(start_time, transaction_id, max_count); +} + + void ChunkVectorInfo::Write(WriteStream &writer) const { SelectionVector sel(STANDARD_VECTOR_SIZE); transaction_t start_time = TRANSACTION_ID_START - 1; diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 6bc1bda6af83..c2fa4fb1329e 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -1063,6 +1063,14 @@ idx_t RowGroup::GetCommittedRowCount() { return count - vinfo->GetCommittedDeletedCount(count); } +idx_t RowGroup::GetCommittedRowCount(transaction_t start_time, transaction_t transaction_id) { + auto vinfo = GetVersionInfo(); + if (!vinfo) { + return count; + } + return vinfo->GetCommittedDeletedCount(start_time, transaction_id, count); +} + bool RowGroup::HasUnloadedDeletes() const { if (deletes_pointers.empty()) { // no stored deletes at all diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index 22f817db2c5b..1a5e7207cee5 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -264,6 +264,10 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec auto row_start = state.current_row_group->GetRowStart(); collection = state.collection; row_group = state.current_row_group; + // if we are emitting row numbers, we need to set the current base row number + if (state.emit_row_numbers) { + scan_state.base_row_number = state.base_row_number.GetIndex(); + } if (ClientConfig::GetConfig(context).verify_parallelism) { vector_index = state.vector_index; max_row = row_start + MinValue(current_row_group.count, @@ -274,11 +278,37 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec state.current_row_group = state.GetNextRowGroup(*state.row_groups, *row_group).get(); state.vector_index = 0; } + // TODO: do this for verify_parallelism, OR do not do this optimization if verify_parallelism is set + // this could just throw an exception + if (state.emit_row_numbers) { + throw InvalidInputException("verify_parallelism is not supported with emitting row numbers"); + } } else { state.processed_rows += current_row_group.count; vector_index = 0; max_row = row_start + current_row_group.count; state.current_row_group = state.GetNextRowGroup(*state.row_groups, *row_group).get(); + // FIXME: this should not be GetCommittedRowCount but use the transaction id + if (state.emit_row_numbers) { + state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(scan_state.transaction.start_time,scan_state.transaction.transaction_id); + auto new_rn = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(); + auto rn = state.base_row_number.GetIndex(); + auto break_here = true; + + // for (idx_t r = 0, i = 0; r < state.collection->row_group_size ; r += STANDARD_VECTOR_SIZE, i++) { + // SelectionVector sel_vector; + // idx_t current_row = i * STANDARD_VECTOR_SIZE; + // auto max_count = MinValue(STANDARD_VECTOR_SIZE, state.collection->row_group_size - current_row); + // auto row_num = state.base_row_number.GetIndex() + current_row_group.GetSelVector(scan_state.transaction, i, sel_vector, max_count); + // state.base_row_number = row_num; + // } + + + // SelectionVector sel_vector; + // idx_t current_row = state.batch_index * STANDARD_VECTOR_SIZE; + // auto max_count = MinValue(STANDARD_VECTOR_SIZE, state.collection->row_group_size - current_row); + // state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetSelVector(scan_state.transaction, scan_state.vector_index, sel_vector, max_count); + } } max_row = MinValue(max_row, state.max_row); scan_state.batch_index = ++state.batch_index; diff --git a/src/storage/table/row_version_manager.cpp b/src/storage/table/row_version_manager.cpp index 747e142f655f..db33645ec51e 100644 --- a/src/storage/table/row_version_manager.cpp +++ b/src/storage/table/row_version_manager.cpp @@ -29,6 +29,28 @@ idx_t RowVersionManager::GetCommittedDeletedCount(idx_t count) { return deleted_count; } +idx_t RowVersionManager::GetCommittedDeletedCount(transaction_t start_time, transaction_t transaction_id, idx_t count) { + lock_guard l(version_lock); + idx_t deleted_count = 0; + for (idx_t r = 0, i = 0; r < count; r += STANDARD_VECTOR_SIZE, i++) { + if (i >= vector_info.size() || !vector_info[i]) { + // deleted_count += STANDARD_VECTOR_SIZE; + continue; + } + idx_t max_count = MinValue(STANDARD_VECTOR_SIZE, count - r); + if (max_count == 0) { + break; + } + deleted_count += vector_info[i]->GetCommittedDeletedCount(start_time, transaction_id, max_count); + // SelectionVector sel_vector; + // TransactionData txn(transaction_id, start_time); + // + // deleted_count += vector_info[i]->GetSelVector(txn, sel_vector, max_count); + // auto break_here = true; + } + return deleted_count; +} + optional_ptr RowVersionManager::GetChunkInfo(idx_t vector_idx) { if (vector_idx >= vector_info.size()) { return nullptr; diff --git a/src/storage/table/scan_state.cpp b/src/storage/table/scan_state.cpp index 01314d7d18d0..b99efeca40db 100644 --- a/src/storage/table/scan_state.cpp +++ b/src/storage/table/scan_state.cpp @@ -176,7 +176,7 @@ TableScanOptions &CollectionScanState::GetOptions() { } ParallelCollectionScanState::ParallelCollectionScanState() - : collection(nullptr), current_row_group(nullptr), processed_rows(0) { + : collection(nullptr), current_row_group(nullptr), base_row_number(0), emit_row_numbers(false), processed_rows(0) { } optional_ptr> ParallelCollectionScanState::GetRootSegment(RowGroupSegmentTree &row_groups) const { @@ -196,7 +196,7 @@ ParallelCollectionScanState::GetNextRowGroup(RowGroupSegmentTree &row_groups, Se CollectionScanState::CollectionScanState(TableScanState &parent_p) : row_group(nullptr), vector_index(0), max_row_group_row(0), row_groups(nullptr), max_row(0), batch_index(0), - valid_sel(STANDARD_VECTOR_SIZE), random(-1), parent(parent_p) { + valid_sel(STANDARD_VECTOR_SIZE), transaction(0, 0), random(-1), parent(parent_p) { } optional_ptr> CollectionScanState::GetNextRowGroup(SegmentNode &row_group) const { diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index ad4d2cd442fd..5a3b581996a4 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -2,117 +2,182 @@ # description: Test row_number virtual column # group: [optimizer] -statement ok -CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(130000); - -# Check that row_number is not accessible -statement error -SELECT a, row_number FROM foo; ----- -Binder Error: Referenced column "row_number" not found in FROM clause - -query I -SELECT a as row_number FROM foo limit 3; ----- -0 -1 -2 - -query I -select row_number() over() from foo where a < 3; ----- -1 -2 -3 +# statement ok +# CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(130000); +# +# # Check that row_number is not accessible +# statement error +# SELECT a, row_number FROM foo; +# ---- +# Binder Error: Referenced column "row_number" not found in FROM clause +# +# query I +# SELECT a as row_number FROM foo limit 3; +# ---- +# 0 +# 1 +# 2 +# +# query I +# select row_number() over() from foo where a < 3; +# ---- +# 1 +# 2 +# 3 +# +# statement ok +# delete from foo where a < 5000; +# +# query III +# select row_number() over(), a, rowid from foo order by a desc limit 1; +# ---- +# 125000 129999 129999 +# +# query III +# select a, rowid, row_number() over() from foo order by a desc limit 1; +# ---- +# 129999 129999 125000 +# +# query II +# explain select a,rowid, row_number() over() as x from foo order by a desc limit 1; +# ---- +# physical_plan :.*WINDOW.* +# +# statement ok +# select setseed(0.1); +# +# statement ok +# create or replace table test as select (random()*10)::INT a, (random()*5)::INT b from range(10); +# +# statement ok +# create or replace table test2 as select (random()*100)::INT c, (random()*5)::INT b from range(10); +# +# # Try with a window function and a Join +# +# query II +# explain SELECT t.row_num, t.a, t2.c FROM ( SELECT a, b, ROW_NUMBER() OVER () AS row_num FROM test) t JOIN test2 t2 ON t.b = t2.b limit 10; +# ---- +# physical_plan :.*WINDOW.* +# +# +# statement ok +# create table t1 as select range::FLOAT a, range b, range c, range d from range (3); +# +# query I +# SELECT CASE WHEN row_number() OVER () = 2 THEN 2222::BIGINT ELSE 1111::BIGINT END FROM t1; +# ---- +# 1111 +# 2222 +# 1111 +# +# statement ok +# CREATE OR REPLACE TABLE numbers AS SELECT range(1, 5) AS num; +# +# query II +# SELECT row_number() OVER(), unnest(num) FROM numbers; +# ---- +# 1 1 +# 1 2 +# 1 3 +# 1 4 +# +# query II +# SELECT row_number() OVER (), nums FROM (SELECT unnest(num) AS nums from numbers); +# ---- +# 1 1 +# 2 2 +# 3 3 +# 4 4 +# +# # Try with a join on the row_number +# +# statement ok +# CREATE OR REPLACE TABLE foo AS SELECT range a from range(3); +# +# statement ok +# CREATE OR REPLACE TABLE bar AS SELECT range b from range(2,5); +# +# query II +# EXPLAIN SELECT f.row_num AS foo_row, b.row_num AS bar_row, f.a, b.b FROM (SELECT row_number() OVER () AS row_num, a FROM foo) f +# JOIN (SELECT row_number() OVER () AS row_num, b FROM bar) b ON f.row_num = b.row_num; +# ---- +# physical_plan :.*WINDOW.* +# +# statement ok +# pragma disable_optimizer; +# +# statement ok +# create table tbl as select range a from range(3); +# +# statement ok +# pragma enable_optimizer; +# +# query II +# EXPLAIN SELECT row_number + 1 FROM (SELECT row_number() OVER () AS row_number FROM tbl); +# ---- +# physical_plan :.*WINDOW.* + +# Try with multiple connections + +# statement ok +# pragma disabled_optimizers = "window_rewriter" + +# statement ok +# SET immediate_transaction_mode=true; +# +# statement ok +# CREATE TABLE big_tbl AS FROM range(1_000_000) t(i); +# +# statement ok con1 +# BEGIN +# +# statement ok con2 +# DELETE FROM big_tbl WHERE i%2=0; +# +# query III con1 +# SELECT MIN(r), MAX(r), AVG(r) FROM ( +# SELECT row_number() OVER () FROM big_tbl +# ) t(r) +# ---- +# 1 1000000 500000.5 +# +# query III con2 +# SELECT MIN(r), MAX(r), AVG(r) FROM ( +# SELECT row_number() OVER () FROM big_tbl +# ) t(r) +# ---- +# 1 500000 250000.5 +# +# mode skip + +# statement ok +# pragma threads=1 statement ok -delete from foo where a < 5000; - -query III -select row_number() over(), a, rowid from foo order by a desc limit 1; ----- -125000 129999 129999 - -query III -select a, rowid, row_number() over() from foo order by a desc limit 1; ----- -129999 129999 125000 - -query II -explain select a,rowid, row_number() over() as x from foo order by a desc limit 1; ----- -physical_plan :.*WINDOW.* +SET immediate_transaction_mode=true; statement ok -select setseed(0.1); +CREATE TABLE big_tbl AS FROM range(1_000_000) t(i); -statement ok -create or replace table test as select (random()*10)::INT a, (random()*5)::INT b from range(10); +# statement ok con1 +# BEGIN statement ok -create or replace table test2 as select (random()*100)::INT c, (random()*5)::INT b from range(10); +INSERT INTO big_tbl SELECT * FROM range(1_000_000); -# Try with a window function and a Join +# statement ok con2 +# DELETE FROM big_tbl WHERE i%2=0; -query II -explain SELECT t.row_num, t.a, t2.c FROM ( SELECT a, b, ROW_NUMBER() OVER () AS row_num FROM test) t JOIN test2 t2 ON t.b = t2.b limit 10; ----- -physical_plan :.*WINDOW.* - - -statement ok -create table t1 as select range::FLOAT a, range b, range c, range d from range (3); - -query I -SELECT CASE WHEN row_number() OVER () = 2 THEN 2222::BIGINT ELSE 1111::BIGINT END FROM t1; ----- -1111 -2222 -1111 - -statement ok -CREATE OR REPLACE TABLE numbers AS SELECT range(1, 5) AS num; - -query II -SELECT row_number() OVER(), unnest(num) FROM numbers; ----- -1 1 -1 2 -1 3 -1 4 - -query II -SELECT row_number() OVER (), nums FROM (SELECT unnest(num) AS nums from numbers); ----- -1 1 -2 2 -3 3 -4 4 - -# Try with a join on the row_number - -statement ok -CREATE OR REPLACE TABLE foo AS SELECT range a from range(3); - -statement ok -CREATE OR REPLACE TABLE bar AS SELECT range b from range(2,5); - -query II -EXPLAIN SELECT f.row_num AS foo_row, b.row_num AS bar_row, f.a, b.b FROM (SELECT row_number() OVER () AS row_num, a FROM foo) f -JOIN (SELECT row_number() OVER () AS row_num, b FROM bar) b ON f.row_num = b.row_num; ----- -physical_plan :.*WINDOW.* - -statement ok -pragma disable_optimizer; - -statement ok -create table tbl as select range a from range(3); - -statement ok -pragma enable_optimizer; - -query II -EXPLAIN SELECT row_number + 1 FROM (SELECT row_number() OVER () AS row_number FROM tbl); +query III +SELECT MIN(r), MAX(r), AVG(r) FROM ( + SELECT row_number() OVER () FROM big_tbl +) t(r) ---- -physical_plan :.*WINDOW.* \ No newline at end of file +1 2000000 1000000.5 + +# query III con2 +# SELECT MIN(r), MAX(r), AVG(r) FROM ( +# SELECT row_number() OVER () FROM big_tbl +# ) t(r) +# ---- +# 1 500000 250000.5 \ No newline at end of file From 4d98672d41aac1e698f203c0f276f97767560ee0 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Tue, 2 Dec 2025 19:05:31 +0100 Subject: [PATCH 18/27] my changes --- src/function/table/table_scan.cpp | 2 +- .../duckdb/storage/table/scan_state.hpp | 2 +- src/storage/table/chunk_info.cpp | 48 ++++++++++--------- src/storage/table/row_group.cpp | 2 +- src/storage/table/row_group_collection.cpp | 13 +++-- src/storage/table/row_version_manager.cpp | 21 ++++++-- src/storage/table/scan_state.cpp | 2 +- 7 files changed, 56 insertions(+), 34 deletions(-) diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index 5a216d2fca60..3e8c7f2f877b 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -289,7 +289,7 @@ class DuckTableScanState : public TableScanGlobalState { vector storage_ids; for (auto &col : input.column_indexes) { if (col.IsRowNumberColumn()) { - continue; + continue; // initializte emit_row_numbers } storage_ids.push_back(GetStorageIndex(bind_data.table, col)); } diff --git a/src/include/duckdb/storage/table/scan_state.hpp b/src/include/duckdb/storage/table/scan_state.hpp index 52c6c8213507..1f5824e56d95 100644 --- a/src/include/duckdb/storage/table/scan_state.hpp +++ b/src/include/duckdb/storage/table/scan_state.hpp @@ -308,7 +308,7 @@ struct ParallelCollectionScanState { idx_t max_row; idx_t batch_index; optional_idx base_row_number; - bool emit_row_numbers; + bool emit_row_numbers = true; // TransactionData transaction_data; atomic processed_rows; mutex lock; diff --git a/src/storage/table/chunk_info.cpp b/src/storage/table/chunk_info.cpp index 71bf30b36128..f3d54ec3ba0c 100644 --- a/src/storage/table/chunk_info.cpp +++ b/src/storage/table/chunk_info.cpp @@ -186,7 +186,7 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti auto deleted = segment.GetPtr(); for (idx_t i = 0; i < max_count; i++) { if (OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { - // sel_vector.set_index(count++, i); + sel_vector.set_index(count++, i); count++; } } @@ -200,7 +200,7 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti idx_t count = 0; for (idx_t i = 0; i < max_count; i++) { if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i])) { - // sel_vector.set_index(count++, i); + sel_vector.set_index(count++, i); count++; } } @@ -217,7 +217,7 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti for (idx_t i = 0; i < max_count; i++) { if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i]) && OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { - // sel_vector.set_index(count++, i); + sel_vector.set_index(count++, i); count++; } } @@ -496,28 +496,32 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(transaction_t start_time, transa idx_t max_count) { if (!AnyDeleted()) { - if (HasConstantInsertionId()) { - if (ConstantInsertId() < start_time || ConstantInsertId() == transaction_id){ - return max_count; - } - } + // we wil come back to this, potentially inserted rows are deleted? Unsure + // if (HasConstantInsertionId()) { + // if (ConstantInsertId() < start_time || ConstantInsertId() == transaction_id){ + // return max_count; + // } + // } return 0; } - - auto segment = allocator.GetHandle(GetDeletedPointer()); - auto deleted = segment.GetPtr(); - - idx_t delete_count = 0; - for (idx_t i = 0; i < max_count; i++) { - // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started - if (!(deleted[i] < start_time || deleted[i] == transaction_id) ) { - delete_count++; - } - } - return delete_count; - - // return TemplatedGetCommittedDeleteCount(start_time, transaction_id, max_count); + // + // auto segment = allocator.GetHandle(GetDeletedPointer()); + // auto deleted = segment.GetPtr(); + // + // idx_t delete_count = 0; + // for (idx_t i = 0; i < max_count; i++) { + // // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started + // if (!(deleted[i] < start_time || deleted[i] == transaction_id) ) { + // delete_count++; + // } + // } + // return delete_count; + + auto sel_vec = SelectionVector(); + auto wat = TemplatedGetSelVector(start_time, transaction_id, sel_vec, max_count); + auto break_here = 0; + return wat; } diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index c2fa4fb1329e..842e5a6543a1 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -1068,7 +1068,7 @@ idx_t RowGroup::GetCommittedRowCount(transaction_t start_time, transaction_t tra if (!vinfo) { return count; } - return vinfo->GetCommittedDeletedCount(start_time, transaction_id, count); + return count - vinfo->GetCommittedDeletedCount(start_time, transaction_id, count); } bool RowGroup::HasUnloadedDeletes() const { diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index 1a5e7207cee5..5fedeb266004 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -290,10 +290,13 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec state.current_row_group = state.GetNextRowGroup(*state.row_groups, *row_group).get(); // FIXME: this should not be GetCommittedRowCount but use the transaction id if (state.emit_row_numbers) { - state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(scan_state.transaction.start_time,scan_state.transaction.transaction_id); - auto new_rn = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(); - auto rn = state.base_row_number.GetIndex(); - auto break_here = true; + idx_t start = state.base_row_number.GetIndex(); + idx_t committed_row_count = current_row_group.GetCommittedRowCount(scan_state.transaction.start_time, scan_state.transaction.transaction_id); + state.base_row_number = start + committed_row_count; + Printer::PrintF("New base row number %d", state.base_row_number.GetIndex()); + // auto new_rn = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(); + // auto rn = state.base_row_number.GetIndex(); + // auto break_here = true; // for (idx_t r = 0, i = 0; r < state.collection->row_group_size ; r += STANDARD_VECTOR_SIZE, i++) { // SelectionVector sel_vector; @@ -308,6 +311,8 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec // idx_t current_row = state.batch_index * STANDARD_VECTOR_SIZE; // auto max_count = MinValue(STANDARD_VECTOR_SIZE, state.collection->row_group_size - current_row); // state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetSelVector(scan_state.transaction, scan_state.vector_index, sel_vector, max_count); + } else { + auto break_here = 0; } } max_row = MinValue(max_row, state.max_row); diff --git a/src/storage/table/row_version_manager.cpp b/src/storage/table/row_version_manager.cpp index db33645ec51e..51129453616d 100644 --- a/src/storage/table/row_version_manager.cpp +++ b/src/storage/table/row_version_manager.cpp @@ -32,16 +32,29 @@ idx_t RowVersionManager::GetCommittedDeletedCount(idx_t count) { idx_t RowVersionManager::GetCommittedDeletedCount(transaction_t start_time, transaction_t transaction_id, idx_t count) { lock_guard l(version_lock); idx_t deleted_count = 0; - for (idx_t r = 0, i = 0; r < count; r += STANDARD_VECTOR_SIZE, i++) { - if (i >= vector_info.size() || !vector_info[i]) { + for (idx_t row_id = 0, row_batch = 0; row_id < count; row_id += STANDARD_VECTOR_SIZE, row_batch++) { + if (row_batch >= vector_info.size() || !vector_info[row_batch]) { // deleted_count += STANDARD_VECTOR_SIZE; continue; } - idx_t max_count = MinValue(STANDARD_VECTOR_SIZE, count - r); + idx_t max_count = MinValue(STANDARD_VECTOR_SIZE, count - row_id); if (max_count == 0) { break; } - deleted_count += vector_info[i]->GetCommittedDeletedCount(start_time, transaction_id, max_count); + idx_t result = max_count; + if (vector_info[row_batch]) { + if (vector_info[row_batch]->type == ChunkInfoType::CONSTANT_INFO) { + auto &constant_vec_info = vector_info[row_batch]->Cast(); + SelectionVector sel; + TransactionData txn_data(transaction_id, start_time); + result = max_count - vector_info[row_batch]->GetSelVector(txn_data,sel, max_count); + // result = constant_vec_info.TemplatedGetSelVector(start_time, transaction_id, sel, result); + } else { + result = vector_info[row_batch]->GetCommittedDeletedCount(start_time, transaction_id, max_count); + } + + } + deleted_count += result; // SelectionVector sel_vector; // TransactionData txn(transaction_id, start_time); // diff --git a/src/storage/table/scan_state.cpp b/src/storage/table/scan_state.cpp index b99efeca40db..23c39b5c11eb 100644 --- a/src/storage/table/scan_state.cpp +++ b/src/storage/table/scan_state.cpp @@ -176,7 +176,7 @@ TableScanOptions &CollectionScanState::GetOptions() { } ParallelCollectionScanState::ParallelCollectionScanState() - : collection(nullptr), current_row_group(nullptr), base_row_number(0), emit_row_numbers(false), processed_rows(0) { + : collection(nullptr), current_row_group(nullptr), base_row_number(0), emit_row_numbers(true), processed_rows(0) { } optional_ptr> ParallelCollectionScanState::GetRootSegment(RowGroupSegmentTree &row_groups) const { From 72de69b012369193ffcf3e204069dbadf60b9dbf Mon Sep 17 00:00:00 2001 From: DinosL Date: Wed, 3 Dec 2025 14:51:48 +0100 Subject: [PATCH 19/27] . --- src/storage/table/chunk_info.cpp | 144 ++++++++++++++---- src/storage/table/row_group.cpp | 3 +- src/storage/table/row_group_collection.cpp | 14 +- src/storage/table/row_version_manager.cpp | 4 +- test/optimizer/row_number_virtual_column.test | 57 +++++-- 5 files changed, 169 insertions(+), 53 deletions(-) diff --git a/src/storage/table/chunk_info.cpp b/src/storage/table/chunk_info.cpp index 71bf30b36128..b1a7f9ff93b6 100644 --- a/src/storage/table/chunk_info.cpp +++ b/src/storage/table/chunk_info.cpp @@ -70,7 +70,7 @@ template idx_t ChunkConstantInfo::TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector, idx_t max_count) const { if (OP::UseInsertedVersion(start_time, transaction_id, insert_id) && - OP::UseDeletedVersion(start_time, transaction_id, delete_id)) { + OP::UseDeletedVersion(start_time, transaction_id, delete_id)) { return max_count; } return 0; @@ -95,9 +95,9 @@ idx_t ChunkConstantInfo::GetCommittedSelVector(transaction_t min_start_id, trans return TemplatedGetSelVector(min_start_id, min_transaction_id, sel_vector, max_count); } -idx_t ChunkConstantInfo::GetCommittedDeletedCount(transaction_t min_start_id, transaction_t min_transaction_id, idx_t max_count) { - return TemplatedGetCommittedDeleteCount(min_start_id, min_transaction_id, max_count); -} +// idx_t ChunkConstantInfo::GetCommittedDeletedCount(transaction_t min_start_id, transaction_t min_transaction_id, idx_t max_count) { +// return TemplatedGetCommittedDeleteCount(min_start_id, min_transaction_id, max_count); +// } bool ChunkConstantInfo::Fetch(TransactionData transaction, row_t row) { return UseVersion(transaction, insert_id) && !UseVersion(transaction, delete_id); @@ -117,11 +117,15 @@ idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) const { return delete_id < TRANSACTION_ID_START ? max_count : 0; } -//FIXME Find out when we use that and change accordingly -// idx_t ChunkConstantInfo::GetCommittedDeletedCount(transaction_t min_start_time, transaction_t transaction_id, -// idx_t max_count) const { -// return delete_id < TRANSACTION_ID_START ? max_count : 0; -// } +// FIXME Find out when we use that and change accordingly +idx_t ChunkConstantInfo::GetCommittedDeletedCount(transaction_t min_start_id, transaction_t min_transaction_id, idx_t max_count) { + // return delete_id < TRANSACTION_ID_START ? max_count : 0; + // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started + if ((delete_id < min_start_id || delete_id == min_transaction_id) ) { + return max_count; + } + return 0; +} bool ChunkConstantInfo::Cleanup(transaction_t lowest_transaction) const { if (delete_id != NOT_DELETED_ID) { @@ -433,6 +437,69 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) const { } return delete_count; } +// +// template +// idx_t ChunkVectorInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time, transaction_t transaction_id, idx_t max_count) const { +// +// // ---------- CONSTANT INSERT ID ---------- +// if (HasConstantInsertionId()) { +// const auto ins = ConstantInsertId(); +// +// // insertion not visible? +// if (!OP::UseInsertedVersion(start_time, transaction_id, ins)) { +// return 0; +// } +// +// // no deletes → all rows visible +// if (!AnyDeleted()) { +// return max_count; +// } +// +// // constant insertion, but deletion may vary per row +// auto del_seg = allocator.GetHandle(GetDeletedPointer()); +// auto deleted = del_seg.GetPtr(); +// +// idx_t count = 0; +// for (idx_t i = 0; i < max_count; i++) { +// // committed-visible row means: delete is NOT visible +// if (!OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { +// count++; +// } +// } +// return count; +// } +// +// // ---------- NO DELETES (per-row insert IDs only) ---------- +// if (!AnyDeleted()) { +// auto ins_seg = allocator.GetHandle(GetInsertedPointer()); +// auto inserted = ins_seg.GetPtr(); +// +// idx_t count = 0; +// for (idx_t i = 0; i < max_count; i++) { +// if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i])) { +// count++; +// } +// } +// return count; +// } +// +// // ---------- PER-ROW INSERTS + PER-ROW DELETES ---------- +// auto ins_seg = allocator.GetHandle(GetInsertedPointer()); +// auto inserted = ins_seg.GetPtr(); +// +// auto del_seg = allocator.GetHandle(GetDeletedPointer()); +// auto deleted = del_seg.GetPtr(); +// +// idx_t count = 0; +// for (idx_t i = 0; i < max_count; i++) { +// if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i]) && +// !OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { +// count++; +// } +// } +// return count; +// } + template idx_t ChunkVectorInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time, transaction_t transaction_id, idx_t max_count) const { @@ -449,16 +516,28 @@ idx_t ChunkVectorInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time return 0; } // have to check deleted flag - idx_t count = 0; + // idx_t count = 0; + // auto segment = allocator.GetHandle(GetDeletedPointer()); + // auto deleted = segment.GetPtr(); + // for (idx_t i = 0; i < max_count; i++) { + // if (OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { + // // sel_vector.set_index(count++, i); + // count++; + // } + // } + // return count; + auto segment = allocator.GetHandle(GetDeletedPointer()); auto deleted = segment.GetPtr(); + + idx_t delete_count = 0; for (idx_t i = 0; i < max_count; i++) { - if (OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { - // sel_vector.set_index(count++, i); - count++; + // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started + if ((deleted[i] < start_time || deleted[i] == transaction_id) ) { + delete_count++; } } - return count; + return delete_count; } if (!AnyDeleted()) { // have to check inserted flag @@ -496,28 +575,33 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(transaction_t start_time, transa idx_t max_count) { if (!AnyDeleted()) { - if (HasConstantInsertionId()) { - if (ConstantInsertId() < start_time || ConstantInsertId() == transaction_id){ - return max_count; - } - } + // if (HasConstantInsertionId()) { + // if (ConstantInsertId() < start_time || ConstantInsertId() == transaction_id){ + // return max_count; + // } + // } return 0; } - auto segment = allocator.GetHandle(GetDeletedPointer()); - auto deleted = segment.GetPtr(); + // auto segment = allocator.GetHandle(GetDeletedPointer()); + // auto deleted = segment.GetPtr(); + // + // idx_t delete_count = 0; + // for (idx_t i = 0; i < max_count; i++) { + // // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started + // if ((deleted[i] < start_time || deleted[i] == transaction_id) ) { + // delete_count++; + // } + // } + // return delete_count; - idx_t delete_count = 0; - for (idx_t i = 0; i < max_count; i++) { - // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started - if (!(deleted[i] < start_time || deleted[i] == transaction_id) ) { - delete_count++; - } - } - return delete_count; + return TemplatedGetCommittedDeleteCount(start_time, transaction_id, max_count); - // return TemplatedGetCommittedDeleteCount(start_time, transaction_id, max_count); + // auto sel_vec = SelectionVector(); + // auto wat = TemplatedGetSelVector(start_time, transaction_id, sel_vec, max_count); + // auto break_here = 0; + // return wat; } diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index c2fa4fb1329e..40f66dce4c9d 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -1068,7 +1068,8 @@ idx_t RowGroup::GetCommittedRowCount(transaction_t start_time, transaction_t tra if (!vinfo) { return count; } - return vinfo->GetCommittedDeletedCount(start_time, transaction_id, count); + auto res = count - vinfo->GetCommittedDeletedCount(start_time, transaction_id, count); + return res; } bool RowGroup::HasUnloadedDeletes() const { diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index 1a5e7207cee5..c318f6cd4e47 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -290,10 +290,16 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec state.current_row_group = state.GetNextRowGroup(*state.row_groups, *row_group).get(); // FIXME: this should not be GetCommittedRowCount but use the transaction id if (state.emit_row_numbers) { - state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(scan_state.transaction.start_time,scan_state.transaction.transaction_id); - auto new_rn = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(); - auto rn = state.base_row_number.GetIndex(); - auto break_here = true; + + idx_t start = state.base_row_number.GetIndex(); + idx_t committed_row_count = current_row_group.GetCommittedRowCount(scan_state.transaction.start_time, scan_state.transaction.transaction_id); + state.base_row_number = start + committed_row_count; + Printer::PrintF("New base row number %d", state.base_row_number.GetIndex()); + + // state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(scan_state.transaction.start_time,scan_state.transaction.transaction_id); + // auto new_rn = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(); + // auto rn = state.base_row_number.GetIndex(); + // auto break_here = true; // for (idx_t r = 0, i = 0; r < state.collection->row_group_size ; r += STANDARD_VECTOR_SIZE, i++) { // SelectionVector sel_vector; diff --git a/src/storage/table/row_version_manager.cpp b/src/storage/table/row_version_manager.cpp index db33645ec51e..e73fb0d78daa 100644 --- a/src/storage/table/row_version_manager.cpp +++ b/src/storage/table/row_version_manager.cpp @@ -34,14 +34,14 @@ idx_t RowVersionManager::GetCommittedDeletedCount(transaction_t start_time, tran idx_t deleted_count = 0; for (idx_t r = 0, i = 0; r < count; r += STANDARD_VECTOR_SIZE, i++) { if (i >= vector_info.size() || !vector_info[i]) { - // deleted_count += STANDARD_VECTOR_SIZE; continue; } idx_t max_count = MinValue(STANDARD_VECTOR_SIZE, count - r); if (max_count == 0) { break; } - deleted_count += vector_info[i]->GetCommittedDeletedCount(start_time, transaction_id, max_count); + auto res = vector_info[i]->GetCommittedDeletedCount(start_time, transaction_id, max_count); + deleted_count += res; // SelectionVector sel_vector; // TransactionData txn(transaction_id, start_time); // diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index 5a3b581996a4..6b5bdecbab69 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -102,10 +102,10 @@ # JOIN (SELECT row_number() OVER () AS row_num, b FROM bar) b ON f.row_num = b.row_num; # ---- # physical_plan :.*WINDOW.* -# -# statement ok -# pragma disable_optimizer; -# + +statement ok +pragma disable_optimizer; + # statement ok # create table tbl as select range a from range(3); # @@ -122,6 +122,9 @@ # statement ok # pragma disabled_optimizers = "window_rewriter" +# statement ok +# pragma threads=1 + # statement ok # SET immediate_transaction_mode=true; # @@ -153,31 +156,53 @@ # statement ok # pragma threads=1 -statement ok -SET immediate_transaction_mode=true; +# statement ok +# SET immediate_transaction_mode=true; statement ok -CREATE TABLE big_tbl AS FROM range(1_000_000) t(i); +CREATE TABLE big_tbl AS FROM range(1_000_000) select range::VARCHAR i; -# statement ok con1 -# BEGIN +statement ok con1 +BEGIN -statement ok -INSERT INTO big_tbl SELECT * FROM range(1_000_000); +statement ok con1 +pragma disable_optimizer; -# statement ok con2 -# DELETE FROM big_tbl WHERE i%2=0; +statement ok con1 +INSERT INTO big_tbl SELECT range::varchar FROM range(1_000_000); -query III +statement ok con1 +DELETE FROM big_tbl WHERE i::INT%2=0; + +mode output_result + +statement ok con1 +EXPLAIN SELECT MIN(r), MAX(r), AVG(r) FROM ( + SELECT row_number() OVER () FROM big_tbl +) t(r) + +query III con1 SELECT MIN(r), MAX(r), AVG(r) FROM ( SELECT row_number() OVER () FROM big_tbl ) t(r) ---- -1 2000000 1000000.5 +1 1000000 500000.5 # query III con2 # SELECT MIN(r), MAX(r), AVG(r) FROM ( # SELECT row_number() OVER () FROM big_tbl # ) t(r) # ---- -# 1 500000 250000.5 \ No newline at end of file +# 1 1000000 500000.5 + +statement ok con1 +commit; + +query III +SELECT MIN(r), MAX(r), AVG(r) FROM ( + SELECT row_number() OVER () FROM big_tbl +) t(r) +---- +1 1000000 500000.5 + + From 45ed68ae8da6e4447eb1fff8bd6089293d076e93 Mon Sep 17 00:00:00 2001 From: DinosL Date: Wed, 3 Dec 2025 15:16:06 +0100 Subject: [PATCH 20/27] . --- src/function/table/table_scan.cpp | 10 ++++++---- test/optimizer/row_number_virtual_column.test | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index 049580c3f9c5..3b510bc22991 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -308,10 +308,12 @@ class DuckTableScanState : public TableScanGlobalState { l_state->scan_state.Initialize(std::move(storage_ids), context.client, input.filters, input.sample_options); - auto &db = bind_data.table.catalog; - auto &txn = DuckTransaction::Get(context.client, db); - l_state->scan_state.local_state.transaction = txn; - l_state->scan_state.table_state.transaction = txn; + if (state.scan_state.emit_row_numbers) { + auto &db = bind_data.table.catalog; + auto &txn = DuckTransaction::Get(context.client, db); + l_state->scan_state.local_state.transaction = txn; + l_state->scan_state.table_state.transaction = txn; + } storage.NextParallelScan(context.client, state, l_state->scan_state); if (input.CanRemoveFilterColumns()) { diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index 6b5bdecbab69..adc1ea5b083f 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -103,8 +103,8 @@ # ---- # physical_plan :.*WINDOW.* -statement ok -pragma disable_optimizer; +# statement ok +# pragma disable_optimizer; # statement ok # create table tbl as select range a from range(3); @@ -156,8 +156,8 @@ pragma disable_optimizer; # statement ok # pragma threads=1 -# statement ok -# SET immediate_transaction_mode=true; +statement ok +SET immediate_transaction_mode=true; statement ok CREATE TABLE big_tbl AS FROM range(1_000_000) select range::VARCHAR i; From d9d4734da46a537170c9e477e7d91722517d97a6 Mon Sep 17 00:00:00 2001 From: DinosL Date: Fri, 5 Dec 2025 09:36:13 +0100 Subject: [PATCH 21/27] . --- src/function/table/table_scan.cpp | 4 +- src/storage/table/chunk_info.cpp | 65 +------------------ src/storage/table/row_group_collection.cpp | 19 ------ test/optimizer/row_number_virtual_column.test | 34 ++++------ 4 files changed, 17 insertions(+), 105 deletions(-) diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index 3b510bc22991..77a495448e92 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -357,6 +357,8 @@ class DuckTableScanState : public TableScanGlobalState { } auto next = storage.NextParallelScan(context, state, l_state.scan_state); + // One thread is assigned more than one batches. When we are done with one batch, we reset the counter. + l_state.row_number_count = 0; if (data_p.results_execution_mode == AsyncResultsExecutionMode::TASK_EXECUTOR) { // We can avoid looping, and just return as appropriate if (!next) { @@ -369,8 +371,6 @@ class DuckTableScanState : public TableScanGlobalState { if (!next) { return; } - // One thread is assigned more than one batches. When we are done with one batch, we reset the counter. - l_state.row_number_count = 0; // Before looping back, check if we are interrupted if (context.interrupted) { diff --git a/src/storage/table/chunk_info.cpp b/src/storage/table/chunk_info.cpp index b1a7f9ff93b6..67bca477773d 100644 --- a/src/storage/table/chunk_info.cpp +++ b/src/storage/table/chunk_info.cpp @@ -437,68 +437,6 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) const { } return delete_count; } -// -// template -// idx_t ChunkVectorInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time, transaction_t transaction_id, idx_t max_count) const { -// -// // ---------- CONSTANT INSERT ID ---------- -// if (HasConstantInsertionId()) { -// const auto ins = ConstantInsertId(); -// -// // insertion not visible? -// if (!OP::UseInsertedVersion(start_time, transaction_id, ins)) { -// return 0; -// } -// -// // no deletes → all rows visible -// if (!AnyDeleted()) { -// return max_count; -// } -// -// // constant insertion, but deletion may vary per row -// auto del_seg = allocator.GetHandle(GetDeletedPointer()); -// auto deleted = del_seg.GetPtr(); -// -// idx_t count = 0; -// for (idx_t i = 0; i < max_count; i++) { -// // committed-visible row means: delete is NOT visible -// if (!OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { -// count++; -// } -// } -// return count; -// } -// -// // ---------- NO DELETES (per-row insert IDs only) ---------- -// if (!AnyDeleted()) { -// auto ins_seg = allocator.GetHandle(GetInsertedPointer()); -// auto inserted = ins_seg.GetPtr(); -// -// idx_t count = 0; -// for (idx_t i = 0; i < max_count; i++) { -// if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i])) { -// count++; -// } -// } -// return count; -// } -// -// // ---------- PER-ROW INSERTS + PER-ROW DELETES ---------- -// auto ins_seg = allocator.GetHandle(GetInsertedPointer()); -// auto inserted = ins_seg.GetPtr(); -// -// auto del_seg = allocator.GetHandle(GetDeletedPointer()); -// auto deleted = del_seg.GetPtr(); -// -// idx_t count = 0; -// for (idx_t i = 0; i < max_count; i++) { -// if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i]) && -// !OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { -// count++; -// } -// } -// return count; -// } template @@ -590,7 +528,8 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(transaction_t start_time, transa // idx_t delete_count = 0; // for (idx_t i = 0; i < max_count; i++) { // // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started - // if ((deleted[i] < start_time || deleted[i] == transaction_id) ) { + // // if ((deleted[i] < start_time || deleted[i] == transaction_id) ) { + // if (deleted[i] != transaction_id && deleted[i] < start_time) { // delete_count++; // } // } diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index c318f6cd4e47..5930e44950aa 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -295,25 +295,6 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec idx_t committed_row_count = current_row_group.GetCommittedRowCount(scan_state.transaction.start_time, scan_state.transaction.transaction_id); state.base_row_number = start + committed_row_count; Printer::PrintF("New base row number %d", state.base_row_number.GetIndex()); - - // state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(scan_state.transaction.start_time,scan_state.transaction.transaction_id); - // auto new_rn = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(); - // auto rn = state.base_row_number.GetIndex(); - // auto break_here = true; - - // for (idx_t r = 0, i = 0; r < state.collection->row_group_size ; r += STANDARD_VECTOR_SIZE, i++) { - // SelectionVector sel_vector; - // idx_t current_row = i * STANDARD_VECTOR_SIZE; - // auto max_count = MinValue(STANDARD_VECTOR_SIZE, state.collection->row_group_size - current_row); - // auto row_num = state.base_row_number.GetIndex() + current_row_group.GetSelVector(scan_state.transaction, i, sel_vector, max_count); - // state.base_row_number = row_num; - // } - - - // SelectionVector sel_vector; - // idx_t current_row = state.batch_index * STANDARD_VECTOR_SIZE; - // auto max_count = MinValue(STANDARD_VECTOR_SIZE, state.collection->row_group_size - current_row); - // state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetSelVector(scan_state.transaction, scan_state.vector_index, sel_vector, max_count); } } max_row = MinValue(max_row, state.max_row); diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index adc1ea5b083f..f702c20b8cb4 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -128,6 +128,8 @@ # statement ok # SET immediate_transaction_mode=true; # +# mode output_result +# # statement ok # CREATE TABLE big_tbl AS FROM range(1_000_000) t(i); # @@ -150,7 +152,7 @@ # ) t(r) # ---- # 1 500000 250000.5 -# + # mode skip # statement ok @@ -160,41 +162,31 @@ statement ok SET immediate_transaction_mode=true; statement ok -CREATE TABLE big_tbl AS FROM range(1_000_000) select range::VARCHAR i; +CREATE TABLE big_tbl AS FROM range(1_000_000) t(i); statement ok con1 BEGIN -statement ok con1 -pragma disable_optimizer; +statement ok con2 +INSERT INTO big_tbl SELECT range FROM range(1_000_000); -statement ok con1 -INSERT INTO big_tbl SELECT range::varchar FROM range(1_000_000); +statement ok con2 +DELETE FROM big_tbl WHERE i%2=0; -statement ok con1 -DELETE FROM big_tbl WHERE i::INT%2=0; - -mode output_result - -statement ok con1 -EXPLAIN SELECT MIN(r), MAX(r), AVG(r) FROM ( +query III con1 +SELECT MIN(r), MAX(r), AVG(r) FROM ( SELECT row_number() OVER () FROM big_tbl ) t(r) +---- +1 1000000 500000.5 -query III con1 +query III con2 SELECT MIN(r), MAX(r), AVG(r) FROM ( SELECT row_number() OVER () FROM big_tbl ) t(r) ---- 1 1000000 500000.5 -# query III con2 -# SELECT MIN(r), MAX(r), AVG(r) FROM ( -# SELECT row_number() OVER () FROM big_tbl -# ) t(r) -# ---- -# 1 1000000 500000.5 - statement ok con1 commit; From be30a58df64d5027e7de1cdbb40e81fe5df5ccfe Mon Sep 17 00:00:00 2001 From: DinosL Date: Mon, 8 Dec 2025 10:37:13 +0100 Subject: [PATCH 22/27] . --- src/function/table/table_scan.cpp | 54 ++++++-------- .../duckdb/storage/table/scan_state.hpp | 4 +- src/storage/data_table.cpp | 10 +++ src/storage/table/row_group_collection.cpp | 32 +++++++- test/optimizer/row_number_virtual_column.test | 74 +++++++++---------- 5 files changed, 99 insertions(+), 75 deletions(-) diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index 77a495448e92..b1c2b2be1719 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -24,7 +24,6 @@ #include "duckdb/main/client_data.hpp" #include "duckdb/common/algorithm.hpp" #include "duckdb/planner/filter/optional_filter.hpp" -#include "duckdb/planner/filter/selectivity_optional_filter.hpp" #include "duckdb/planner/filter/in_filter.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/expression/bound_comparison_expression.hpp" @@ -100,7 +99,7 @@ class TableScanGlobalState : public GlobalTableFunctionState { vector projection_ids; //! The types of all scanned columns. vector scanned_types; - //! row_number column index + //! The position of the row_number column optional_idx row_number_col_index; //! Synchronize changes to the global scan state. mutex global_state_mutex; @@ -225,12 +224,6 @@ class DuckIndexScanState : public TableScanGlobalState { storage.Fetch(tx, output, column_ids, local_vector, scan_count, l_state.fetch_state); } if (output.size() == 0) { - if (data_p.results_execution_mode == AsyncResultsExecutionMode::TASK_EXECUTOR) { - // We can avoid looping, and just return as appropriate - data_p.async_result = AsyncResultType::HAVE_MORE_OUTPUT; - return; - } - // output is empty, loop back, since there might be results to be picked up from LOCAL_STORAGE phase continue; } @@ -309,10 +302,9 @@ class DuckTableScanState : public TableScanGlobalState { l_state->scan_state.Initialize(std::move(storage_ids), context.client, input.filters, input.sample_options); if (state.scan_state.emit_row_numbers) { - auto &db = bind_data.table.catalog; - auto &txn = DuckTransaction::Get(context.client, db); - l_state->scan_state.local_state.transaction = txn; - l_state->scan_state.table_state.transaction = txn; + auto &transaction = DuckTransaction::Get(context.client, bind_data.table.catalog); + l_state->scan_state.local_state.transaction = transaction; + l_state->scan_state.table_state.transaction = transaction; } storage.NextParallelScan(context.client, state, l_state->scan_state); @@ -328,7 +320,11 @@ class DuckTableScanState : public TableScanGlobalState { auto &l_state = data_p.local_state->Cast(); l_state.scan_state.options.force_fetch_row = ClientConfig::GetConfig(context).force_fetch_row; + // bool use_local_state = state.local_state.has_emitted_row_numbers; do { + if (context.interrupted) { + throw InterruptException(); + } if (bind_data.is_create_index) { storage.CreateIndexScan(l_state.scan_state, output, TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED); @@ -346,7 +342,14 @@ class DuckTableScanState : public TableScanGlobalState { auto row_number_data = FlatVector::GetData(row_number_vec); auto count = output.size(); - idx_t base = l_state.scan_state.table_state.base_row_number + l_state.row_number_count; + idx_t base; + if (state.local_state.has_emitted_row_numbers) { + base = l_state.scan_state.local_state.base_row_number + l_state.row_number_count; + // base = state.scan_state.base_row_number.GetIndex() + l_state.row_number_count; + } else { + base = l_state.scan_state.table_state.base_row_number + l_state.row_number_count; + } + // idx_t base = l_state.scan_state.local_state.base_row_number + l_state.row_number_count; for (idx_t i = 0; i < count; i++) { row_number_data[i] = static_cast(base + i + 1); @@ -357,25 +360,11 @@ class DuckTableScanState : public TableScanGlobalState { } auto next = storage.NextParallelScan(context, state, l_state.scan_state); - // One thread is assigned more than one batches. When we are done with one batch, we reset the counter. - l_state.row_number_count = 0; - if (data_p.results_execution_mode == AsyncResultsExecutionMode::TASK_EXECUTOR) { - // We can avoid looping, and just return as appropriate - if (!next) { - data_p.async_result = AsyncResultType::FINISHED; - } else { - data_p.async_result = AsyncResultType::HAVE_MORE_OUTPUT; - } - return; - } if (!next) { return; } - - // Before looping back, check if we are interrupted - if (context.interrupted) { - throw InterruptException(); - } + // One thread is assigned more than one batches. When we are done with one batch, we reset the counter. + l_state.row_number_count = 0; } while (true); } @@ -429,6 +418,10 @@ unique_ptr DuckTableScanInitGlobal(ClientContext &cont if (input.column_ids[i] == COLUMN_IDENTIFIER_ROW_NUMBER) { g_state->row_number_col_index = i; g_state->state.scan_state.emit_row_numbers = true; + g_state->state.local_state.emit_row_numbers = true; + // g_state->state.local_state.initial_base_row_number = g_state->state.scan_state.base_row_number; + // g_state->state.scan_state.initial_base_row_number = g_state->state.scan_state.base_row_number; + break; } } @@ -508,9 +501,6 @@ bool ExtractComparisonsAndInFilters(TableFilter &filter, vector()); return true; } - case TableFilterType::BLOOM_FILTER: { - return true; // We can't use it for finding cmp/in filters, but we can just ignore it - } case TableFilterType::CONJUNCTION_AND: { auto &conjunction_and = filter.Cast(); for (idx_t i = 0; i < conjunction_and.child_filters.size(); i++) { diff --git a/src/include/duckdb/storage/table/scan_state.hpp b/src/include/duckdb/storage/table/scan_state.hpp index 52c6c8213507..f82c79c5417c 100644 --- a/src/include/duckdb/storage/table/scan_state.hpp +++ b/src/include/duckdb/storage/table/scan_state.hpp @@ -308,8 +308,9 @@ struct ParallelCollectionScanState { idx_t max_row; idx_t batch_index; optional_idx base_row_number; + bool has_emitted_row_numbers = false; + optional_idx initial_base_row_number; bool emit_row_numbers; - // TransactionData transaction_data; atomic processed_rows; mutex lock; @@ -324,6 +325,7 @@ struct ParallelTableScanState { ParallelCollectionScanState local_state; //! Shared lock over the checkpoint to prevent checkpoints while reading shared_ptr checkpoint_lock; + optional_idx global_row_number; }; struct PrefetchState { diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 94729b918809..06392fbf7ad4 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -285,7 +285,17 @@ bool DataTable::NextParallelScan(ClientContext &context, ParallelTableScanState if (row_groups->NextParallelScan(context, state.scan_state, scan_state.table_state)) { return true; } + auto &local_storage = LocalStorage::Get(context, db); + + if (!state.local_state.has_emitted_row_numbers) { + state.local_state.base_row_number = state.scan_state.base_row_number; + state.local_state.has_emitted_row_numbers = true; + } + // if (!state.local_state.has_emitted_row_numbers) { + // state.local_state.base_row_number = state.scan_state.base_row_number; + // state.local_state.has_emitted_row_numbers = true; + // } if (local_storage.NextParallelScan(context, *this, state.local_state, scan_state.local_state)) { return true; } else { diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index 5930e44950aa..4c93d9708221 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -268,6 +268,33 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec if (state.emit_row_numbers) { scan_state.base_row_number = state.base_row_number.GetIndex(); } + + // if (state.emit_row_numbers) { + // scan_state.base_row_number = state.base_row_number.GetIndex(); + // if (!state.initial_base_row_number.IsValid()) { + // scan_state.base_row_number = state.base_row_number.GetIndex(); + // } + // else { + // state.base_row_number = state.initial_base_row_number; + // scan_state.base_row_number = state.base_row_number.GetIndex(); + // state.initial_base_row_number.SetInvalid(); + // state.has_emitted_row_numbers = true; + // } + // } + + // if (state.emit_row_numbers) { + // // First batch of this scan + // if (!state.initial_base_row_number.IsValid()) { + // // First thread entering parallel scan sets the initial base + // state.initial_base_row_number = state.base_row_number; + // } + // + // // Current batch uses the base row number + // scan_state.base_row_number = state.base_row_number.GetIndex(); + // } + + + if (ClientConfig::GetConfig(context).verify_parallelism) { vector_index = state.vector_index; max_row = row_start + MinValue(current_row_group.count, @@ -290,10 +317,7 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec state.current_row_group = state.GetNextRowGroup(*state.row_groups, *row_group).get(); // FIXME: this should not be GetCommittedRowCount but use the transaction id if (state.emit_row_numbers) { - - idx_t start = state.base_row_number.GetIndex(); - idx_t committed_row_count = current_row_group.GetCommittedRowCount(scan_state.transaction.start_time, scan_state.transaction.transaction_id); - state.base_row_number = start + committed_row_count; + state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(scan_state.transaction.start_time, scan_state.transaction.transaction_id); Printer::PrintF("New base row number %d", state.base_row_number.GetIndex()); } } diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index f702c20b8cb4..a0ac5a7410d0 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -102,10 +102,10 @@ # JOIN (SELECT row_number() OVER () AS row_num, b FROM bar) b ON f.row_num = b.row_num; # ---- # physical_plan :.*WINDOW.* - +# # statement ok # pragma disable_optimizer; - +# # statement ok # create table tbl as select range a from range(3); # @@ -116,20 +116,13 @@ # EXPLAIN SELECT row_number + 1 FROM (SELECT row_number() OVER () AS row_number FROM tbl); # ---- # physical_plan :.*WINDOW.* - -# Try with multiple connections - -# statement ok -# pragma disabled_optimizers = "window_rewriter" - -# statement ok -# pragma threads=1 +# +# +# # Try with multiple connections # statement ok # SET immediate_transaction_mode=true; # -# mode output_result -# # statement ok # CREATE TABLE big_tbl AS FROM range(1_000_000) t(i); # @@ -152,49 +145,54 @@ # ) t(r) # ---- # 1 500000 250000.5 +# +# statement ok con1 +# COMMIT -# mode skip # statement ok -# pragma threads=1 +# pragma disabled_optimizers="window_rewriter"; statement ok -SET immediate_transaction_mode=true; +pragma threads=1 statement ok -CREATE TABLE big_tbl AS FROM range(1_000_000) t(i); +CREATE OR REPLACE TABLE big_tbl AS FROM range(1_000_000) t(i); + +mode output_result statement ok con1 BEGIN -statement ok con2 +statement ok con1 INSERT INTO big_tbl SELECT range FROM range(1_000_000); -statement ok con2 -DELETE FROM big_tbl WHERE i%2=0; +# statement ok con1 +# DELETE FROM big_tbl WHERE i%2=0; query III con1 SELECT MIN(r), MAX(r), AVG(r) FROM ( SELECT row_number() OVER () FROM big_tbl ) t(r) ---- -1 1000000 500000.5 - -query III con2 -SELECT MIN(r), MAX(r), AVG(r) FROM ( - SELECT row_number() OVER () FROM big_tbl -) t(r) ----- -1 1000000 500000.5 - -statement ok con1 -commit; - -query III -SELECT MIN(r), MAX(r), AVG(r) FROM ( - SELECT row_number() OVER () FROM big_tbl -) t(r) ----- -1 1000000 500000.5 - +1 2000000 1000000 +# statement ok con2 +# DELETE FROM big_tbl WHERE i%2=0; +# # +# query III con2 +# SELECT MIN(r), MAX(r), AVG(r) FROM ( +# SELECT row_number() OVER () FROM big_tbl +# ) t(r) +# ---- +# 1 500000 250000.5 +# +# statement ok con1 +# commit; +# +# query III +# SELECT MIN(r), MAX(r), AVG(r) FROM ( +# SELECT row_number() OVER () FROM big_tbl +# ) t(r) +# ---- +# 1 1500000 750000.5 \ No newline at end of file From 37604258198fffbbeee50e57284f69d5e807ca5c Mon Sep 17 00:00:00 2001 From: Tmonster Date: Mon, 8 Dec 2025 14:12:26 +0100 Subject: [PATCH 23/27] do not use l_state base row number if you are still reading committed rows --- src/function/table/table_scan.cpp | 17 +++++--- .../duckdb/storage/table/scan_state.hpp | 1 + src/storage/data_table.cpp | 8 ++-- src/storage/table/chunk_info.cpp | 41 +++++++++---------- src/storage/table/row_group_collection.cpp | 2 +- test/optimizer/row_number_virtual_column.test | 28 ++++++------- 6 files changed, 50 insertions(+), 47 deletions(-) diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index 83e45990013a..ffda447bf2e1 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -342,14 +342,19 @@ class DuckTableScanState : public TableScanGlobalState { auto row_number_data = FlatVector::GetData(row_number_vec); auto count = output.size(); + if (l_state.scan_state.local_state.base_row_number == 0) { + auto break_here = 0; + } idx_t base; - if (state.local_state.has_emitted_row_numbers) { - base = l_state.scan_state.local_state.base_row_number + l_state.row_number_count; - // base = state.scan_state.base_row_number.GetIndex() + l_state.row_number_count; - } else { - base = l_state.scan_state.table_state.base_row_number + l_state.row_number_count; + if (state.local_state.has_emitted_row_numbers && l_state.scan_state.local_state.base_row_number == 0) { + auto break_here = 0; } - // idx_t base = l_state.scan_state.local_state.base_row_number + l_state.row_number_count; + // if (state.local_state.has_emitted_row_numbers && l_state.scan_state.local_state.base_row_number >= l_state.scan_state.table_state.base_row_number) { + // base = l_state.scan_state.local_state.base_row_number + l_state.row_number_count; + // } else { + // base = l_state.scan_state.table_state.base_row_number + l_state.row_number_count; + // } + base = l_state.scan_state.local_state.base_row_number + l_state.row_number_count; for (idx_t i = 0; i < count; i++) { row_number_data[i] = static_cast(base + i + 1); diff --git a/src/include/duckdb/storage/table/scan_state.hpp b/src/include/duckdb/storage/table/scan_state.hpp index f82c79c5417c..b754f83f221c 100644 --- a/src/include/duckdb/storage/table/scan_state.hpp +++ b/src/include/duckdb/storage/table/scan_state.hpp @@ -269,6 +269,7 @@ class TableScanState { CollectionScanState table_state; //! Transaction-local scan state CollectionScanState local_state; + bool have_emitted_local_row_numbers = false; //! Options for scanning TableScanOptions options; //! Shared lock over the checkpoint to prevent checkpoints while reading diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 06392fbf7ad4..7c602a8ebb75 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -291,11 +291,11 @@ bool DataTable::NextParallelScan(ClientContext &context, ParallelTableScanState if (!state.local_state.has_emitted_row_numbers) { state.local_state.base_row_number = state.scan_state.base_row_number; state.local_state.has_emitted_row_numbers = true; + scan_state.have_emitted_local_row_numbers = true; + } + if (state.local_state.has_emitted_row_numbers && !scan_state.have_emitted_local_row_numbers) { + auto break_here = 0; } - // if (!state.local_state.has_emitted_row_numbers) { - // state.local_state.base_row_number = state.scan_state.base_row_number; - // state.local_state.has_emitted_row_numbers = true; - // } if (local_storage.NextParallelScan(context, *this, state.local_state, scan_state.local_state)) { return true; } else { diff --git a/src/storage/table/chunk_info.cpp b/src/storage/table/chunk_info.cpp index 68691b48fa4b..36b3a571caa0 100644 --- a/src/storage/table/chunk_info.cpp +++ b/src/storage/table/chunk_info.cpp @@ -514,31 +514,30 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(transaction_t start_time, transa if (!AnyDeleted()) { // we wil come back to this, potentially inserted rows are deleted? Unsure - // if (HasConstantInsertionId()) { - // if (ConstantInsertId() < start_time || ConstantInsertId() == transaction_id){ - // return max_count; - // } - // } + if (HasConstantInsertionId()) { + if (ConstantInsertId() < start_time || ConstantInsertId() == transaction_id){ + return max_count; + } + } return 0; } - // - // auto segment = allocator.GetHandle(GetDeletedPointer()); - // auto deleted = segment.GetPtr(); - // - // idx_t delete_count = 0; - // for (idx_t i = 0; i < max_count; i++) { - // // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started - // if (!(deleted[i] < start_time || deleted[i] == transaction_id) ) { - // delete_count++; - // } - // } - // return delete_count; + auto segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = segment.GetPtr(); - auto sel_vec = SelectionVector(); - auto wat = TemplatedGetSelVector(start_time, transaction_id, sel_vec, max_count); - auto break_here = 0; - return wat; + idx_t delete_count = 0; + for (idx_t i = 0; i < max_count; i++) { + // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started + if (!(deleted[i] < start_time || deleted[i] == transaction_id) ) { + delete_count++; + } + } + return delete_count; + + // auto sel_vec = SelectionVector(); + // auto wat = TemplatedGetSelVector(start_time, transaction_id, sel_vec, max_count); + // auto break_here = 0; + // return wat; // auto segment = allocator.GetHandle(GetDeletedPointer()); // auto deleted = segment.GetPtr(); // diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index 590e47d0eca7..999d41233446 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -341,7 +341,7 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec } else { auto break_here = 0; state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(scan_state.transaction.start_time, scan_state.transaction.transaction_id); - Printer::PrintF("New base row number in else %d", state.base_row_number.GetIndex()); + // Printer::PrintF("New base row number in else %d", state.base_row_number.GetIndex()); } } max_row = MinValue(max_row, state.max_row); diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index a0ac5a7410d0..129bac0c8a91 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -154,39 +154,37 @@ # pragma disabled_optimizers="window_rewriter"; statement ok -pragma threads=1 +pragma threads=2; statement ok CREATE OR REPLACE TABLE big_tbl AS FROM range(1_000_000) t(i); -mode output_result - statement ok con1 BEGIN statement ok con1 INSERT INTO big_tbl SELECT range FROM range(1_000_000); +statement ok con1 +DELETE FROM big_tbl WHERE i%2=0; + +# query III con1 +# SELECT MIN(r), MAX(r), AVG(r) FROM ( +# SELECT row_number() OVER () FROM big_tbl +# ) t(r) +# ---- +# 1 2000000 1000000.5 + # statement ok con1 # DELETE FROM big_tbl WHERE i%2=0; - +# query III con1 SELECT MIN(r), MAX(r), AVG(r) FROM ( SELECT row_number() OVER () FROM big_tbl ) t(r) ---- -1 2000000 1000000 +1 1000000 500000.5 -# statement ok con2 -# DELETE FROM big_tbl WHERE i%2=0; -# # -# query III con2 -# SELECT MIN(r), MAX(r), AVG(r) FROM ( -# SELECT row_number() OVER () FROM big_tbl -# ) t(r) -# ---- -# 1 500000 250000.5 -# # statement ok con1 # commit; # From b2a7486ee57305e39973ff3478e070ce7f54e36c Mon Sep 17 00:00:00 2001 From: Tmonster Date: Mon, 8 Dec 2025 16:38:26 +0100 Subject: [PATCH 24/27] some more changes --- src/function/table/table_scan.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index ffda447bf2e1..01601fb47ae3 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -320,7 +320,7 @@ class DuckTableScanState : public TableScanGlobalState { auto &l_state = data_p.local_state->Cast(); l_state.scan_state.options.force_fetch_row = ClientConfig::GetConfig(context).force_fetch_row; - // bool use_local_state = state.local_state.has_emitted_row_numbers; + bool use_local = state.local_state.has_emitted_row_numbers; do { if (context.interrupted) { throw InterruptException(); @@ -346,15 +346,17 @@ class DuckTableScanState : public TableScanGlobalState { auto break_here = 0; } idx_t base; - if (state.local_state.has_emitted_row_numbers && l_state.scan_state.local_state.base_row_number == 0) { + if (use_local && l_state.scan_state.local_state.base_row_number == 0) { + auto break_here = 0; + } + if (use_local && l_state.scan_state.local_state.base_row_number >= l_state.scan_state.table_state.base_row_number) { + base = l_state.scan_state.local_state.base_row_number + l_state.row_number_count; + } else { + base = l_state.scan_state.table_state.base_row_number + l_state.row_number_count; + } + if (l_state.scan_state.table_state.base_row_number + l_state.row_number_count > 500000) { auto break_here = 0; } - // if (state.local_state.has_emitted_row_numbers && l_state.scan_state.local_state.base_row_number >= l_state.scan_state.table_state.base_row_number) { - // base = l_state.scan_state.local_state.base_row_number + l_state.row_number_count; - // } else { - // base = l_state.scan_state.table_state.base_row_number + l_state.row_number_count; - // } - base = l_state.scan_state.local_state.base_row_number + l_state.row_number_count; for (idx_t i = 0; i < count; i++) { row_number_data[i] = static_cast(base + i + 1); From 395cb38cd7be9eb1e819df98f8815a96af1c8e0d Mon Sep 17 00:00:00 2001 From: Tmonster Date: Mon, 8 Dec 2025 16:38:53 +0100 Subject: [PATCH 25/27] also add test --- test/optimizer/row_number_virtual_column.test | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index 129bac0c8a91..b5a032d2f689 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -154,18 +154,18 @@ # pragma disabled_optimizers="window_rewriter"; statement ok -pragma threads=2; +pragma threads=1; statement ok CREATE OR REPLACE TABLE big_tbl AS FROM range(1_000_000) t(i); -statement ok con1 -BEGIN +# statement ok con1 +# BEGIN -statement ok con1 +statement ok INSERT INTO big_tbl SELECT range FROM range(1_000_000); -statement ok con1 +statement ok DELETE FROM big_tbl WHERE i%2=0; # query III con1 @@ -178,7 +178,7 @@ DELETE FROM big_tbl WHERE i%2=0; # statement ok con1 # DELETE FROM big_tbl WHERE i%2=0; # -query III con1 +query III SELECT MIN(r), MAX(r), AVG(r) FROM ( SELECT row_number() OVER () FROM big_tbl ) t(r) From 754afd3026d36608d31511dabf902b79261425ca Mon Sep 17 00:00:00 2001 From: Tmonster Date: Mon, 8 Dec 2025 17:10:21 +0100 Subject: [PATCH 26/27] fix merge conflicts --- src/common/enum_util.cpp | 91 +++++--------------------------------- src/storage/data_table.cpp | 2 +- 2 files changed, 12 insertions(+), 81 deletions(-) diff --git a/src/common/enum_util.cpp b/src/common/enum_util.cpp index 5c1e0c1ca856..2fa1bde3d553 100644 --- a/src/common/enum_util.cpp +++ b/src/common/enum_util.cpp @@ -2888,73 +2888,6 @@ MetricGroup EnumUtil::FromString(const char *value) { const StringUtil::EnumStringLiteral *GetMetricTypeValues() { static constexpr StringUtil::EnumStringLiteral values[] { - { static_cast(MetricsType::ATTACH_LOAD_STORAGE_LATENCY), "ATTACH_LOAD_STORAGE_LATENCY" }, - { static_cast(MetricsType::ATTACH_REPLAY_WAL_LATENCY), "ATTACH_REPLAY_WAL_LATENCY" }, - { static_cast(MetricsType::BLOCKED_THREAD_TIME), "BLOCKED_THREAD_TIME" }, - { static_cast(MetricsType::CHECKPOINT_LATENCY), "CHECKPOINT_LATENCY" }, - { static_cast(MetricsType::CPU_TIME), "CPU_TIME" }, - { static_cast(MetricsType::CUMULATIVE_CARDINALITY), "CUMULATIVE_CARDINALITY" }, - { static_cast(MetricsType::CUMULATIVE_ROWS_SCANNED), "CUMULATIVE_ROWS_SCANNED" }, - { static_cast(MetricsType::EXTRA_INFO), "EXTRA_INFO" }, - { static_cast(MetricsType::LATENCY), "LATENCY" }, - { static_cast(MetricsType::OPERATOR_CARDINALITY), "OPERATOR_CARDINALITY" }, - { static_cast(MetricsType::OPERATOR_NAME), "OPERATOR_NAME" }, - { static_cast(MetricsType::OPERATOR_ROWS_SCANNED), "OPERATOR_ROWS_SCANNED" }, - { static_cast(MetricsType::OPERATOR_TIMING), "OPERATOR_TIMING" }, - { static_cast(MetricsType::OPERATOR_TYPE), "OPERATOR_TYPE" }, - { static_cast(MetricsType::QUERY_NAME), "QUERY_NAME" }, - { static_cast(MetricsType::RESULT_SET_SIZE), "RESULT_SET_SIZE" }, - { static_cast(MetricsType::ROWS_RETURNED), "ROWS_RETURNED" }, - { static_cast(MetricsType::SYSTEM_PEAK_BUFFER_MEMORY), "SYSTEM_PEAK_BUFFER_MEMORY" }, - { static_cast(MetricsType::SYSTEM_PEAK_TEMP_DIR_SIZE), "SYSTEM_PEAK_TEMP_DIR_SIZE" }, - { static_cast(MetricsType::TOTAL_BYTES_READ), "TOTAL_BYTES_READ" }, - { static_cast(MetricsType::TOTAL_BYTES_WRITTEN), "TOTAL_BYTES_WRITTEN" }, - { static_cast(MetricsType::TOTAL_MEMORY_ALLOCATED), "TOTAL_MEMORY_ALLOCATED" }, - { static_cast(MetricsType::WAITING_TO_ATTACH_LATENCY), "WAITING_TO_ATTACH_LATENCY" }, - { static_cast(MetricsType::COMMIT_LOCAL_STORAGE_LATENCY), "COMMIT_LOCAL_STORAGE_LATENCY" }, - { static_cast(MetricsType::WRITE_TO_WAL_LATENCY), "WRITE_TO_WAL_LATENCY" }, - { static_cast(MetricsType::WAL_REPLAY_ENTRY_COUNT), "WAL_REPLAY_ENTRY_COUNT" }, - { static_cast(MetricsType::ALL_OPTIMIZERS), "ALL_OPTIMIZERS" }, - { static_cast(MetricsType::CUMULATIVE_OPTIMIZER_TIMING), "CUMULATIVE_OPTIMIZER_TIMING" }, - { static_cast(MetricsType::PHYSICAL_PLANNER), "PHYSICAL_PLANNER" }, - { static_cast(MetricsType::PHYSICAL_PLANNER_COLUMN_BINDING), "PHYSICAL_PLANNER_COLUMN_BINDING" }, - { static_cast(MetricsType::PHYSICAL_PLANNER_CREATE_PLAN), "PHYSICAL_PLANNER_CREATE_PLAN" }, - { static_cast(MetricsType::PHYSICAL_PLANNER_RESOLVE_TYPES), "PHYSICAL_PLANNER_RESOLVE_TYPES" }, - { static_cast(MetricsType::PLANNER), "PLANNER" }, - { static_cast(MetricsType::PLANNER_BINDING), "PLANNER_BINDING" }, - { static_cast(MetricsType::OPTIMIZER_EXPRESSION_REWRITER), "OPTIMIZER_EXPRESSION_REWRITER" }, - { static_cast(MetricsType::OPTIMIZER_FILTER_PULLUP), "OPTIMIZER_FILTER_PULLUP" }, - { static_cast(MetricsType::OPTIMIZER_FILTER_PUSHDOWN), "OPTIMIZER_FILTER_PUSHDOWN" }, - { static_cast(MetricsType::OPTIMIZER_EMPTY_RESULT_PULLUP), "OPTIMIZER_EMPTY_RESULT_PULLUP" }, - { static_cast(MetricsType::OPTIMIZER_CTE_FILTER_PUSHER), "OPTIMIZER_CTE_FILTER_PUSHER" }, - { static_cast(MetricsType::OPTIMIZER_REGEX_RANGE), "OPTIMIZER_REGEX_RANGE" }, - { static_cast(MetricsType::OPTIMIZER_IN_CLAUSE), "OPTIMIZER_IN_CLAUSE" }, - { static_cast(MetricsType::OPTIMIZER_JOIN_ORDER), "OPTIMIZER_JOIN_ORDER" }, - { static_cast(MetricsType::OPTIMIZER_DELIMINATOR), "OPTIMIZER_DELIMINATOR" }, - { static_cast(MetricsType::OPTIMIZER_UNNEST_REWRITER), "OPTIMIZER_UNNEST_REWRITER" }, - { static_cast(MetricsType::OPTIMIZER_UNUSED_COLUMNS), "OPTIMIZER_UNUSED_COLUMNS" }, - { static_cast(MetricsType::OPTIMIZER_STATISTICS_PROPAGATION), "OPTIMIZER_STATISTICS_PROPAGATION" }, - { static_cast(MetricsType::OPTIMIZER_COMMON_SUBEXPRESSIONS), "OPTIMIZER_COMMON_SUBEXPRESSIONS" }, - { static_cast(MetricsType::OPTIMIZER_COMMON_AGGREGATE), "OPTIMIZER_COMMON_AGGREGATE" }, - { static_cast(MetricsType::OPTIMIZER_COLUMN_LIFETIME), "OPTIMIZER_COLUMN_LIFETIME" }, - { static_cast(MetricsType::OPTIMIZER_BUILD_SIDE_PROBE_SIDE), "OPTIMIZER_BUILD_SIDE_PROBE_SIDE" }, - { static_cast(MetricsType::OPTIMIZER_LIMIT_PUSHDOWN), "OPTIMIZER_LIMIT_PUSHDOWN" }, - { static_cast(MetricsType::OPTIMIZER_ROW_GROUP_PRUNER), "OPTIMIZER_ROW_GROUP_PRUNER" }, - { static_cast(MetricsType::OPTIMIZER_TOP_N), "OPTIMIZER_TOP_N" }, - { static_cast(MetricsType::OPTIMIZER_TOP_N_WINDOW_ELIMINATION), "OPTIMIZER_TOP_N_WINDOW_ELIMINATION" }, - { static_cast(MetricsType::OPTIMIZER_COMPRESSED_MATERIALIZATION), "OPTIMIZER_COMPRESSED_MATERIALIZATION" }, - { static_cast(MetricsType::OPTIMIZER_DUPLICATE_GROUPS), "OPTIMIZER_DUPLICATE_GROUPS" }, - { static_cast(MetricsType::OPTIMIZER_REORDER_FILTER), "OPTIMIZER_REORDER_FILTER" }, - { static_cast(MetricsType::OPTIMIZER_SAMPLING_PUSHDOWN), "OPTIMIZER_SAMPLING_PUSHDOWN" }, - { static_cast(MetricsType::OPTIMIZER_JOIN_FILTER_PUSHDOWN), "OPTIMIZER_JOIN_FILTER_PUSHDOWN" }, - { static_cast(MetricsType::OPTIMIZER_EXTENSION), "OPTIMIZER_EXTENSION" }, - { static_cast(MetricsType::OPTIMIZER_MATERIALIZED_CTE), "OPTIMIZER_MATERIALIZED_CTE" }, - { static_cast(MetricsType::OPTIMIZER_SUM_REWRITER), "OPTIMIZER_SUM_REWRITER" }, - { static_cast(MetricsType::OPTIMIZER_LATE_MATERIALIZATION), "OPTIMIZER_LATE_MATERIALIZATION" }, - { static_cast(MetricsType::OPTIMIZER_CTE_INLINING), "OPTIMIZER_CTE_INLINING" }, - { static_cast(MetricsType::OPTIMIZER_COMMON_SUBPLAN), "OPTIMIZER_COMMON_SUBPLAN" }, - { static_cast(MetricsType::OPTIMIZER_JOIN_ELIMINATION), "OPTIMIZER_JOIN_ELIMINATION" }, - { static_cast(MetricsType::OPTIMIZER_WINDOW_REWRITER), "OPTIMIZER_WINDOW_REWRITER" } { static_cast(MetricType::CPU_TIME), "CPU_TIME" }, { static_cast(MetricType::CUMULATIVE_CARDINALITY), "CUMULATIVE_CARDINALITY" }, { static_cast(MetricType::CUMULATIVE_ROWS_SCANNED), "CUMULATIVE_ROWS_SCANNED" }, @@ -3013,6 +2946,7 @@ const StringUtil::EnumStringLiteral *GetMetricTypeValues() { { static_cast(MetricType::OPTIMIZER_CTE_INLINING), "OPTIMIZER_CTE_INLINING" }, { static_cast(MetricType::OPTIMIZER_COMMON_SUBPLAN), "OPTIMIZER_COMMON_SUBPLAN" }, { static_cast(MetricType::OPTIMIZER_JOIN_ELIMINATION), "OPTIMIZER_JOIN_ELIMINATION" }, + { static_cast(MetricType::OPTIMIZER_WINDOW_REWRITER), "OPTIMIZER_WINDOW_REWRITER" }, { static_cast(MetricType::ALL_OPTIMIZERS), "ALL_OPTIMIZERS" }, { static_cast(MetricType::CUMULATIVE_OPTIMIZER_TIMING), "CUMULATIVE_OPTIMIZER_TIMING" }, { static_cast(MetricType::PHYSICAL_PLANNER), "PHYSICAL_PLANNER" }, @@ -3025,19 +2959,6 @@ const StringUtil::EnumStringLiteral *GetMetricTypeValues() { return values; } -template<> -const char* EnumUtil::ToChars(MetricsType value) { - return StringUtil::EnumToString(GetMetricsTypeValues(), 67, "MetricsType", static_cast(value)); -const char* EnumUtil::ToChars(MetricType value) { - return StringUtil::EnumToString(GetMetricTypeValues(), 66, "MetricType", static_cast(value)); -} - -template<> -MetricsType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetMetricsTypeValues(), 67, "MetricsType", value)); -MetricType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetMetricTypeValues(), 66, "MetricType", value)); -} const StringUtil::EnumStringLiteral *GetMultiFileColumnMappingModeValues() { static constexpr StringUtil::EnumStringLiteral values[] { @@ -3047,11 +2968,21 @@ const StringUtil::EnumStringLiteral *GetMultiFileColumnMappingModeValues() { return values; } +template<> +MetricType EnumUtil::FromString(const char *value) { + return static_cast(StringUtil::StringToEnum(GetMetricTypeValues(), 67, "MetricType", value)); +} + template<> const char* EnumUtil::ToChars(MultiFileColumnMappingMode value) { return StringUtil::EnumToString(GetMultiFileColumnMappingModeValues(), 2, "MultiFileColumnMappingMode", static_cast(value)); } +template<> +const char* EnumUtil::ToChars(MetricType value) { + return StringUtil::EnumToString(GetMetricTypeValues(), 67, "MetricType", static_cast(value)); +} + template<> MultiFileColumnMappingMode EnumUtil::FromString(const char *value) { return static_cast(StringUtil::StringToEnum(GetMultiFileColumnMappingModeValues(), 2, "MultiFileColumnMappingMode", value)); diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 23ff0bf67af0..073f955e8e45 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -300,7 +300,7 @@ bool DataTable::NextParallelScan(ClientContext &context, ParallelTableScanState } } -void DataTable::Scan(DuckxTransaction &transaction, DataChunk &result, TableScanState &state) { +void DataTable::Scan(DuckTransaction &transaction, DataChunk &result, TableScanState &state) { // scan the persistent segments if (state.table_state.Scan(transaction, result)) { D_ASSERT(result.size() > 0); From efe3347741cfe2715a639738a9e1ba32bac04942 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Tue, 9 Dec 2025 09:27:07 +0100 Subject: [PATCH 27/27] most row number virtual column tests pass. Need to clean up a whole bunch of stuff now --- .../duckdb/storage/table/row_group.hpp | 1 - src/storage/table/chunk_info.cpp | 54 ++- src/storage/table/row_group_collection.cpp | 49 +-- src/storage/table/row_version_manager.cpp | 15 +- src/storage/table/scan_state.cpp | 2 +- test/optimizer/row_number_virtual_column.test | 322 +++++++++--------- 6 files changed, 206 insertions(+), 237 deletions(-) diff --git a/src/include/duckdb/storage/table/row_group.hpp b/src/include/duckdb/storage/table/row_group.hpp index 5fb7eca92e6c..0af5b2a7ec9f 100644 --- a/src/include/duckdb/storage/table/row_group.hpp +++ b/src/include/duckdb/storage/table/row_group.hpp @@ -242,7 +242,6 @@ class RowGroup : public SegmentBase { private: mutable mutex row_group_lock; - mutex row_number_group_lock; vector column_pointers; //! Whether or not each column is loaded (mutable because `const` can lazy load) mutable unique_ptr[]> is_loaded; diff --git a/src/storage/table/chunk_info.cpp b/src/storage/table/chunk_info.cpp index 58f35d0a061e..77471b71157c 100644 --- a/src/storage/table/chunk_info.cpp +++ b/src/storage/table/chunk_info.cpp @@ -70,18 +70,19 @@ template idx_t ChunkConstantInfo::TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector, idx_t max_count) const { if (OP::UseInsertedVersion(start_time, transaction_id, insert_id) && - OP::UseDeletedVersion(start_time, transaction_id, delete_id)) { + OP::UseDeletedVersion(start_time, transaction_id, delete_id)) { return max_count; } return 0; } template -idx_t ChunkConstantInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time, transaction_t transaction_id, idx_t max_count) const { +idx_t ChunkConstantInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time, transaction_t transaction_id, + idx_t max_count) const { if (OP::UseInsertedVersion(start_time, transaction_id, insert_id) && - OP::UseDeletedVersion(start_time, transaction_id, delete_id)) { + OP::UseDeletedVersion(start_time, transaction_id, delete_id)) { return max_count; - } + } return 0; } @@ -95,9 +96,10 @@ idx_t ChunkConstantInfo::GetCommittedSelVector(transaction_t min_start_id, trans return TemplatedGetSelVector(min_start_id, min_transaction_id, sel_vector, max_count); } -// idx_t ChunkConstantInfo::GetCommittedDeletedCount(transaction_t min_start_id, transaction_t min_transaction_id, idx_t max_count) { -// return TemplatedGetCommittedDeleteCount(min_start_id, min_transaction_id, max_count); -// } +idx_t ChunkConstantInfo::GetCommittedDeletedCount(transaction_t min_start_id, transaction_t min_transaction_id, + idx_t max_count) { + return TemplatedGetCommittedDeleteCount(min_start_id, min_transaction_id, max_count); +} idx_t ChunkConstantInfo::GetCheckpointRowCount(TransactionData transaction, idx_t max_count) { if (TransactionVersionOperator::UseInsertedVersion(transaction.start_time, transaction.transaction_id, insert_id)) { @@ -125,14 +127,15 @@ idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) const { } // FIXME Find out when we use that and change accordingly -idx_t ChunkConstantInfo::GetCommittedDeletedCount(transaction_t min_start_id, transaction_t min_transaction_id, idx_t max_count) { - // return delete_id < TRANSACTION_ID_START ? max_count : 0; - // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started - if ((delete_id < min_start_id || delete_id == min_transaction_id) ) { - return max_count; - } - return 0; -} +// idx_t ChunkConstantInfo::GetCommittedDeletedCount(transaction_t min_start_id, transaction_t min_transaction_id, idx_t +// max_count) { +// // return delete_id < TRANSACTION_ID_START ? max_count : 0; +// // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started +// if ((delete_id < min_start_id || delete_id == min_transaction_id) ) { +// return max_count; +// } +// return 0; +// } bool ChunkConstantInfo::Cleanup(transaction_t lowest_transaction) const { if (delete_id != NOT_DELETED_ID) { @@ -210,7 +213,6 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti for (idx_t i = 0; i < max_count; i++) { if (OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { sel_vector.set_index(count++, i); - count++; } } return count; @@ -224,7 +226,6 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti for (idx_t i = 0; i < max_count; i++) { if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i])) { sel_vector.set_index(count++, i); - count++; } } return count; @@ -241,7 +242,6 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i]) && OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { sel_vector.set_index(count++, i); - count++; } } return count; @@ -505,9 +505,9 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) const { return delete_count; } - template -idx_t ChunkVectorInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time, transaction_t transaction_id, idx_t max_count) const { +idx_t ChunkVectorInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time, transaction_t transaction_id, + idx_t max_count) const { if (HasConstantInsertionId()) { if (!AnyDeleted()) { // all tuples have the same inserted id: and no tuples were deleted @@ -538,7 +538,7 @@ idx_t ChunkVectorInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time idx_t delete_count = 0; for (idx_t i = 0; i < max_count; i++) { // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started - if ((deleted[i] < start_time || deleted[i] == transaction_id) ) { + if ((deleted[i] < start_time || deleted[i] == transaction_id)) { delete_count++; } } @@ -568,21 +568,20 @@ idx_t ChunkVectorInfo::TemplatedGetCommittedDeleteCount(transaction_t start_time auto deleted = delete_segment.GetPtr(); for (idx_t i = 0; i < max_count; i++) { if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i]) && - OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { + OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { // sel_vector.set_index(count++, i); count++; - } + } } return count; } idx_t ChunkVectorInfo::GetCommittedDeletedCount(transaction_t start_time, transaction_t transaction_id, - idx_t max_count) { - + idx_t max_count) { if (!AnyDeleted()) { // we wil come back to this, potentially inserted rows are deleted? Unsure if (HasConstantInsertionId()) { - if (ConstantInsertId() < start_time || ConstantInsertId() == transaction_id){ + if (ConstantInsertId() < start_time || ConstantInsertId() == transaction_id) { return max_count; } } @@ -595,7 +594,7 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(transaction_t start_time, transa idx_t delete_count = 0; for (idx_t i = 0; i < max_count; i++) { // A delete is visible to a snapshot if it was committed by another transaction after the snapshot started - if (!(deleted[i] < start_time || deleted[i] == transaction_id) ) { + if (!(deleted[i] < start_time || deleted[i] == transaction_id)) { delete_count++; } } @@ -627,7 +626,6 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(transaction_t start_time, transa // return wat; } - void ChunkVectorInfo::Write(WriteStream &writer) const { SelectionVector sel(STANDARD_VECTOR_SIZE); transaction_t start_time = TRANSACTION_ID_START - 1; diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index 7bc8a6f87fd9..512593b5dda9 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -269,32 +269,6 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec scan_state.base_row_number = state.base_row_number.GetIndex(); } - // if (state.emit_row_numbers) { - // scan_state.base_row_number = state.base_row_number.GetIndex(); - // if (!state.initial_base_row_number.IsValid()) { - // scan_state.base_row_number = state.base_row_number.GetIndex(); - // } - // else { - // state.base_row_number = state.initial_base_row_number; - // scan_state.base_row_number = state.base_row_number.GetIndex(); - // state.initial_base_row_number.SetInvalid(); - // state.has_emitted_row_numbers = true; - // } - // } - - // if (state.emit_row_numbers) { - // // First batch of this scan - // if (!state.initial_base_row_number.IsValid()) { - // // First thread entering parallel scan sets the initial base - // state.initial_base_row_number = state.base_row_number; - // } - // - // // Current batch uses the base row number - // scan_state.base_row_number = state.base_row_number.GetIndex(); - // } - - - if (ClientConfig::GetConfig(context).verify_parallelism) { vector_index = state.vector_index; max_row = row_start + MinValue(current_row_group.count, @@ -317,10 +291,11 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec state.current_row_group = state.GetNextRowGroup(*state.row_groups, *row_group).get(); // FIXME: this should not be GetCommittedRowCount but use the transaction id if (state.emit_row_numbers) { - idx_t start = state.base_row_number.GetIndex(); - idx_t committed_row_count = current_row_group.GetCommittedRowCount(scan_state.transaction.start_time, scan_state.transaction.transaction_id); + idx_t start = state.base_row_number.GetIndex(); + idx_t committed_row_count = current_row_group.GetCommittedRowCount( + scan_state.transaction.start_time, scan_state.transaction.transaction_id); state.base_row_number = start + committed_row_count; - Printer::PrintF("New base row number %d", state.base_row_number.GetIndex()); + // Printer::PrintF("New base row number %d", state.base_row_number.GetIndex()); // auto new_rn = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(); // auto rn = state.base_row_number.GetIndex(); // auto break_here = true; @@ -328,20 +303,18 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec // for (idx_t r = 0, i = 0; r < state.collection->row_group_size ; r += STANDARD_VECTOR_SIZE, i++) { // SelectionVector sel_vector; // idx_t current_row = i * STANDARD_VECTOR_SIZE; - // auto max_count = MinValue(STANDARD_VECTOR_SIZE, state.collection->row_group_size - current_row); - // auto row_num = state.base_row_number.GetIndex() + current_row_group.GetSelVector(scan_state.transaction, i, sel_vector, max_count); + // auto max_count = MinValue(STANDARD_VECTOR_SIZE, state.collection->row_group_size - + // current_row); auto row_num = state.base_row_number.GetIndex() + + // current_row_group.GetSelVector(scan_state.transaction, i, sel_vector, max_count); // state.base_row_number = row_num; // } - // SelectionVector sel_vector; // idx_t current_row = state.batch_index * STANDARD_VECTOR_SIZE; - // auto max_count = MinValue(STANDARD_VECTOR_SIZE, state.collection->row_group_size - current_row); - // state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetSelVector(scan_state.transaction, scan_state.vector_index, sel_vector, max_count); - } else { - auto break_here = 0; - state.base_row_number = state.base_row_number.GetIndex() + current_row_group.GetCommittedRowCount(scan_state.transaction.start_time, scan_state.transaction.transaction_id); - // Printer::PrintF("New base row number in else %d", state.base_row_number.GetIndex()); + // auto max_count = MinValue(STANDARD_VECTOR_SIZE, state.collection->row_group_size - + // current_row); state.base_row_number = state.base_row_number.GetIndex() + + // current_row_group.GetSelVector(scan_state.transaction, scan_state.vector_index, sel_vector, + // max_count); } } max_row = MinValue(max_row, state.max_row); diff --git a/src/storage/table/row_version_manager.cpp b/src/storage/table/row_version_manager.cpp index d3fa069fbb97..f6bb4f816bd3 100644 --- a/src/storage/table/row_version_manager.cpp +++ b/src/storage/table/row_version_manager.cpp @@ -47,18 +47,17 @@ idx_t RowVersionManager::GetCommittedDeletedCount(transaction_t start_time, tran auto &constant_vec_info = vector_info[row_batch]->Cast(); SelectionVector sel; TransactionData txn_data(transaction_id, start_time); - result = max_count - vector_info[row_batch]->GetSelVector(txn_data,sel, max_count); - // result = constant_vec_info.TemplatedGetSelVector(start_time, transaction_id, sel, result); + result = max_count - vector_info[row_batch]->GetSelVector(txn_data, sel, max_count); } else { - result = vector_info[row_batch]->GetCommittedDeletedCount(start_time, transaction_id, max_count); + result = + max_count - vector_info[row_batch]->GetCommittedDeletedCount(start_time, transaction_id, max_count); } - } deleted_count += result; -// ======= -// auto res = vector_info[i]->GetCommittedDeletedCount(start_time, transaction_id, max_count); -// deleted_count += res; -// >>>>>>> be30a58df64d5027e7de1cdbb40e81fe5df5ccfe + // ======= + // auto res = vector_info[i]->GetCommittedDeletedCount(start_time, transaction_id, max_count); + // deleted_count += res; + // >>>>>>> be30a58df64d5027e7de1cdbb40e81fe5df5ccfe // SelectionVector sel_vector; // TransactionData txn(transaction_id, start_time); // diff --git a/src/storage/table/scan_state.cpp b/src/storage/table/scan_state.cpp index 23c39b5c11eb..b99efeca40db 100644 --- a/src/storage/table/scan_state.cpp +++ b/src/storage/table/scan_state.cpp @@ -176,7 +176,7 @@ TableScanOptions &CollectionScanState::GetOptions() { } ParallelCollectionScanState::ParallelCollectionScanState() - : collection(nullptr), current_row_group(nullptr), base_row_number(0), emit_row_numbers(true), processed_rows(0) { + : collection(nullptr), current_row_group(nullptr), base_row_number(0), emit_row_numbers(false), processed_rows(0) { } optional_ptr> ParallelCollectionScanState::GetRootSegment(RowGroupSegmentTree &row_groups) const { diff --git a/test/optimizer/row_number_virtual_column.test b/test/optimizer/row_number_virtual_column.test index b5a032d2f689..d998c941c4c2 100644 --- a/test/optimizer/row_number_virtual_column.test +++ b/test/optimizer/row_number_virtual_column.test @@ -2,183 +2,183 @@ # description: Test row_number virtual column # group: [optimizer] -# statement ok -# CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(130000); -# -# # Check that row_number is not accessible -# statement error -# SELECT a, row_number FROM foo; -# ---- -# Binder Error: Referenced column "row_number" not found in FROM clause -# -# query I -# SELECT a as row_number FROM foo limit 3; -# ---- -# 0 -# 1 -# 2 -# -# query I -# select row_number() over() from foo where a < 3; -# ---- -# 1 -# 2 -# 3 -# -# statement ok -# delete from foo where a < 5000; -# -# query III -# select row_number() over(), a, rowid from foo order by a desc limit 1; -# ---- -# 125000 129999 129999 -# -# query III -# select a, rowid, row_number() over() from foo order by a desc limit 1; -# ---- -# 129999 129999 125000 -# -# query II -# explain select a,rowid, row_number() over() as x from foo order by a desc limit 1; -# ---- -# physical_plan :.*WINDOW.* -# -# statement ok -# select setseed(0.1); -# -# statement ok -# create or replace table test as select (random()*10)::INT a, (random()*5)::INT b from range(10); -# -# statement ok -# create or replace table test2 as select (random()*100)::INT c, (random()*5)::INT b from range(10); -# -# # Try with a window function and a Join -# -# query II -# explain SELECT t.row_num, t.a, t2.c FROM ( SELECT a, b, ROW_NUMBER() OVER () AS row_num FROM test) t JOIN test2 t2 ON t.b = t2.b limit 10; -# ---- -# physical_plan :.*WINDOW.* -# -# -# statement ok -# create table t1 as select range::FLOAT a, range b, range c, range d from range (3); -# -# query I -# SELECT CASE WHEN row_number() OVER () = 2 THEN 2222::BIGINT ELSE 1111::BIGINT END FROM t1; -# ---- -# 1111 -# 2222 -# 1111 -# -# statement ok -# CREATE OR REPLACE TABLE numbers AS SELECT range(1, 5) AS num; -# -# query II -# SELECT row_number() OVER(), unnest(num) FROM numbers; -# ---- -# 1 1 -# 1 2 -# 1 3 -# 1 4 -# -# query II -# SELECT row_number() OVER (), nums FROM (SELECT unnest(num) AS nums from numbers); -# ---- -# 1 1 -# 2 2 -# 3 3 -# 4 4 -# -# # Try with a join on the row_number -# -# statement ok -# CREATE OR REPLACE TABLE foo AS SELECT range a from range(3); -# -# statement ok -# CREATE OR REPLACE TABLE bar AS SELECT range b from range(2,5); -# -# query II -# EXPLAIN SELECT f.row_num AS foo_row, b.row_num AS bar_row, f.a, b.b FROM (SELECT row_number() OVER () AS row_num, a FROM foo) f -# JOIN (SELECT row_number() OVER () AS row_num, b FROM bar) b ON f.row_num = b.row_num; -# ---- -# physical_plan :.*WINDOW.* -# -# statement ok -# pragma disable_optimizer; -# -# statement ok -# create table tbl as select range a from range(3); -# -# statement ok -# pragma enable_optimizer; -# -# query II -# EXPLAIN SELECT row_number + 1 FROM (SELECT row_number() OVER () AS row_number FROM tbl); -# ---- -# physical_plan :.*WINDOW.* -# -# -# # Try with multiple connections +statement ok +CREATE OR REPLACE TABLE foo AS SELECT range a FROM range(130000); -# statement ok -# SET immediate_transaction_mode=true; -# -# statement ok -# CREATE TABLE big_tbl AS FROM range(1_000_000) t(i); -# -# statement ok con1 -# BEGIN -# -# statement ok con2 -# DELETE FROM big_tbl WHERE i%2=0; -# -# query III con1 -# SELECT MIN(r), MAX(r), AVG(r) FROM ( -# SELECT row_number() OVER () FROM big_tbl -# ) t(r) -# ---- -# 1 1000000 500000.5 -# -# query III con2 -# SELECT MIN(r), MAX(r), AVG(r) FROM ( -# SELECT row_number() OVER () FROM big_tbl -# ) t(r) -# ---- -# 1 500000 250000.5 -# -# statement ok con1 -# COMMIT +# Check that row_number is not accessible +statement error +SELECT a, row_number FROM foo; +---- +Binder Error: Referenced column "row_number" not found in FROM clause +query I +SELECT a as row_number FROM foo limit 3; +---- +0 +1 +2 -# statement ok -# pragma disabled_optimizers="window_rewriter"; +query I +select row_number() over() from foo where a < 3; +---- +1 +2 +3 statement ok -pragma threads=1; +delete from foo where a < 5000; + +query III +select row_number() over(), a, rowid from foo order by a desc limit 1; +---- +125000 129999 129999 + +query III +select a, rowid, row_number() over() from foo order by a desc limit 1; +---- +129999 129999 125000 + +query II +explain select a,rowid, row_number() over() as x from foo order by a desc limit 1; +---- +physical_plan :.*WINDOW.* statement ok -CREATE OR REPLACE TABLE big_tbl AS FROM range(1_000_000) t(i); +select setseed(0.1); -# statement ok con1 -# BEGIN +statement ok +create or replace table test as select (random()*10)::INT a, (random()*5)::INT b from range(10); statement ok -INSERT INTO big_tbl SELECT range FROM range(1_000_000); +create or replace table test2 as select (random()*100)::INT c, (random()*5)::INT b from range(10); + +# Try with a window function and a Join + +query II +explain SELECT t.row_num, t.a, t2.c FROM ( SELECT a, b, ROW_NUMBER() OVER () AS row_num FROM test) t JOIN test2 t2 ON t.b = t2.b limit 10; +---- +physical_plan :.*WINDOW.* + + +statement ok +create table t1 as select range::FLOAT a, range b, range c, range d from range (3); + +query I +SELECT CASE WHEN row_number() OVER () = 2 THEN 2222::BIGINT ELSE 1111::BIGINT END FROM t1; +---- +1111 +2222 +1111 + +statement ok +CREATE OR REPLACE TABLE numbers AS SELECT range(1, 5) AS num; + +query II +SELECT row_number() OVER(), unnest(num) FROM numbers; +---- +1 1 +1 2 +1 3 +1 4 + +query II +SELECT row_number() OVER (), nums FROM (SELECT unnest(num) AS nums from numbers); +---- +1 1 +2 2 +3 3 +4 4 + +# Try with a join on the row_number + +statement ok +CREATE OR REPLACE TABLE foo AS SELECT range a from range(3); + +statement ok +CREATE OR REPLACE TABLE bar AS SELECT range b from range(2,5); + +query II +EXPLAIN SELECT f.row_num AS foo_row, b.row_num AS bar_row, f.a, b.b FROM (SELECT row_number() OVER () AS row_num, a FROM foo) f +JOIN (SELECT row_number() OVER () AS row_num, b FROM bar) b ON f.row_num = b.row_num; +---- +physical_plan :.*WINDOW.* + +statement ok +pragma disable_optimizer; + +statement ok +create table tbl as select range a from range(3); + +statement ok +pragma enable_optimizer; + +query II +EXPLAIN SELECT row_number + 1 FROM (SELECT row_number() OVER () AS row_number FROM tbl); +---- +physical_plan :.*WINDOW.* + + +# Try with multiple connections statement ok +SET immediate_transaction_mode=true; + +statement ok +CREATE TABLE big_tbl AS FROM range(1_000_000) t(i); + +statement ok con1 +BEGIN + +statement ok con2 DELETE FROM big_tbl WHERE i%2=0; -# query III con1 -# SELECT MIN(r), MAX(r), AVG(r) FROM ( -# SELECT row_number() OVER () FROM big_tbl -# ) t(r) -# ---- -# 1 2000000 1000000.5 +query III con1 +SELECT MIN(r), MAX(r), AVG(r) FROM ( + SELECT row_number() OVER () FROM big_tbl +) t(r) +---- +1 1000000 500000.5 -# statement ok con1 -# DELETE FROM big_tbl WHERE i%2=0; +query III con2 +SELECT MIN(r), MAX(r), AVG(r) FROM ( + SELECT row_number() OVER () FROM big_tbl +) t(r) +---- +1 500000 250000.5 + +statement ok con1 +COMMIT + + +# statement ok +# pragma disabled_optimizers="window_rewriter"; + +statement ok +pragma threads=2; + +statement ok +CREATE OR REPLACE TABLE big_tbl AS FROM range(1_000_000) t(i); + +statement ok con1 +BEGIN + +statement ok con1 +INSERT INTO big_tbl SELECT range FROM range(1_000_000); + +query III con1 +SELECT MIN(r), MAX(r), AVG(r) FROM ( + SELECT row_number() OVER () FROM big_tbl +) t(r) +---- +1 2000000 1000000.5 + +statement ok con1 +DELETE FROM big_tbl WHERE i%2=0; # -query III +# # statement ok con1 +# # DELETE FROM big_tbl WHERE i%2=0; + +query III con1 SELECT MIN(r), MAX(r), AVG(r) FROM ( SELECT row_number() OVER () FROM big_tbl ) t(r)