diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index ffebc661..be607d5a 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,15 +14,15 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main # NOTE: manually update ref :( + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main secrets: inherit # required for writing to vcpkg binary cache with: # Main config extension_name: &ext_name delta - duckdb_version: &duckdb_ref v1.5.2 + duckdb_version: &duckdb_ref main ci_tools_version: &ci_tools_ref main enable_rust: true - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;linux_amd64_musl' + exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;linux_amd64_musl" # Config to write vcpkg binary cache extra_toolchains: ${{ github.event_name != 'pull_request' && ';downgraded_aws_cli;python3;' || 'python3' }} @@ -30,27 +30,26 @@ jobs: vcpkg_binary_sources: ${{ github.event_name != 'pull_request' && vars.VCPKG_BINARY_SOURCES || '' }} reduced_ci_mode: ${{ github.event_name == 'pull_request' && 'enabled' || 'disabled'}} - duckdb-stable-deploy: name: Deploy extension binaries needs: duckdb-stable-build - uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main # NOTE: manually update ref :( + uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main secrets: inherit with: extension_name: *ext_name duckdb_version: *duckdb_ref ci_tools_version: *ci_tools_ref - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;linux_amd64_musl' + exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;linux_amd64_musl" # deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} reduced_ci_mode: ${{ github.event_name == 'pull_request' && 'enabled' || 'disabled'}} code-quality-check: name: Code Quality Check - uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@main # NOTE: manually update ref :( + uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@main with: extension_name: *ext_name duckdb_version: *duckdb_ref ci_tools_version: *ci_tools_ref - format_checks: format - extra_toolchains: 'python3' + format_checks: format + extra_toolchains: "python3" diff --git a/duckdb b/duckdb index 8a585197..2daa4fc9 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 8a5851971fae891f292c2714d86046ee018e9737 +Subproject commit 2daa4fc9a48c638d10dcc613b1ed15d56c66f5b0 diff --git a/extension-ci-tools b/extension-ci-tools index ec20f45a..5cad6aa9 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit ec20f45aabeb9fcfcfa044dda249597f066d4826 +Subproject commit 5cad6aa9886a8ec6c9dbb772bed6fdae67c42d64 diff --git a/src/delta_extension.cpp b/src/delta_extension.cpp index acefa36c..15b35d0f 100644 --- a/src/delta_extension.cpp +++ b/src/delta_extension.cpp @@ -65,15 +65,17 @@ static unique_ptr DeltaCatalogAttach(optional_ptr string commit_fun_name = "__internal_delta_ccv2_commit_staged"; CatalogEntryRetriever retriever(context); - EntryLookupInfo lookup_info(CatalogType::TABLE_FUNCTION_ENTRY, commit_fun_name); - auto fun = retriever.GetEntry(res->parent_catalog_name, schema, lookup_info, OnEntryNotFound::RETURN_NULL); + EntryLookupInfo lookup_info( + CatalogType::TABLE_FUNCTION_ENTRY, + QualifiedName(Identifier(res->parent_catalog_name), Identifier(schema), Identifier(commit_fun_name))); + auto fun = retriever.GetEntry(lookup_info, OnEntryNotFound::RETURN_NULL); if (!fun) { throw InternalException("Parent catalog does not have a __internal_delta_ccv2_commit_staged function"); } res->commit_function = fun->Cast(); } - res->SetDefaultTable(DEFAULT_SCHEMA, res->GetInternalTableName()); + res->SetDefaultTable(Identifier::DefaultSchema(), Identifier(res->GetInternalTableName())); return std::move(res); } diff --git a/src/delta_macros.cpp b/src/delta_macros.cpp index c18f81d3..729bb116 100644 --- a/src/delta_macros.cpp +++ b/src/delta_macros.cpp @@ -58,16 +58,17 @@ void DeltaMacros::RegisterTableMacro(ExtensionLoader &loader, const string &name auto func = make_uniq(std::move(node)); for (auto ¶m : params) { - func->parameters.push_back(make_uniq(param)); + func->parameters.push_back(make_uniq(Identifier(param))); } for (auto ¶m : named_params) { - func->default_parameters[param.first] = make_uniq(param.second); + func->default_parameters[Identifier(param.first)] = make_uniq(Value(param.second)); } CreateMacroInfo info(CatalogType::TABLE_MACRO_ENTRY); - info.schema = DEFAULT_SCHEMA; - info.name = name; + info.SetQualifiedName( + QualifiedName(info.GetQualifiedName().Catalog(), Identifier::DefaultSchema(), info.GetQualifiedName().Name())); + info.SetFunctionName(Identifier(name)); info.temporary = true; info.internal = true; info.macros.push_back(std::move(func)); @@ -76,11 +77,9 @@ void DeltaMacros::RegisterTableMacro(ExtensionLoader &loader, const string &name } static DefaultMacro delta_macros[] = { - {DEFAULT_SCHEMA, - "parse_delta_filter_logline", - {"x", nullptr}, - {{nullptr, nullptr}}, - "x::STRUCT(path VARCHAR, type VARCHAR, filters_before VARCHAR[], filters_after VARCHAR[], files_before BIGINT, " + {DEFAULT_SCHEMA, "parse_delta_filter_logline", + "(x) AS x::STRUCT(path VARCHAR, type VARCHAR, filters_before VARCHAR[], filters_after VARCHAR[], files_before " + "BIGINT, " "files_after BIGINT)"}, }; diff --git a/src/delta_utils.cpp b/src/delta_utils.cpp index 48fdb79a..a0ef85cb 100644 --- a/src/delta_utils.cpp +++ b/src/delta_utils.cpp @@ -4,9 +4,17 @@ #include #include "delta_log_types.hpp" +#include "functions/delta_scan/delta_multi_file_list.hpp" #include "duckdb/common/operator/decimal_cast_operators.hpp" #include "duckdb.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_comparison_expression.hpp" +#include "duckdb/planner/expression/bound_conjunction_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/bound_operator_expression.hpp" +#include "duckdb/function/scalar/struct_utils.hpp" #include "duckdb/common/extension_type_info.hpp" #include "duckdb/common/types/decimal.hpp" #include "duckdb/main/database.hpp" @@ -279,7 +287,7 @@ void ExpressionVisitor::VisitStructLiteral(void *state, uintptr_t sibling_list_i } for (idx_t i = 0; i < children_keys->size(); i++) { - (*children_values)[i]->alias = (*children_keys)[i]->ToString(); + (*children_values)[i]->SetAlias(Identifier((*children_keys)[i]->ToString())); } unique_ptr expression = make_uniq("struct_pack", std::move(*children_values)); @@ -331,8 +339,8 @@ void ExpressionVisitor::VisitLiteralMap(void *state, uintptr_t sibling_list_id, ErrorData("DuckDB only supports parsing Map literals from delta kernel that consist for constants!"); return; } - key_values.push_back(key_field->Cast().value); - key_type = key_field->Cast().value.type(); + key_values.push_back(key_field->Cast().GetValue()); + key_type = key_field->Cast().GetValue().type(); } vector value_values; @@ -343,8 +351,8 @@ void ExpressionVisitor::VisitLiteralMap(void *state, uintptr_t sibling_list_id, ErrorData("DuckDB only supports parsing Map literals from delta kernel that consist for constants!"); return; } - value_values.push_back(value_field->Cast().value); - value_type = value_field->Cast().value.type(); + value_values.push_back(value_field->Cast().GetValue()); + value_type = value_field->Cast().GetValue().type(); } unique_ptr expression = @@ -423,7 +431,7 @@ void ExpressionVisitor::VisitColumnExpression(void *state, uintptr_t sibling_lis col_ref_string = col_ref_string.substr(1, col_ref_string.size() - 2); } - auto expression = make_uniq(col_ref_string); + auto expression = make_uniq(Identifier(col_ref_string)); static_cast(state)->AppendToList(sibling_list_id, std::move(expression)); } @@ -648,7 +656,7 @@ void SchemaVisitor::VisitStruct(SchemaVisitor *state, uintptr_t sibling_list_id, child_list_t children_types; for (const auto &child_col_def : children) { - children_types.push_back({child_col_def.name, child_col_def.type}); + children_types.emplace_back(child_col_def.name, child_col_def.type); } auto struct_type = LogicalType::STRUCT(children_types); @@ -734,7 +742,7 @@ void SchemaVisitor::AppendToList(uintptr_t id, ffi::KernelStringSlice name, Delt } // Inject the name - child.name = string(name.ptr, name.len); + child.name = Identifier(string(name.ptr, name.len)); it->second.emplace_back(std::move(child)); } @@ -915,7 +923,7 @@ string KernelUtils::FetchFromStringMap(ffi::Handle engi return val; } -vector> & +vector> KernelUtils::UnpackTransformExpression(const vector> &parsed_expression) { if (parsed_expression.size() != 1) { throw IOException("Unexpected size of transformation expression returned by delta kernel: %d", @@ -923,26 +931,33 @@ KernelUtils::UnpackTransformExpression(const vector } const auto &root_expression = parsed_expression.get(0); - if (root_expression->type != ExpressionType::FUNCTION) { - throw IOException("Unexpected type of root expression returned by delta kernel: %d", root_expression->type); + if (root_expression->GetExpressionType() != ExpressionType::FUNCTION) { + throw IOException("Unexpected type of root expression returned by delta kernel: %d", + root_expression->GetExpressionType()); } - if (root_expression->Cast().function_name != "delta_kernel_transform_expression") { + if (root_expression->Cast().FunctionName() != "delta_kernel_transform_expression") { throw IOException("Unexpected function of root expression returned by delta kernel: %s", - root_expression->Cast().function_name); + root_expression->Cast().FunctionName()); } - return root_expression->Cast().children; + vector> children; + for (const auto &child : root_expression->Cast().GetArguments()) { + children.push_back(child.GetExpression().Copy()); + } + return children; } PredicateVisitor::PredicateVisitor(const vector &columns, - optional_ptr filters) { + optional_ptr filters) { predicate = this; visitor = (uintptr_t(*)(void *, ffi::KernelExpressionVisitorState *)) & VisitPredicate; if (filters) { - for (auto &filter : filters->filters) { - column_filters[columns[filter.first].name] = filter.second.get(); + for (auto &entry : *filters) { + auto &column = columns[entry.first]; + column_filters[column.name.GetIdentifierName()] = entry.second.get(); + column_types[column.name.GetIdentifierName()] = column.type; } } } @@ -978,8 +993,8 @@ uintptr_t PredicateVisitor::VisitPredicate(PredicateVisitor *predicate, ffi::Ker return ffi::visit_predicate_and(state, &eit); } -uintptr_t PredicateVisitor::VisitConstantFilter(const string &col_name, const ConstantFilter &filter, - ffi::KernelExpressionVisitorState *state) { +uintptr_t PredicateVisitor::VisitConstantFilter(const string &col_name, ExpressionType comparison_type, + const Value &value, ffi::KernelExpressionVisitorState *state) { auto maybe_left = ffi::visit_expression_column(state, KernelUtils::ToDeltaString(col_name), DuckDBEngineError::AllocateError); @@ -991,7 +1006,6 @@ uintptr_t PredicateVisitor::VisitConstantFilter(const string &col_name, const Co } uintptr_t right = ~0; - auto &value = filter.constant; switch (value.type().id()) { case LogicalType::BIGINT: right = visit_expression_literal_long(state, BigIntValue::Get(value)); @@ -1071,7 +1085,7 @@ uintptr_t PredicateVisitor::VisitConstantFilter(const string &col_name, const Co } // TODO support other comparison types? - switch (filter.comparison_type) { + switch (comparison_type) { case ExpressionType::COMPARE_LESSTHAN: return visit_predicate_lt(state, left, right); case ExpressionType::COMPARE_LESSTHANOREQUALTO: @@ -1097,20 +1111,40 @@ uintptr_t PredicateVisitor::VisitConstantFilter(const string &col_name, const Co } } -uintptr_t PredicateVisitor::VisitAndFilter(const string &col_name, const ConjunctionAndFilter &filter, - ffi::KernelExpressionVisitorState *state) { - auto it = filter.child_filters.begin(); - auto end = filter.child_filters.end(); - auto get_next = [this, col_name, state, &it, &end]() -> uintptr_t { - if (it == end) { - return 0; +// Resolves the (possibly struct-nested) column being filtered to the dot-separated path the kernel +// expects. `base` is the top-level column name this filter is keyed on: a bare column subject yields +// `base`, while struct field access (struct_extract / struct_extract_at) appends the nested field +// names, e.g. base "i" with subject i.a.b -> "i.a.b". Returns false for unsupported subjects. +static bool ResolveFilterColumnPath(const Expression &expr, const string &base, string &result) { + switch (expr.GetExpressionClass()) { + case ExpressionClass::BOUND_REF: + case ExpressionClass::BOUND_COLUMN_REF: + result = base; + return true; + case ExpressionClass::BOUND_FUNCTION: { + auto &func = expr.Cast(); + const auto &name = func.Function().GetName(); + if (name != "struct_extract" && name != "struct_extract_at") { + return false; } - auto &child_filter = *it++; - - return VisitFilter(col_name, *child_filter, state); - }; - auto eit = EngineIteratorFromCallable(get_next); - return visit_predicate_and(state, &eit); + if (func.GetChildren().empty()) { + return false; + } + auto &struct_type = func.GetChildren()[0]->GetReturnType(); + idx_t child_idx; + if (struct_type.id() != LogicalTypeId::STRUCT || !TryGetStructExtractChildIndex(func, child_idx)) { + return false; + } + string parent; + if (!ResolveFilterColumnPath(*func.GetChildren()[0], base, parent)) { + return false; + } + result = parent + "." + StructType::GetChildName(struct_type, child_idx).GetIdentifierName(); + return true; + } + default: + return false; + } } uintptr_t PredicateVisitor::VisitIsNull(const string &col_name, ffi::KernelExpressionVisitorState *state) { @@ -1130,48 +1164,90 @@ uintptr_t PredicateVisitor::VisitIsNotNull(const string &col_name, ffi::KernelEx return ffi::visit_predicate_not(state, VisitIsNull(col_name, state)); } -uintptr_t PredicateVisitor::VisitStructExtractFilter(const string &col_name, const StructFilter &filter, - ffi::KernelExpressionVisitorState *state) { - // Build the full dot-separated path by recursing through nested StructFilters. - // E.g. col "i" with StructFilter{child_name="a", child_filter=StructFilter{child_name="b", leaf}} - // becomes "i.a.b". visit_expression_column splits on "." to construct the kernel ColumnName. - string full_path = col_name + "." + filter.child_name; - const TableFilter *child = filter.child_filter.get(); - while (child->filter_type == TableFilterType::STRUCT_EXTRACT) { - const auto &nested = static_cast(*child); - full_path += "." + nested.child_name; - child = nested.child_filter.get(); +uintptr_t PredicateVisitor::VisitFilterExpression(const string &col_name, const Expression &expr, + ffi::KernelExpressionVisitorState *state) { + // A filter subject that is a bare reference to a nested (struct/list/map) column can't be expressed to the kernel: + // the same predicate also arrives in struct_extract form (which resolves to the full leaf path, e.g. i.a.b), so + // the bare form (resolving only to the top-level column) is skipped. + auto type_entry = column_types.find(col_name); + bool base_is_nested = type_entry != column_types.end() && type_entry->second.IsNested(); + + if (BoundComparisonExpression::IsComparison(expr)) { + auto &comparison = expr.Cast(); + auto comparison_type = comparison.GetExpressionType(); + auto &left = BoundComparisonExpression::Left(comparison); + auto &right = BoundComparisonExpression::Right(comparison); + string path; + if (left.GetExpressionClass() == ExpressionClass::BOUND_CONSTANT && + ResolveFilterColumnPath(right, col_name, path) && !(path == col_name && base_is_nested)) { + return VisitConstantFilter(path, FlipComparisonExpression(comparison_type), + left.Cast().GetValue(), state); + } + if (right.GetExpressionClass() == ExpressionClass::BOUND_CONSTANT && + ResolveFilterColumnPath(left, col_name, path) && !(path == col_name && base_is_nested)) { + return VisitConstantFilter(path, comparison_type, right.Cast().GetValue(), state); + } + return ~0; } - return VisitFilter(full_path, *child, state); -} -uintptr_t PredicateVisitor::VisitFilter(const string &col_name, const TableFilter &filter, - ffi::KernelExpressionVisitorState *state) { - switch (filter.filter_type) { - case TableFilterType::CONSTANT_COMPARISON: - return VisitConstantFilter(col_name, static_cast(filter), state); - case TableFilterType::CONJUNCTION_AND: - return VisitAndFilter(col_name, static_cast(filter), state); - case TableFilterType::IS_NULL: - return VisitIsNull(col_name, state); - case TableFilterType::IS_NOT_NULL: - return VisitIsNotNull(col_name, state); - case TableFilterType::STRUCT_EXTRACT: - return VisitStructExtractFilter(col_name, static_cast(filter), state); - // TODO: implement once kernel can do arbitrary expressions - case TableFilterType::EXPRESSION_FILTER: - // TODO: implement once kernel adds support for IN filters / arbitrary expressions - case TableFilterType::IN_FILTER: - // TODO: figure out if this is ever useful - case TableFilterType::DYNAMIC_FILTER: - // TODO: can we even push these down? - case TableFilterType::CONJUNCTION_OR: - case TableFilterType::OPTIONAL_FILTER: + switch (expr.GetExpressionClass()) { + case ExpressionClass::BOUND_CONJUNCTION: { + auto &conjunction = expr.Cast(); + if (conjunction.GetExpressionType() != ExpressionType::CONJUNCTION_AND) { + return ~0; + } + auto it = conjunction.GetChildren().begin(); + auto end = conjunction.GetChildren().end(); + auto get_next = [this, col_name, state, &it, &end]() -> uintptr_t { + if (it == end) { + return 0; + } + return VisitFilterExpression(col_name, *(*it++), state); + }; + auto eit = EngineIteratorFromCallable(get_next); + return visit_predicate_and(state, &eit); + } + case ExpressionClass::BOUND_OPERATOR: { + auto &op = expr.Cast(); + string path; + if (op.GetChildren().size() != 1 || !ResolveFilterColumnPath(*op.GetChildren()[0], col_name, path) || + (path == col_name && base_is_nested)) { + return ~0; + } + if (op.GetExpressionType() == ExpressionType::OPERATOR_IS_NULL) { + return VisitIsNull(path, state); + } + if (op.GetExpressionType() == ExpressionType::OPERATOR_IS_NOT_NULL) { + return VisitIsNotNull(path, state); + } + return ~0; + } + case ExpressionClass::BOUND_FUNCTION: { + auto &func = expr.Cast(); + if (func.Function().GetName() == OptionalFilterScalarFun::NAME && func.BindInfo()) { + auto &data = func.BindInfo()->Cast(); + if (data.child_filter_expr) { + return VisitFilterExpression(col_name, *data.child_filter_expr, state); + } + } + if (func.Function().GetName() == SelectivityOptionalFilterScalarFun::NAME && func.BindInfo()) { + auto &data = func.BindInfo()->Cast(); + if (data.child_filter_expr) { + return VisitFilterExpression(col_name, *data.child_filter_expr, state); + } + } + return ~0; + } default: return ~0; } } +uintptr_t PredicateVisitor::VisitFilter(const string &col_name, const ExpressionFilter &filter, + ffi::KernelExpressionVisitorState *state) { + return VisitFilterExpression(col_name, *filter.expr, state); +} + void LoggerCallback::Initialize(DatabaseInstance &db_p) { auto &instance = GetInstance(); unique_lock lck(instance.lock); diff --git a/src/functions/delta_domain_metadata.cpp b/src/functions/delta_domain_metadata.cpp index 0e0eeeed..54dc9211 100644 --- a/src/functions/delta_domain_metadata.cpp +++ b/src/functions/delta_domain_metadata.cpp @@ -23,7 +23,7 @@ static unique_ptr DeltaDomainMetadataBind(ClientContext &context, idx_t version = DConstants::INVALID_INDEX; for (auto &kv : input.named_parameters) { - auto loption = StringUtil::Lower(kv.first); + auto loption = StringUtil::Lower(kv.first.GetIdentifierName()); if (loption == "version") { version = kv.second.GetValue(); } @@ -32,7 +32,7 @@ static unique_ptr DeltaDomainMetadataBind(ClientContext &context, auto file_list = make_uniq(context, input_string, version); // Trigger snapshot initialization - vector _n; + vector _n; vector _t; file_list->Bind(_t, _n); diff --git a/src/functions/delta_metadata_scan.cpp b/src/functions/delta_metadata_scan.cpp index 3ee43c4e..b31593dd 100644 --- a/src/functions/delta_metadata_scan.cpp +++ b/src/functions/delta_metadata_scan.cpp @@ -26,7 +26,7 @@ static void AddFileInfo(OpenFileInfo &file_info, DeltaFileMetaData &metadata, ve auto partitions = metadata.partition_map; InsertionOrderPreservingMap map; for (auto &kv : partitions) { - map[kv.first] = kv.second.ToString(); + map[kv.first.GetIdentifierName()] = kv.second.ToString(); } row_values.emplace_back(Value::MAP(map)); @@ -71,7 +71,7 @@ static unique_ptr DeltaFileListBind(ClientContext &context, TableF DeltaFileListOptions options; for (auto &kv : input.named_parameters) { - auto loption = StringUtil::Lower(kv.first); + auto loption = StringUtil::Lower(kv.first.GetIdentifierName()); auto &val = kv.second; if (loption == "version") { version = val.GetValue(); @@ -87,7 +87,7 @@ static unique_ptr DeltaFileListBind(ClientContext &context, TableF auto file_list = make_uniq(context, input_string, version); // TODO: this is weird - vector _n; + vector _n; vector _t; file_list->Bind(_t, _n); diff --git a/src/functions/delta_scan/delta_multi_file_list.cpp b/src/functions/delta_scan/delta_multi_file_list.cpp index 4ce9e205..87bf811b 100644 --- a/src/functions/delta_scan/delta_multi_file_list.cpp +++ b/src/functions/delta_scan/delta_multi_file_list.cpp @@ -18,6 +18,7 @@ #include "duckdb/parser/constraints/not_null_constraint.hpp" #include +#include #include "duckdb/planner/constraints/bound_not_null_constraint.hpp" @@ -346,30 +347,37 @@ static void KernelPartitionStringVisitor(ffi::NullableCvoid engine_context, ffi: static unordered_map FindPartitionValues(ParsedExpression &transformation, const vector &cols) { - if (transformation.Cast().function_name != "delta_kernel_transform_expression") { + if (transformation.Cast().FunctionName() != "delta_kernel_transform_expression") { throw IOException("Unexpected function of root expression returned by delta kernel: %s", - transformation.Cast().function_name); + transformation.Cast().FunctionName()); } unordered_map res; // Iterate the children of the transform - for (auto &child : transformation.Cast().children) { - auto &transform_op = child->Cast(); - if (transform_op.function_name != "delta_transform_op") { + for (auto &child : transformation.Cast().GetArguments()) { + auto &transform_op = child.GetExpression().Cast(); + if (transform_op.FunctionName() != "delta_transform_op") { throw IOException("Unexpected function for delta_transform_op returned by delta kernel: %s", - child->Cast().function_name); + child.GetExpression().Cast().FunctionName()); } bool is_replace = false; string field_name; vector values; - for (auto &transform_op_child : transform_op.children) { - if (transform_op_child->GetExpressionType() == ExpressionType::COMPARE_EQUAL) { - auto name = - transform_op_child->Cast().left->Cast().GetName(); - auto value = transform_op_child->Cast().right->Cast().value; + for (auto &transform_op_child : transform_op.GetArguments()) { + if (transform_op_child.GetExpression().GetExpressionType() == ExpressionType::COMPARE_EQUAL) { + auto name = transform_op_child.GetExpression() + .Cast() + .Left() + .Cast() + .GetName(); + auto value = transform_op_child.GetExpression() + .Cast() + .Right() + .Cast() + .GetValue(); if (name == "is_replace") { is_replace = value.GetValue(); @@ -381,12 +389,12 @@ static unordered_map FindPartitionValues(ParsedExpression &transfo throw InternalException("Unexpected name for delta_transform_op returned by delta kernel: %s", name); } - } else if (transform_op_child->GetExpressionType() == ExpressionType::VALUE_CONSTANT) { - values.push_back(transform_op_child->Cast().value); + } else if (transform_op_child.GetExpression().GetExpressionType() == ExpressionType::VALUE_CONSTANT) { + values.push_back(transform_op_child.GetExpression().Cast().GetValue()); } else { throw NotImplementedException( "Unexpected expression for delta_transform_op returned by delta kernel: %s", - transform_op_child->ToString()); + transform_op_child.GetExpression().ToString()); } } @@ -482,12 +490,12 @@ void ScanDataCallBack::VisitCallbackInternal(ffi::NullableCvoid engine_context, auto transform_partitions = FindPartitionValues(*(*parsed_transformation_expression)[0], snapshot.global_columns); - case_insensitive_map_t constant_map; + identifier_map_t constant_map; for (idx_t i = 0; i < snapshot.partitions.size(); ++i) { const auto &partition_id = context->snapshot.partition_ids[i]; const auto &partition_name = context->snapshot.partitions[i]; - constant_map[partition_name] = transform_partitions[partition_id]; + constant_map[Identifier(partition_name)] = transform_partitions[partition_id]; } snapshot.metadata.back()->partition_map = std::move(constant_map); snapshot.metadata.back()->transform_expression = @@ -604,12 +612,12 @@ static bool ExtractHasNullConstraintsInArrays(const vector &return_types, vector &names) { +void DeltaMultiFileList::Bind(vector &return_types, vector &names) { unique_lock lck(lock); if (have_bound) { for (const auto &field : global_columns) { - names.push_back(field.name); + names.push_back(Identifier(field.name)); return_types.push_back(field.type); } return; @@ -624,7 +632,7 @@ void DeltaMultiFileList::Bind(vector &return_types, vector } for (const auto &field : visited_schema) { - names.push_back(field.name); + names.push_back(Identifier(field.name)); return_types.push_back(field.type); } @@ -811,21 +819,24 @@ void DeltaMultiFileList::EnsureScanInitialized() const { } } -unique_ptr DeltaMultiFileList::PushdownInternal(ClientContext &context, - TableFilterSet &new_filters) const { +unique_ptr DeltaMultiFileList::PushdownInternal(ClientContext &context, TableFilterSet &new_filters, + vector column_indexes) const { auto filtered_list = make_uniq(context, paths[0].path, version); - TableFilterSet result_filter_set; + DeltaTableFilters result_filter_set; // Add pre-existing filters - for (auto &entry : table_filters.filters) { - result_filter_set.PushFilter(ColumnIndex(entry.first), entry.second->Copy()); + for (auto &entry : table_filters) { + result_filter_set.PushFilter(entry.first, entry.second->Copy()); } // Add new filters - for (auto &entry : new_filters.filters) { - if (entry.first < global_columns.size()) { - result_filter_set.PushFilter(ColumnIndex(entry.first), entry.second->Copy()); + for (auto &entry : new_filters) { + auto &column_id = column_indexes[entry.GetIndex()]; + if (column_id < global_columns.size()) { + auto &filter = + ExpressionFilter::GetExpressionFilter(entry.Filter(), "DeltaMultiFileList::PushdownInternal"); + result_filter_set.PushFilter(column_id, filter.Copy()); } } @@ -874,11 +885,11 @@ unique_ptr DeltaMultiFileList::ComplexFilterPushdown(ClientContex vector pushdown_results; auto filter_set = combiner.GenerateTableScanFilters(info.column_indexes, pushdown_results); - if (filter_set.filters.empty()) { + if (!filter_set.HasFilters()) { return nullptr; } - auto filtered_list = PushdownInternal(context, filter_set); + auto filtered_list = PushdownInternal(context, filter_set, info.column_ids); ReportFilterPushdown(context, *filtered_list, info.column_ids, "constant", info); @@ -922,64 +933,64 @@ void DeltaMultiFileList::ReportFilterPushdown(ClientContext &context, DeltaMulti new_total = new_list.GetTotalFileCount(); if (should_report_explain_output) { - if (!mfr_info->extra_info.total_files.IsValid()) { + // Filter pushdown can run multiple times for one query (e.g. RemoveUnusedColumns re-runs it on the + // already-filtered list), so old_total shrinks across passes. Keep the largest (the true pre-filter total). + if (!mfr_info->extra_info.total_files.IsValid() || + old_total > mfr_info->extra_info.total_files.GetIndex()) { mfr_info->extra_info.total_files = old_total; - } else if (mfr_info->extra_info.total_files.GetIndex() != old_total) { - throw InternalException( - "Error encountered when analyzing filtered out files for delta scan: total_files inconsistent!"); } + // Likewise keep the smallest post-filter count across passes (the most files skipped). if (!mfr_info->extra_info.filtered_files.IsValid() || - mfr_info->extra_info.filtered_files.GetIndex() >= new_total) { + new_total < mfr_info->extra_info.filtered_files.GetIndex()) { mfr_info->extra_info.filtered_files = new_total; - } else { - throw InternalException( - "Error encountered when analyzing filtered out files for delta scan: filtered_files inconsistent!"); } } } - // Report the new filters - vector old_filters_value_list; - for (auto &f : table_filters.filters) { - auto &column_index = f.first; - auto &filter = f.second; - if (column_index < global_columns.size()) { - auto &col_name = global_columns[column_index].name; - old_filters_value_list.push_back(filter->ToString(col_name)); + // Collect a filter set's rendered strings, sorted for deterministic output (the underlying map is unordered). + auto collect_filter_strings = [&](const DeltaTableFilters &tf) { + vector result; + for (auto &entry : tf) { + auto column_id = entry.first; + if (column_id < global_columns.size()) { + result.push_back(entry.second->ToString(global_columns[column_id].name.GetIdentifierName())); + } } - } - auto old_filters_value = Value::LIST(LogicalType::VARCHAR, old_filters_value_list); - - // Report the new filters - vector filters_value_list; - for (auto &f : new_list.table_filters.filters) { - auto &column_index = f.first; - auto &filter = f.second; - if (column_index < global_columns.size()) { - auto &col_name = global_columns[column_index].name; - filters_value_list.push_back(filter->ToString(col_name)); + std::sort(result.begin(), result.end()); + return result; + }; + + auto to_value_list = [](const vector &strings) { + vector values; + for (auto &s : strings) { + values.push_back(Value(s)); } - } - auto filters_value = Value::LIST(LogicalType::VARCHAR, filters_value_list); + return Value::LIST(LogicalType::VARCHAR, values); + }; + + // Report the pre- and post-pushdown filters + auto old_filters_value = to_value_list(collect_filter_strings(table_filters)); + auto new_filter_strings = collect_filter_strings(new_list.table_filters); + auto filters_value = to_value_list(new_filter_strings); if (should_report_explain_output) { string files_string; - for (auto &filter : filters_value_list) { - files_string += filter.ToString() + "\n"; + for (auto &filter : new_filter_strings) { + files_string += filter + "\n"; } mfr_info->extra_info.file_filters = files_string.substr(0, files_string.size() - 1); } if (should_log) { child_list_t struct_fields; - struct_fields.push_back({"path", Value(GetPath())}); - struct_fields.push_back({"type", Value(pushdown_type)}); - struct_fields.push_back({"filters_before", old_filters_value}); - struct_fields.push_back({"filters_after", filters_value}); + struct_fields.emplace_back("path", Value(GetPath())); + struct_fields.emplace_back("type", Value(pushdown_type)); + struct_fields.emplace_back("filters_before", old_filters_value); + struct_fields.emplace_back("filters_after", filters_value); if (new_total != DConstants::INVALID_INDEX) { - struct_fields.push_back({"files_before", Value::BIGINT(old_total)}); - struct_fields.push_back({"files_after", Value::BIGINT(new_total)}); + struct_fields.emplace_back("files_before", Value::BIGINT(old_total)); + struct_fields.emplace_back("files_after", Value::BIGINT(new_total)); } auto struct_value = Value::STRUCT(struct_fields); logger.WriteLog(delta_log_type, log_level, struct_value.ToString()); @@ -988,31 +999,33 @@ void DeltaMultiFileList::ReportFilterPushdown(ClientContext &context, DeltaMulti unique_ptr DeltaMultiFileList::DynamicFilterPushdown(ClientContext &context, const MultiFileOptions &options, - const vector &names, const vector &types, + const vector &names, const vector &types, const vector &column_ids, TableFilterSet &filters) const { auto pushdown_mode = GetDeltaFilterPushdownMode(context, options); if (pushdown_mode == DeltaFilterPushdownMode::NONE || pushdown_mode == DeltaFilterPushdownMode::CONSTANT_ONLY) { return nullptr; } - if (filters.filters.empty()) { + if (!filters.HasFilters()) { return nullptr; } TableFilterSet filters_copy; - for (auto &filter : filters.filters) { - auto column_id = column_ids[filter.first]; - auto previously_pushed_down_filter = this->table_filters.filters.find(column_id); - if (previously_pushed_down_filter != this->table_filters.filters.end() && - filter.second->Equals(*previously_pushed_down_filter->second)) { + for (auto &entry : filters) { + auto proj_id = entry.GetIndex(); + auto &filter = + ExpressionFilter::GetExpressionFilter(entry.Filter(), "DeltaMultiFileList::DynamicFilterPushdown"); + auto column_id = column_ids[proj_id]; + auto previously_pushed_down_filter = table_filters.TryGetFilterByColumnIndex(column_id); + if (previously_pushed_down_filter && filter.Equals(*previously_pushed_down_filter)) { // Skip filters that we already have pushed down continue; } - filters_copy.PushFilter(ColumnIndex(column_id), filter.second->Copy()); + filters_copy.PushFilter(proj_id, filter.Copy()); } - if (!filters_copy.filters.empty()) { - auto new_snap = PushdownInternal(context, filters_copy); + if (filters_copy.HasFilters()) { + auto new_snap = PushdownInternal(context, filters_copy, column_ids); ReportFilterPushdown(context, *new_snap, column_ids, "dynamic", nullptr); return std::move(new_snap); } diff --git a/src/functions/delta_scan/delta_multi_file_reader.cpp b/src/functions/delta_scan/delta_multi_file_reader.cpp index 37a01199..d38ad28e 100644 --- a/src/functions/delta_scan/delta_multi_file_reader.cpp +++ b/src/functions/delta_scan/delta_multi_file_reader.cpp @@ -55,7 +55,7 @@ void FinalizeBindBaseOverride(MultiFileReaderData &reader_data, const MultiFileO // create a map of name -> column index auto &local_columns = reader_data.reader->GetColumns(); auto &filename = reader_data.reader->GetFileName(); - case_insensitive_map_t name_map; + identifier_map_t name_map; if (file_options.union_by_name) { for (idx_t col_idx = 0; col_idx < local_columns.size(); col_idx++) { auto &column = local_columns[col_idx]; @@ -104,7 +104,7 @@ void FinalizeBindBaseOverride(MultiFileReaderData &reader_data, const MultiFileO } bool DeltaMultiFileReader::Bind(MultiFileOptions &options, MultiFileList &files, vector &return_types, - vector &names, MultiFileReaderBindData &bind_data) { + vector &names, MultiFileReaderBindData &bind_data) { auto &delta_snapshot = dynamic_cast(files); auto log_tail_setting = options.custom_options.find("log_tail"); @@ -118,7 +118,7 @@ bool DeltaMultiFileReader::Bind(MultiFileOptions &options, MultiFileList &files, } void DeltaMultiFileReader::BindOptions(MultiFileOptions &options, MultiFileList &files, - vector &return_types, vector &names, + vector &return_types, vector &names, MultiFileReaderBindData &bind_data) { // Disable all other multifilereader options options.auto_detect_hive_partitioning = false; @@ -137,8 +137,8 @@ void DeltaMultiFileReader::BindOptions(MultiFileOptions &options, MultiFileList auto partitions = snapshot.GetPartitionColumns(); for (auto &part : partitions) { idx_t hive_partitioning_index; - auto lookup = std::find_if(names.begin(), names.end(), - [&](const string &col_name) { return StringUtil::CIEquals(col_name, part); }); + auto lookup = + std::find_if(names.begin(), names.end(), [&](const Identifier &col_name) { return col_name == part; }); if (lookup != names.end()) { // hive partitioning column also exists in file - override auto idx = NumericCast(lookup - names.begin()); @@ -147,6 +147,9 @@ void DeltaMultiFileReader::BindOptions(MultiFileOptions &options, MultiFileList throw IOException("Delta Snapshot returned partition column that is not present in the schema"); } bind_data.hive_partitioning_indexes.emplace_back(part, hive_partitioning_index); + // Register the partition column's type so pre-open filter skipping resolves the partition value as the + // column type instead of defaulting to VARCHAR (which crashes typed ExpressionFilter evaluation). + options.hive_types_schema[part] = return_types[hive_partitioning_index]; } } @@ -264,7 +267,7 @@ DeltaMultiFileReader::InitializeGlobalState(ClientContext &context, const MultiF } auto global_name = global_columns[global_id].name; - selected_columns.insert({global_name, i}); + selected_columns.insert({global_name.GetIdentifierName(), i}); } auto res = make_uniq(extra_columns, &file_list); diff --git a/src/functions/delta_scan/delta_scan.cpp b/src/functions/delta_scan/delta_scan.cpp index f4b45ce8..64949b30 100644 --- a/src/functions/delta_scan/delta_scan.cpp +++ b/src/functions/delta_scan/delta_scan.cpp @@ -113,10 +113,10 @@ TableFunctionSet DeltaFunctions::GetDeltaScanFunction(ExtensionLoader &loader) { function.named_parameters["pushdown_filters"] = LogicalType::VARCHAR; function.named_parameters["log_tail"] = KernelUtils::GetLogPathType(); - function.name = "delta_scan"; + function.SetName("delta_scan"); } - parquet_scan_copy.name = "delta_scan"; + parquet_scan_copy.SetName("delta_scan"); return parquet_scan_copy; } diff --git a/src/functions/delta_transaction_utils/idempotency_helpers.cpp b/src/functions/delta_transaction_utils/idempotency_helpers.cpp index 4a5558f9..0f5896dd 100644 --- a/src/functions/delta_transaction_utils/idempotency_helpers.cpp +++ b/src/functions/delta_transaction_utils/idempotency_helpers.cpp @@ -91,7 +91,7 @@ static unique_ptr DeltaGetTransactionVersionBind(ClientContext &co names.emplace_back("version"); // TODO: support catalog.schema.table format - EntryLookupInfo lookup(CatalogType::TABLE_ENTRY, path); + EntryLookupInfo lookup(CatalogType::TABLE_ENTRY, Identifier(path)); auto lookup_result = Catalog::GetEntry(context, "", "", lookup, OnEntryNotFound::RETURN_NULL); if (lookup_result.get()) { res->delta_table_entry = lookup_result->Cast(); @@ -113,7 +113,7 @@ static unique_ptr DeltaSetTransactionVersionBind(ClientContext &co res->expected_version = input.inputs[3]; // TODO: support catalog.schema.table format - EntryLookupInfo lookup(CatalogType::TABLE_ENTRY, path); + EntryLookupInfo lookup(CatalogType::TABLE_ENTRY, Identifier(path)); auto lookup_result = Catalog::GetEntry(context, "", "", lookup, OnEntryNotFound::RETURN_NULL); if (lookup_result.get()) { res->delta_table_entry = lookup_result->Cast(); diff --git a/src/functions/expression_functions.cpp b/src/functions/expression_functions.cpp index bb2f9768..1227c33e 100644 --- a/src/functions/expression_functions.cpp +++ b/src/functions/expression_functions.cpp @@ -15,11 +15,11 @@ static void AddTestExpressions(vector &result, vectorGetExpressionType() == ExpressionType::FUNCTION) { - for (auto &expr : expression->Cast().children) { - result.push_back(expr->ToString()); + for (auto &arg : expression->Cast().GetArguments()) { + result.push_back(arg.GetExpression().ToString()); } } else if (expression->GetExpressionType() == ExpressionType::CONJUNCTION_AND) { - for (auto &expr : expression->Cast().children) { + for (auto &expr : expression->Cast().GetChildren()) { result.push_back(expr->ToString()); } } else { @@ -46,7 +46,7 @@ static void GetDeltaTestExpression(DataChunk &input, ExpressionState &state, Vec ScalarFunctionSet DeltaFunctions::GetExpressionFunction(ExtensionLoader &loader) { ScalarFunctionSet result; - result.name = "get_delta_test_expression"; + result.SetName("get_delta_test_expression"); ScalarFunction getvar({}, LogicalType::LIST(LogicalType::VARCHAR), GetDeltaTestExpression, nullptr, nullptr); result.AddFunction(getvar); diff --git a/src/functions/metadata_functions.cpp b/src/functions/metadata_functions.cpp index a2add2cf..70947bef 100644 --- a/src/functions/metadata_functions.cpp +++ b/src/functions/metadata_functions.cpp @@ -44,7 +44,7 @@ static void MetadataFunctionExecute(ClientContext &context, TableFunctionInput & output.SetCardinality(count); } -DeltaBaseMetadataFunction::DeltaBaseMetadataFunction(string name_p, table_function_bind_t bind) +DeltaBaseMetadataFunction::DeltaBaseMetadataFunction(Identifier name_p, table_function_bind_t bind) : TableFunction(std::move(name_p), {LogicalType::VARCHAR}, MetadataFunctionExecute, bind, MetadataFunctionInit) { } diff --git a/src/functions/util_functions.cpp b/src/functions/util_functions.cpp index a4da2403..987ae3f3 100644 --- a/src/functions/util_functions.cpp +++ b/src/functions/util_functions.cpp @@ -33,7 +33,7 @@ static void GetWriteDataFunction(DataChunk &input, ExpressionState &state, Vecto ScalarFunctionSet DeltaFunctions::GetWriteFileFunction(ExtensionLoader &loader) { ScalarFunctionSet result; - result.name = "write_blob"; + result.SetName("write_blob"); ScalarFunction write_file({LogicalType::VARCHAR, LogicalType::BLOB}, {LogicalType::BOOLEAN}, GetWriteDataFunction, nullptr, nullptr); diff --git a/src/include/delta_functions.hpp b/src/include/delta_functions.hpp index 2f40e5fa..4d9780c1 100644 --- a/src/include/delta_functions.hpp +++ b/src/include/delta_functions.hpp @@ -15,7 +15,7 @@ class ExtensionLoader; class DeltaBaseMetadataFunction : public TableFunction { public: - DeltaBaseMetadataFunction(string name, table_function_bind_t bind); + DeltaBaseMetadataFunction(Identifier name, table_function_bind_t bind); }; class DeltaFileListFunction : public DeltaBaseMetadataFunction { diff --git a/src/include/delta_utils.hpp b/src/include/delta_utils.hpp index 90598039..f65988a8 100644 --- a/src/include/delta_utils.hpp +++ b/src/include/delta_utils.hpp @@ -4,8 +4,8 @@ #define DEFINE_DEFAULT_ENGINE_BASE 1 #include "delta_kernel_ffi.hpp" #include "duckdb/common/enum_util.hpp" -#include "duckdb/planner/filter/conjunction_filter.hpp" -#include "duckdb/planner/filter/constant_filter.hpp" +#include "duckdb/planner/filter/expression_filter.hpp" +#include "duckdb/planner/filter/table_filter_functions.hpp" #include "duckdb/planner/expression.hpp" #include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/common/multi_file/multi_file_data.hpp" @@ -92,7 +92,7 @@ struct KernelUtils { return none; } - static vector> & + static vector> UnpackTransformExpression(const vector> &parsed_expression); }; @@ -461,29 +461,33 @@ struct SharedKernelPointer { }; typedef SharedKernelPointer SharedKernelSnapshot; +struct DeltaTableFilters; class PredicateVisitor : public ffi::EnginePredicate { public: - PredicateVisitor(const vector &columns, optional_ptr filters); + PredicateVisitor(const vector &columns, + optional_ptr filters); ErrorData error_data; private: - unordered_map column_filters; + unordered_map> column_filters; + // Top-level column types, keyed by the same name as column_filters, used to reject filters whose subject is a bare + // reference to a nested (struct/list/map) column - the path to the scalar leaf is not recoverable in that form. + unordered_map column_types; static uintptr_t VisitPredicate(PredicateVisitor *predicate, ffi::KernelExpressionVisitorState *state); - uintptr_t VisitConstantFilter(const string &col_name, const ConstantFilter &filter, + uintptr_t VisitConstantFilter(const string &col_name, ExpressionType comparison_type, const Value &value, ffi::KernelExpressionVisitorState *state); - uintptr_t VisitAndFilter(const string &col_name, const ConjunctionAndFilter &filter, - ffi::KernelExpressionVisitorState *state); + uintptr_t VisitFilterExpression(const string &col_name, const Expression &expr, + ffi::KernelExpressionVisitorState *state); uintptr_t VisitIsNull(const string &col_name, ffi::KernelExpressionVisitorState *state); uintptr_t VisitIsNotNull(const string &col_name, ffi::KernelExpressionVisitorState *state); - uintptr_t VisitStructExtractFilter(const string &col_name, const StructFilter &filter, - ffi::KernelExpressionVisitorState *state); - uintptr_t VisitFilter(const string &col_name, const TableFilter &filter, ffi::KernelExpressionVisitorState *state); + uintptr_t VisitFilter(const string &col_name, const ExpressionFilter &filter, + ffi::KernelExpressionVisitorState *state); }; // Singleton class to forward logs to DuckDB diff --git a/src/include/functions/delta_scan/delta_multi_file_list.hpp b/src/include/functions/delta_scan/delta_multi_file_list.hpp index 003887cf..2b8ab34c 100644 --- a/src/include/functions/delta_scan/delta_multi_file_list.hpp +++ b/src/include/functions/delta_scan/delta_multi_file_list.hpp @@ -16,6 +16,7 @@ #include "duckdb/common/multi_file/multi_file_data.hpp" #include "duckdb/common/multi_file/multi_file_list.hpp" #include "duckdb/parser/constraints/not_null_constraint.hpp" +#include "duckdb/planner/filter/expression_filter.hpp" namespace duckdb { @@ -37,11 +38,48 @@ struct DeltaFileMetaData { idx_t cardinality = DConstants::INVALID_INDEX; ffi::KernelBoolSlice selection_vector = {nullptr, 0}; - case_insensitive_map_t partition_map; + identifier_map_t partition_map; unique_ptr>> transform_expression; }; +struct DeltaTableFilters { + using filter_set_t = unordered_map>; + using iterator = filter_set_t::iterator; + using const_iterator = filter_set_t::const_iterator; + +public: + bool HasFilters() const { + return !table_filters.empty(); + } + void PushFilter(column_t column_idx, unique_ptr table_filter) { + table_filters[column_idx] = std::move(table_filter); + } + optional_ptr TryGetFilterByColumnIndex(column_t column_idx) const { + auto entry = table_filters.find(column_idx); + if (entry == table_filters.end()) { + return nullptr; + } + return entry->second.get(); + } + + iterator begin() { // NOLINT: match stl API + return table_filters.begin(); + } + iterator end() { // NOLINT: match stl API + return table_filters.end(); + } + const_iterator begin() const { // NOLINT: match stl API + return table_filters.begin(); + } + const_iterator end() const { // NOLINT: match stl API + return table_filters.end(); + } + +private: + filter_set_t table_filters; +}; + // Constraint only for internal delta extension use // Todo: refactor to use duckdb constraint classes, updating the DuckDB side NotNullConstraint class NestedNotNullConstraint { @@ -65,17 +103,18 @@ class DeltaMultiFileList : public SimpleMultiFileList { //! MultiFileList API public: - void Bind(vector &return_types, vector &names); + void Bind(vector &return_types, vector &names); unique_ptr ComplexFilterPushdown(ClientContext &context, const MultiFileOptions &options, MultiFilePushdownInfo &info, vector> &filters) const override; unique_ptr DynamicFilterPushdown(ClientContext &context, const MultiFileOptions &options, - const vector &names, const vector &types, + const vector &names, const vector &types, const vector &column_ids, TableFilterSet &filters) const override; - unique_ptr PushdownInternal(ClientContext &context, TableFilterSet &new_filters) const; + unique_ptr PushdownInternal(ClientContext &context, TableFilterSet &new_filters, + vector column_indexes) const; vector GetAllFiles() const override; FileExpandResult GetExpandResult() const override; @@ -148,7 +187,7 @@ class DeltaMultiFileList : public SimpleMultiFileList { mutable vector> metadata; mutable vector resolved_files; - mutable TableFilterSet table_filters; + mutable DeltaTableFilters table_filters; mutable vector not_null_constraints; mutable bool has_null_constraints_in_arrays = false; diff --git a/src/include/functions/delta_scan/delta_multi_file_reader.hpp b/src/include/functions/delta_scan/delta_multi_file_reader.hpp index d56b7a0b..5ed4acbe 100644 --- a/src/include/functions/delta_scan/delta_multi_file_reader.hpp +++ b/src/include/functions/delta_scan/delta_multi_file_reader.hpp @@ -33,12 +33,12 @@ struct DeltaMultiFileReader : public MultiFileReader { //! Override the regular parquet bind using the MultiFileReader Bind. The bind from these are what DuckDB's file //! readers will try read - bool Bind(MultiFileOptions &options, MultiFileList &files, vector &return_types, vector &names, - MultiFileReaderBindData &bind_data) override; + bool Bind(MultiFileOptions &options, MultiFileList &files, vector &return_types, + vector &names, MultiFileReaderBindData &bind_data) override; //! Override the Options bind void BindOptions(MultiFileOptions &options, MultiFileList &files, vector &return_types, - vector &names, MultiFileReaderBindData &bind_data) override; + vector &names, MultiFileReaderBindData &bind_data) override; unique_ptr InitializeGlobalState(ClientContext &context, const MultiFileOptions &file_options, diff --git a/src/storage/delta_insert.cpp b/src/storage/delta_insert.cpp index f48d9b3a..eadde223 100644 --- a/src/storage/delta_insert.cpp +++ b/src/storage/delta_insert.cpp @@ -286,7 +286,7 @@ string DeltaInsert::GetName() const { InsertionOrderPreservingMap DeltaInsert::ParamsToString() const { InsertionOrderPreservingMap result; - result["Table Name"] = table ? table->name : info->Base().table; + result["Table Name"] = table ? table->name.GetIdentifierName() : info->Base().GetTableName().GetIdentifierName(); return result; } @@ -297,8 +297,8 @@ static optional_ptr TryGetCopyFunction(DatabaseInstanc D_ASSERT(!name.empty()); auto &system_catalog = Catalog::GetSystemCatalog(db); auto data = CatalogTransaction::GetSystemTransaction(db); - auto &schema = system_catalog.GetSchema(data, DEFAULT_SCHEMA); - return schema.GetEntry(data, CatalogType::COPY_FUNCTION_ENTRY, name)->Cast(); + auto &schema = system_catalog.GetSchema(data, Identifier::DefaultSchema()); + return schema.GetEntry(data, CatalogType::COPY_FUNCTION_ENTRY, Identifier(name))->Cast(); } PhysicalOperator &DeltaCatalog::PlanInsert(ClientContext &context, PhysicalPlanGenerator &planner, LogicalInsert &op, @@ -316,7 +316,7 @@ PhysicalOperator &DeltaCatalog::PlanInsert(ClientContext &context, PhysicalPlanG // In child catalog mode, the LogicalInsert will not actually contain the table entry, so we need to look it up // here CatalogEntryRetriever retriever(context); - EntryLookupInfo lookup_info(CatalogType::TABLE_ENTRY, default_table); + EntryLookupInfo lookup_info(CatalogType::TABLE_ENTRY, Identifier(default_table)); auto default_table_entry = LookupEntry(retriever, default_schema, lookup_info, OnEntryNotFound::THROW_EXCEPTION); table_entry = default_table_entry.entry->Cast(); @@ -359,7 +359,8 @@ PhysicalOperator &DeltaCatalog::PlanInsert(ClientContext &context, PhysicalPlanG auto names_to_write = columns.GetColumnNames(); auto types_to_write = columns.GetColumnTypes(); - auto function_data = copy_fun->function.copy_to_bind(context, bind_input, names_to_write, types_to_write); + auto function_data = + copy_fun->function.copy_to_bind(context, bind_input, StringsToIdentifiers(names_to_write), types_to_write); auto &insert = planner.Make(op, *table_entry, op.column_index_map); @@ -394,11 +395,10 @@ PhysicalOperator &DeltaCatalog::PlanInsert(ClientContext &context, PhysicalPlanG physical_copy_ref.file_extension = "parquet"; physical_copy_ref.overwrite_mode = CopyOverwriteMode::COPY_OVERWRITE_OR_IGNORE; physical_copy_ref.per_thread_output = false; - physical_copy_ref.rotate = false; physical_copy_ref.return_type = CopyFunctionReturnType::WRITTEN_FILE_STATISTICS; physical_copy_ref.write_partition_columns = true; physical_copy_ref.children.push_back(*plan); - physical_copy_ref.names = names_to_write; + physical_copy_ref.names = StringsToIdentifiers(names_to_write); physical_copy_ref.expected_types = types_to_write; physical_copy_ref.hive_file_pattern = true; diff --git a/src/storage/delta_schema_entry.cpp b/src/storage/delta_schema_entry.cpp index b2a18a59..fbcda943 100644 --- a/src/storage/delta_schema_entry.cpp +++ b/src/storage/delta_schema_entry.cpp @@ -42,10 +42,10 @@ optional_ptr DeltaSchemaEntry::CreateFunction(CatalogTransaction t } void DeltaUnqualifyColumnRef(ParsedExpression &expr) { - if (expr.type == ExpressionType::COLUMN_REF) { + if (expr.GetExpressionType() == ExpressionType::COLUMN_REF) { auto &colref = expr.Cast(); - auto name = std::move(colref.column_names.back()); - colref.column_names = {std::move(name)}; + auto name = std::move(colref.ColumnNamesMutable().back()); + colref.ColumnNamesMutable() = {std::move(name)}; return; } ParsedExpressionIterator::EnumerateChildren(expr, DeltaUnqualifyColumnRef); @@ -117,17 +117,17 @@ unique_ptr DeltaSchemaEntry::CreateTableEntry(ClientContext &co // Get the names and types from the delta snapshot vector return_types; - vector names; + vector names; snapshot->Bind(return_types, names); // TODO: forward nullability constraints CreateTableInfo table_info; for (idx_t i = 0; i < return_types.size(); i++) { - table_info.columns.AddColumn(ColumnDefinition(names[i], return_types[i])); + table_info.columns.AddColumn(ColumnDefinition(Identifier(names[i]), return_types[i])); } - table_info.table = - !delta_catalog.internal_table_name.empty() ? delta_catalog.internal_table_name : catalog.GetName(); + table_info.SetTableName(!delta_catalog.internal_table_name.empty() ? Identifier(delta_catalog.internal_table_name) + : catalog.GetName()); // Copy over constraints to table info TODO: these are incompatible currently // table_info.constraints = snapshot->not_null_constraints;} diff --git a/src/storage/delta_table_entry.cpp b/src/storage/delta_table_entry.cpp index ac50afbd..1831d369 100644 --- a/src/storage/delta_table_entry.cpp +++ b/src/storage/delta_table_entry.cpp @@ -36,7 +36,7 @@ TableFunction DeltaTableEntry::GetScanFunctionInternal(ClientContext &context, u auto &system_catalog = Catalog::GetSystemCatalog(db); auto data = CatalogTransaction::GetSystemTransaction(db); - auto &schema = system_catalog.GetSchema(data, DEFAULT_SCHEMA); + auto &schema = system_catalog.GetSchema(data, Identifier::DefaultSchema()); auto catalog_entry = schema.GetEntry(data, CatalogType::TABLE_FUNCTION_ENTRY, "delta_scan"); if (!catalog_entry) { throw InvalidInputException("Function with name \"%s\" not found in ExtensionLoader::GetTableFunction", name); @@ -64,13 +64,13 @@ TableFunction DeltaTableEntry::GetScanFunctionInternal(ClientContext &context, u } function_info->snapshot = this->snapshot; - function_info->table_name = delta_catalog.GetName(); + function_info->table_name = delta_catalog.GetName().GetIdentifierName(); delta_scan_function.function_info = std::move(function_info); vector inputs = {delta_catalog.GetDBPath()}; named_parameter_map_t param_map; vector return_types; - vector names; + vector names; TableFunctionRef empty_ref; // Propagate settings @@ -80,7 +80,9 @@ TableFunction DeltaTableEntry::GetScanFunctionInternal(ClientContext &context, u TableFunctionBindInput bind_input(inputs, param_map, return_types, names, nullptr, nullptr, delta_scan_function, empty_ref); - auto result = delta_scan_function.bind(context, bind_input, return_types, names); + vector bind_names; + auto result = delta_scan_function.bind(context, bind_input, return_types, bind_names); + names = StringsToIdentifiers(bind_names); bind_data = std::move(result); return delta_scan_function; @@ -90,7 +92,7 @@ case_insensitive_map_t> DeltaTableEntry::GetNotN case_insensitive_map_t> result; for (auto &constraint : snapshot->GetNestedNotNullConstraints()) { auto &col = GetColumn(constraint.index); - auto &item = result[col.Name()]; + auto &item = result[col.Name().GetIdentifierName()]; item.push_back(constraint); } return result; diff --git a/src/storage/delta_transaction.cpp b/src/storage/delta_transaction.cpp index 7af0fe6a..58db4319 100644 --- a/src/storage/delta_transaction.cpp +++ b/src/storage/delta_transaction.cpp @@ -43,12 +43,6 @@ static void *allocate_string(const struct ffi::KernelStringSlice slice) { } struct DeltaCommitInfo { -public: - DeltaCommitInfo() { - buffer.Initialize(Allocator::DefaultAllocator(), GetTypes()); - buffer.SetCardinality(0); - } - public: static vector GetTypes() { return {LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)}; @@ -59,15 +53,7 @@ struct DeltaCommitInfo { public: void Append(Value commit_info_map) { - idx_t current_size = buffer.size(); - idx_t current_capacity = buffer.GetCapacity(); - - if (current_size == current_capacity) { - buffer.SetCapacity(2 * current_capacity); - } - - buffer.SetValue(0, current_size, commit_info_map); - buffer.SetCardinality(current_size + 1); + values.push_back(std::move(commit_info_map)); } void (*release)(); @@ -79,6 +65,14 @@ struct DeltaCommitInfo { ffi::ArrowFFIData ToArrow(optional_ptr context) { LoggerCallback::TryLog("delta", LogLevel::LOG_TRACE, "Delta ToArrow debug: created CommitInfo"); + DataChunk buffer; + idx_t capacity = MaxValue(values.size(), STANDARD_VECTOR_SIZE); + buffer.Initialize(Allocator::DefaultAllocator(), GetTypes(), capacity); + for (idx_t i = 0; i < values.size(); i++) { + buffer.SetValue(0, i, values[i]); + } + buffer.SetCardinality(values.size()); + ffi::ArrowFFIData ffi_data; unordered_map> extension_types; ClientProperties props("UTC", ArrowOffsetSize::REGULAR, false, false, false, ArrowFormatVersion::V1_0, context); @@ -90,7 +84,7 @@ struct DeltaCommitInfo { } private: - DataChunk buffer; + vector values; }; struct StatNode { @@ -152,22 +146,25 @@ static Value CreateValueLogicalTypeFromStatNode(const unordered_map &outstanding_appends) { const DeltaDataFile *first_file = outstanding_appends.empty() ? nullptr : &outstanding_appends[0]; buffer_types = GetTypes(first_file); - buffer = make_uniq(); - buffer->Initialize(Allocator::DefaultAllocator(), buffer_types); for (const auto &file : outstanding_appends) { auto table_path = snapshot.GetPath(); @@ -250,21 +245,11 @@ struct WriteMetaData { } void Append(const string &path, Value partition_values, const DeltaDataFile &file) { - idx_t current_size = buffer->size(); - idx_t current_capacity = buffer->GetCapacity(); - - if (current_size == current_capacity) { - buffer->SetCapacity(2 * current_capacity); - } - - auto stats = CreateStatsValue(file, true); - - buffer->SetValue(0, current_size, path); - buffer->SetValue(1, current_size, partition_values); - buffer->SetValue(2, current_size, Value::BIGINT(file.file_size_bytes)); - buffer->SetValue(3, current_size, Value::BIGINT(Timestamp::GetEpochMs(file.last_modified_time))); - buffer->SetValue(4, current_size, stats); - buffer->SetCardinality(current_size + 1); + paths.push_back(Value(path)); + partition_values_list.push_back(std::move(partition_values)); + sizes.push_back(Value::BIGINT(file.file_size_bytes)); + modification_times.push_back(Value::BIGINT(Timestamp::GetEpochMs(file.last_modified_time))); + stats.push_back(CreateStatsValue(file, true)); } void (*release)(); @@ -276,11 +261,24 @@ struct WriteMetaData { ffi::ArrowFFIData ToArrow(ClientContext &context) { LoggerCallback::TryLog("delta", LogLevel::LOG_TRACE, "Delta ToArrow debug: created WriteMetaData"); + DataChunk buffer; + idx_t n = paths.size(); + idx_t capacity = MaxValue(n, STANDARD_VECTOR_SIZE); + buffer.Initialize(Allocator::DefaultAllocator(), buffer_types, capacity); + for (idx_t i = 0; i < n; i++) { + buffer.SetValue(0, i, paths[i]); + buffer.SetValue(1, i, partition_values_list[i]); + buffer.SetValue(2, i, sizes[i]); + buffer.SetValue(3, i, modification_times[i]); + buffer.SetValue(4, i, stats[i]); + } + buffer.SetCardinality(n); + ffi::ArrowFFIData ffi_data; unordered_map> extension_types; ClientProperties props("UTC", ArrowOffsetSize::REGULAR, false, false, false, ArrowFormatVersion::V1_0, optional_ptr(&context)); - ArrowConverter::ToArrowArray(*buffer, (ArrowArray *)(&ffi_data.array), props, extension_types); + ArrowConverter::ToArrowArray(buffer, (ArrowArray *)(&ffi_data.array), props, extension_types); ArrowConverter::ToArrowSchema((ArrowSchema *)(&ffi_data.schema), buffer_types, GetNames(), props); ffi_data.array.release = reinterpret_cast(InstrumentedRelease); @@ -288,8 +286,13 @@ struct WriteMetaData { return ffi_data; } +private: vector buffer_types; - unique_ptr buffer; + vector paths; + vector partition_values_list; + vector sizes; + vector modification_times; + vector stats; }; vector DeltaTransaction::GetWriteSchema(ClientContext &context) { diff --git a/test/sql/cloud/azure/access_token_auth.test b/test/sql/cloud/azure/access_token_auth.test index d32ed04e..8de1cadd 100644 --- a/test/sql/cloud/azure/access_token_auth.test +++ b/test/sql/cloud/azure/access_token_auth.test @@ -24,7 +24,7 @@ statement ok CREATE OR REPLACE SECRET az1 ( TYPE AZURE, PROVIDER ACCESS_TOKEN, - ACCOUNT_NAME '${AZURE_STORAGE_ACCOUNT}' + ACCOUNT_NAME '{AZURE_STORAGE_ACCOUNT}' ) statement error @@ -36,8 +36,8 @@ statement ok CREATE OR REPLACE SECRET az1 ( TYPE AZURE, PROVIDER ACCESS_TOKEN, - ACCESS_TOKEN '${AZURE_ACCESS_TOKEN}', - ACCOUNT_NAME '${AZURE_STORAGE_ACCOUNT}' + ACCESS_TOKEN '{AZURE_ACCESS_TOKEN}', + ACCOUNT_NAME '{AZURE_STORAGE_ACCOUNT}' ) query I diff --git a/test/sql/cloud/azure/cli_auth.test b/test/sql/cloud/azure/cli_auth.test index 53ae5fbe..c97e1e76 100644 --- a/test/sql/cloud/azure/cli_auth.test +++ b/test/sql/cloud/azure/cli_auth.test @@ -20,7 +20,7 @@ CREATE SECRET az1 ( TYPE AZURE, PROVIDER CREDENTIAL_CHAIN, CHAIN 'cli', - ACCOUNT_NAME '${AZURE_STORAGE_ACCOUNT}' + ACCOUNT_NAME '{AZURE_STORAGE_ACCOUNT}' ) mode output_result diff --git a/test/sql/cloud/azure/hierarchical_namespace.test b/test/sql/cloud/azure/hierarchical_namespace.test index 936740c2..34926ba9 100644 --- a/test/sql/cloud/azure/hierarchical_namespace.test +++ b/test/sql/cloud/azure/hierarchical_namespace.test @@ -24,10 +24,10 @@ statement ok CREATE SECRET spn ( TYPE AZURE, PROVIDER SERVICE_PRINCIPAL, - TENANT_ID '${AZURE_TENANT_ID}', - CLIENT_ID '${AZURE_CLIENT_ID}', - CLIENT_SECRET '${AZURE_CLIENT_SECRET}', - ACCOUNT_NAME '${AZURE_STORAGE_ACCOUNT}' + TENANT_ID '{AZURE_TENANT_ID}', + CLIENT_ID '{AZURE_CLIENT_ID}', + CLIENT_SECRET '{AZURE_CLIENT_SECRET}', + ACCOUNT_NAME '{AZURE_STORAGE_ACCOUNT}' ); # Run a remote DAT test on abfss diff --git a/test/sql/cloud/azure/spn_auth.test b/test/sql/cloud/azure/spn_auth.test index 06c4a9c0..9d87d9a8 100644 --- a/test/sql/cloud/azure/spn_auth.test +++ b/test/sql/cloud/azure/spn_auth.test @@ -20,10 +20,10 @@ statement ok CREATE SECRET spn ( TYPE AZURE, PROVIDER SERVICE_PRINCIPAL, - TENANT_ID '${AZURE_TENANT_ID}', - CLIENT_ID '${AZURE_CLIENT_ID}', - CLIENT_SECRET '${AZURE_CLIENT_SECRET}', - ACCOUNT_NAME '${AZURE_STORAGE_ACCOUNT}' + TENANT_ID '{AZURE_TENANT_ID}', + CLIENT_ID '{AZURE_CLIENT_ID}', + CLIENT_SECRET '{AZURE_CLIENT_SECRET}', + ACCOUNT_NAME '{AZURE_STORAGE_ACCOUNT}' ); # Run a remote DAT test diff --git a/test/sql/cloud/azure/unauthenticated.test b/test/sql/cloud/azure/unauthenticated.test index fde35338..3cfb88c0 100644 --- a/test/sql/cloud/azure/unauthenticated.test +++ b/test/sql/cloud/azure/unauthenticated.test @@ -20,7 +20,7 @@ mode skip query I SELECT int32 -FROM delta_scan('azure://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/dat/all_primitive_types/delta') +FROM delta_scan('azure://{AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/dat/all_primitive_types/delta') ---- 0 1 @@ -32,7 +32,7 @@ mode unskip # Using a secret to set the account name, we can omit the fully qualified url statement ok -CREATE SECRET s1 (TYPE AZURE, ACCOUNT_NAME '${AZURE_STORAGE_ACCOUNT}') +CREATE SECRET s1 (TYPE AZURE, ACCOUNT_NAME '{AZURE_STORAGE_ACCOUNT}') query I SELECT int32 diff --git a/test/sql/cloud/azurite/azurite.test b/test/sql/cloud/azurite/azurite.test index 930c1702..3a2917e4 100644 --- a/test/sql/cloud/azurite/azurite.test +++ b/test/sql/cloud/azurite/azurite.test @@ -13,14 +13,14 @@ require-env AZURE_STORAGE_CONNECTION_STRING # Set connection string from env var statement ok -CREATE SECRET (TYPE AZURE, CONNECTION_STRING '${AZURE_STORAGE_CONNECTION_STRING}'); +CREATE SECRET (TYPE AZURE, CONNECTION_STRING '{AZURE_STORAGE_CONNECTION_STRING}'); # We need a connection string to do requests foreach prefix azure:// az:// query I SELECT int32 -FROM delta_scan('${prefix}delta-testing-private/dat/all_primitive_types/delta') +FROM delta_scan('{prefix}delta-testing-private/dat/all_primitive_types/delta') ---- 0 1 diff --git a/test/sql/cloud/minio_local/gcs_r2.test b/test/sql/cloud/minio_local/gcs_r2.test index 743462c3..56e727be 100644 --- a/test/sql/cloud/minio_local/gcs_r2.test +++ b/test/sql/cloud/minio_local/gcs_r2.test @@ -21,7 +21,7 @@ require-env AWS_DEFAULT_REGION require-env AWS_ENDPOINT statement ok -set secret_directory='__TEST_DIR__/minio_local_gcs_env' +set secret_directory='{TEMP_DIR}/minio_local_gcs_env' statement error FROM delta_scan('gcs://test-bucket/dat/all_primitive_types/delta') @@ -42,10 +42,10 @@ Can not scan a gcs:// gs:// or r2:// url without a secret providing its endpoint statement ok CREATE SECRET ( TYPE GCS, - KEY_ID '${AWS_ACCESS_KEY_ID}', - SECRET '${AWS_SECRET_ACCESS_KEY}', - REGION '${AWS_DEFAULT_REGION}', - ENDPOINT '${AWS_ENDPOINT}', + KEY_ID '{AWS_ACCESS_KEY_ID}', + SECRET '{AWS_SECRET_ACCESS_KEY}', + REGION '{AWS_DEFAULT_REGION}', + ENDPOINT '{AWS_ENDPOINT}', USE_SSL false ) @@ -75,10 +75,10 @@ CREATE SECRET s1 ( TYPE R2, PROVIDER config, account_id 'some_bogus_account', - KEY_ID '${AWS_ACCESS_KEY_ID}', - SECRET '${AWS_SECRET_ACCESS_KEY}', - REGION '${AWS_DEFAULT_REGION}', - ENDPOINT '${AWS_ENDPOINT}', + KEY_ID '{AWS_ACCESS_KEY_ID}', + SECRET '{AWS_SECRET_ACCESS_KEY}', + REGION '{AWS_DEFAULT_REGION}', + ENDPOINT '{AWS_ENDPOINT}', USE_SSL false ) diff --git a/test/sql/cloud/minio_local/minio_local.test b/test/sql/cloud/minio_local/minio_local.test index b70670aa..1fdd3d4a 100644 --- a/test/sql/cloud/minio_local/minio_local.test +++ b/test/sql/cloud/minio_local/minio_local.test @@ -21,13 +21,13 @@ require-env AWS_DEFAULT_REGION require-env AWS_ENDPOINT statement ok -set secret_directory='__TEST_DIR__/aws_secret_chains_env' +set secret_directory='{TEMP_DIR}/aws_secret_chains_env' # Secret with just the endpoint statement ok CREATE SECRET s1 ( TYPE S3, - ENDPOINT '${AWS_ENDPOINT}', + ENDPOINT '{AWS_ENDPOINT}', USE_SSL false ); @@ -57,10 +57,10 @@ statement ok CREATE SECRET s1 ( TYPE S3, PROVIDER config, - KEY_ID '${AWS_ACCESS_KEY_ID}', - SECRET '${AWS_SECRET_ACCESS_KEY}', - REGION '${AWS_DEFAULT_REGION}', - ENDPOINT '${AWS_ENDPOINT}', + KEY_ID '{AWS_ACCESS_KEY_ID}', + SECRET '{AWS_SECRET_ACCESS_KEY}', + REGION '{AWS_DEFAULT_REGION}', + ENDPOINT '{AWS_ENDPOINT}', USE_SSL false ); @@ -94,7 +94,7 @@ statement ok CREATE SECRET s1 ( TYPE S3, PROVIDER credential_chain, - ENDPOINT '${AWS_ENDPOINT}', + ENDPOINT '{AWS_ENDPOINT}', USE_SSL false ); diff --git a/test/sql/dat/all.test b/test/sql/dat/all.test index 06d4e349..bdc38145 100644 --- a/test/sql/dat/all.test +++ b/test/sql/dat/all.test @@ -13,18 +13,18 @@ require-env DAT_PATH # all_primitive_types query I rowsort all_primitive_types SELECT * EXCLUDE(timestamp) -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/all_primitive_types/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/all_primitive_types/delta') ---- # TODO: exclude is necessary due to issue with the golden tables, this should be fixed upstream query I rowsort all_primitive_types SELECT * EXCLUDE(timestamp) -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/all_primitive_types/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/all_primitive_types/expected/latest/**/*.parquet') ---- query I SELECT timestamp -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/all_primitive_types/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/all_primitive_types/delta') ---- 1970-01-01 00:00:00+00 1970-01-01 01:00:00+00 @@ -35,95 +35,95 @@ FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/all_primitive_types/delt # nested_types query I rowsort nested_types SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/nested_types/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/nested_types/delta') ---- query I rowsort nested_types SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/nested_types/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/nested_types/expected/latest/**/*.parquet') ---- # basic_append query I rowsort basic_append SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') ---- query I rowsort basic_append SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/expected/latest/**/*.parquet') ---- query I rowsort basic_append_count SELECT count(*) -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') ---- query I rowsort basic_append_count SELECT count(*) -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/expected/latest/**/*.parquet') ---- # with_schema_change query I rowsort with_schema_change SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/with_schema_change/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/with_schema_change/delta') ---- query I rowsort with_schema_change SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/with_schema_change/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/with_schema_change/expected/latest/**/*.parquet') ---- query I rowsort with_schema_change_count SELECT count(*) -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/with_schema_change/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/with_schema_change/delta') ---- query I rowsort with_schema_change_count SELECT count(*) -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/with_schema_change/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/with_schema_change/expected/latest/**/*.parquet') ---- # basic_partitioned query I rowsort basic_partitioned SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_partitioned/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_partitioned/delta') ---- query I rowsort basic_partitioned SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/basic_partitioned/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/basic_partitioned/expected/latest/**/*.parquet') ---- query I rowsort basic_partitioned_count SELECT count(*) -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_partitioned/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_partitioned/delta') ---- query I rowsort basic_partitioned_count SELECT count(*) -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/basic_partitioned/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/basic_partitioned/expected/latest/**/*.parquet') ---- # multi_partitioned query I rowsort multi_partitioned SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/multi_partitioned/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/multi_partitioned/delta') ---- query I rowsort multi_partitioned SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/multi_partitioned/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/multi_partitioned/expected/latest/**/*.parquet') ---- query I rowsort multi_partitioned_count SELECT count(*) -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/multi_partitioned/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/multi_partitioned/delta') ---- query I rowsort multi_partitioned_count SELECT count(*) -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/multi_partitioned/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/multi_partitioned/expected/latest/**/*.parquet') ---- # TODO: fix this @@ -132,18 +132,18 @@ require notwindows # multi_partitioned_2 query I rowsort multi_partitioned_2 SELECT * EXCLUDE(time) -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/multi_partitioned_2/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/multi_partitioned_2/delta') ---- # TODO: exclude is necessary due to issue with the golden tables, this should be fixed upstream query I rowsort multi_partitioned_2 SELECT * EXCLUDE(time) -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/multi_partitioned_2/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/multi_partitioned_2/expected/latest/**/*.parquet') ---- query I SELECT time -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/multi_partitioned_2/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/multi_partitioned_2/delta') order by time ---- 1970-01-01 00:00:00+00 @@ -153,100 +153,100 @@ order by time # no_replay query I rowsort no_replay SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/no_replay/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/no_replay/delta') ---- query I rowsort no_replay SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/no_replay/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/no_replay/expected/latest/**/*.parquet') ---- # no_stats query I rowsort no_stats SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/no_stats/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/no_stats/delta') ---- query I rowsort no_stats SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/no_stats/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/no_stats/expected/latest/**/*.parquet') ---- # stats_as_struct query I rowsort stats_as_struct SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/stats_as_struct/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/stats_as_struct/delta') ---- query I rowsort stats_as_struct SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/stats_as_struct/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/stats_as_struct/expected/latest/**/*.parquet') ---- # with_checkpoint query I rowsort with_checkpoint SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/with_checkpoint/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/with_checkpoint/delta') ---- query I rowsort with_checkpoint SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/with_checkpoint/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/with_checkpoint/expected/latest/**/*.parquet') ---- # cdf query I rowsort cdf SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/cdf/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/cdf/delta') ---- query I rowsort cdf SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/cdf/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/cdf/expected/latest/**/*.parquet') ---- # check_constraints query I rowsort check_constraints SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/check_constraints/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/check_constraints/delta') ---- query I rowsort check_constraints SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/check_constraints/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/check_constraints/expected/latest/**/*.parquet') ---- # column_mapping query I rowsort column_mapping_dat SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/column_mapping/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/column_mapping/delta') ---- query I rowsort column_mapping_dat SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/column_mapping/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/column_mapping/expected/latest/**/*.parquet') ---- # deletion_vectors query I rowsort deletion_vectors SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/deletion_vectors/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/deletion_vectors/delta') ---- query I rowsort deletion_vectors SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/deletion_vectors/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/deletion_vectors/expected/latest/**/*.parquet') ---- # generated_columns query I rowsort generated_columns SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/generated_columns/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/generated_columns/delta') ---- query I rowsort generated_columns SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/generated_columns/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/generated_columns/expected/latest/**/*.parquet') ---- # iceberg_compat_v1 @@ -255,32 +255,32 @@ FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/generated_columns/expe # # query I rowsort iceberg_compat_v1 # SELECT * -# FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/iceberg_compat_v1/delta') +# FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/iceberg_compat_v1/delta') # ---- # # query I rowsort iceberg_compat_v1 # SELECT * -# FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/iceberg_compat_v1/expected/latest/**/*.parquet') +# FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/iceberg_compat_v1/expected/latest/**/*.parquet') # ---- # partitioned_with_null query I rowsort partitioned_with_null SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/partitioned_with_null/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/partitioned_with_null/delta') ---- query I rowsort partitioned_with_null SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/partitioned_with_null/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/partitioned_with_null/expected/latest/**/*.parquet') ---- # timestamp_ntz query I rowsort timestamp_ntz_dat SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/timestamp_ntz/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/timestamp_ntz/delta') ---- query I rowsort timestamp_ntz_dat SELECT * -FROM parquet_scan('${DAT_PATH}/out/reader_tests/generated/timestamp_ntz/expected/latest/**/*.parquet') +FROM parquet_scan('{DAT_PATH}/out/reader_tests/generated/timestamp_ntz/expected/latest/**/*.parquet') ---- diff --git a/test/sql/dat/attach.test b/test/sql/dat/attach.test index e3dab882..e2a1a3e7 100644 --- a/test/sql/dat/attach.test +++ b/test/sql/dat/attach.test @@ -9,7 +9,7 @@ require delta require-env DAT_PATH statement ok -ATTACH '${DAT_PATH}/out/reader_tests/generated/all_primitive_types/delta' as dt (TYPE delta) +ATTACH '{DAT_PATH}/out/reader_tests/generated/all_primitive_types/delta' as dt (TYPE delta) # We can query the table by the catalog name query I @@ -97,7 +97,7 @@ DETACH dt # Test the PIN_SNAPSHOT option: the snapshot is now pinned on attaching statement ok -ATTACH '${DAT_PATH}/out/reader_tests/generated/all_primitive_types/delta' as dt (TYPE delta, PIN_SNAPSHOT) +ATTACH '{DAT_PATH}/out/reader_tests/generated/all_primitive_types/delta' as dt (TYPE delta, PIN_SNAPSHOT) # This query will now reuse query I diff --git a/test/sql/dat/basic_append.test b/test/sql/dat/basic_append.test index 672a1303..d7a7e753 100644 --- a/test/sql/dat/basic_append.test +++ b/test/sql/dat/basic_append.test @@ -16,25 +16,25 @@ require-env DAT_PATH query I SELECT count(*) -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') ---- 5 # Cardinality estimation should correctly show this query II -EXPLAIN FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +EXPLAIN FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') ---- physical_plan :.*5 rows.* query I SELECT count(number) -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') ---- 5 query III SELECT * -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') ---- d 4 4.4 e 5 5.5 @@ -44,7 +44,7 @@ c 3 3.3 query I SELECT letter -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') ---- d e @@ -54,7 +54,7 @@ c query I SELECT number -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') ---- 4 5 @@ -65,7 +65,7 @@ FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') # Now we add a filter that filters out one of the files query II SELECT letter, number -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') WHERE number < 2 ---- a 1 @@ -73,7 +73,7 @@ a 1 # Now we add a filter that filters out the other file query III SELECT a_float, letter, number, -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') WHERE number > 4 ---- 5.5 e 5 @@ -81,19 +81,19 @@ WHERE number > 4 # Now we add a filter that filters out all columns query III SELECT a_float, number, letter -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') WHERE number > 6 ---- # Filters are reflected in cardinality estimation: filtering out all files shows 0 EC query II -EXPLAIN FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +EXPLAIN FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') WHERE number > 6 ---- physical_plan :.*0 rows.* query II -EXPLAIN FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +EXPLAIN FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') WHERE number > 4 ---- physical_plan :.*1 row.* diff --git a/test/sql/dat/custom_parameters.test b/test/sql/dat/custom_parameters.test index 2ac854c3..1607cfda 100644 --- a/test/sql/dat/custom_parameters.test +++ b/test/sql/dat/custom_parameters.test @@ -17,7 +17,7 @@ require-env DAT_PATH # Test with appends and several custom options query IIIII SELECT parse_filename(filename)[-15:-1], file_row_number, letter, delta_file_number, number -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') ---- .snappy.parquet 0 d 7 4 .snappy.parquet 1 e 7 5 @@ -32,7 +32,7 @@ set enable_logging=true # Test with appends and several custom options query IIIII SELECT parse_filename(filename)[-15:-1], file_row_number, letter, delta_file_number, number -FROM delta_scan('${DAT_PATH}/out/reader_tests/generated/basic_append/delta') +FROM delta_scan('{DAT_PATH}/out/reader_tests/generated/basic_append/delta') WHERE filename != 'henk' and letter = 'd' ---- .snappy.parquet 0 d 7 4 @@ -42,5 +42,5 @@ WHERE filename != 'henk' and letter = 'd' query IIIII SELECT filter_type, files_before, files_after, filters_before, filters_after FROM delta_filter_pushdown_log() order by filter_type; ---- -constant 2 1 [] ['letter=\'d\''] -dynamic 1 1 ['letter=\'d\''] ['letter=\'d\''] +constant 2 1 [] ['(letter = \'d\')'] +dynamic 1 1 ['(letter = \'d\')'] ['(letter = \'d\')'] diff --git a/test/sql/delta_kernel_rs/basic_partitioned.test b/test/sql/delta_kernel_rs/basic_partitioned.test index bb63111a..90f06fe7 100644 --- a/test/sql/delta_kernel_rs/basic_partitioned.test +++ b/test/sql/delta_kernel_rs/basic_partitioned.test @@ -9,7 +9,7 @@ require delta require-env DELTA_KERNEL_TESTS_PATH query III -SELECT * FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/basic_partitioned') +SELECT * FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/basic_partitioned') ---- NULL 6 6.6 a 4 4.4 @@ -19,6 +19,6 @@ b 2 2.2 c 3 3.3 query II -EXPLAIN FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/basic_partitioned') +EXPLAIN FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/basic_partitioned') ---- physical_plan :.*6 rows.* diff --git a/test/sql/delta_kernel_rs/logging.test b/test/sql/delta_kernel_rs/logging.test index 0289b75d..71c7f390 100644 --- a/test/sql/delta_kernel_rs/logging.test +++ b/test/sql/delta_kernel_rs/logging.test @@ -22,7 +22,7 @@ statement ok SET enable_logging=true; statement ok -SELECT * FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/basic_partitioned') +SELECT * FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/basic_partitioned') # No kernel logging available yet: we need to set delta_kernel_logging=true #SELECT count(*) FROM duckdb_logs WHERE type='DeltaKernel' @@ -41,7 +41,7 @@ statement ok SET logging_level = 'TRACE'; statement ok -SELECT * FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/basic_partitioned') +SELECT * FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/basic_partitioned') # Now we have log! query I diff --git a/test/sql/delta_kernel_rs/simple_with_dv.test b/test/sql/delta_kernel_rs/simple_with_dv.test index 4a493180..1f6cda8e 100644 --- a/test/sql/delta_kernel_rs/simple_with_dv.test +++ b/test/sql/delta_kernel_rs/simple_with_dv.test @@ -8,27 +8,24 @@ require delta require-env DELTA_KERNEL_TESTS_PATH -statement ok -pragma enable_verification - query I -SELECT count(*) FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') +SELECT count(*) FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') ---- 8 query I -SELECT count(value) FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') +SELECT count(value) FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') ---- 8 query II -SELECT count(value), count(*) FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') +SELECT count(value), count(*) FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') ---- 8 8 # Simplest example query I -FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') +FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') ---- 1 2 @@ -41,7 +38,7 @@ FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') # With filter: ensures the deletion vector is applied properly on top of pushed down filters query I -FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') +FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') WHERE value > 3 ---- 4 @@ -52,7 +49,7 @@ WHERE value > 3 # With filter: ensures the deletion vector is applied properly on top of pushed down filters with the file_row_number column query II -SELECT *, file_row_number FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') +SELECT *, file_row_number FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') WHERE value > 3 ---- 4 4 @@ -63,7 +60,7 @@ WHERE value > 3 # With filter and a delta scan based extra constant column query II -select value, delta_file_number FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') +select value, delta_file_number FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') WHERE value > 3 ---- 4 7 @@ -75,7 +72,7 @@ WHERE value > 3 # With filter, delta-extension-originated const column, and parquet-originated const column query III SELECT value, parse_filename(filename)[-15:-1], delta_file_number -FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') +FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') WHERE value > 3 ---- 4 .snappy.parquet 7 @@ -87,7 +84,7 @@ WHERE value > 3 # With PRUNED filter, delta-extension-originated const column, and parquet-originated const column query II SELECT parse_filename(filename)[-15:-1], delta_file_number -FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') +FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') WHERE value > 3 ---- .snappy.parquet 7 @@ -99,7 +96,7 @@ WHERE value > 3 # With PRUNED filters, delta-extension-originated const column, and parquet-originated const column query I SELECT delta_file_number -FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') +FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') WHERE value > 3 and filename is not null ---- 7 @@ -111,7 +108,7 @@ WHERE value > 3 and filename is not null # Enabling the file_row_number option, but projecting it out query I SELECT value -FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') +FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-with-dv-small/') ---- 1 2 diff --git a/test/sql/delta_kernel_rs/simple_without_dv.test b/test/sql/delta_kernel_rs/simple_without_dv.test index 0526342d..e8101b75 100644 --- a/test/sql/delta_kernel_rs/simple_without_dv.test +++ b/test/sql/delta_kernel_rs/simple_without_dv.test @@ -8,12 +8,9 @@ require delta require-env DELTA_KERNEL_TESTS_PATH -statement ok -pragma enable_verification - # Filename param (i.e. MultiFileReader provided) query II -SELECT value, parse_filename(filename)[-15:-1] FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-without-dv-small', filename=1) +SELECT value, parse_filename(filename)[-15:-1] FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-without-dv-small', filename=1) ---- 0 .snappy.parquet 1 .snappy.parquet @@ -28,7 +25,7 @@ SELECT value, parse_filename(filename)[-15:-1] FROM delta_scan('${DELTA_KERNEL_T # FileRowNumer param (i.e. ParquetReader provided) query II -SELECT *, file_row_number FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/table-without-dv-small') +SELECT *, file_row_number FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/table-without-dv-small') ---- 0 0 1 1 diff --git a/test/sql/delta_kernel_rs/timestamp_ntz.test b/test/sql/delta_kernel_rs/timestamp_ntz.test index 1f89b097..2cb00245 100644 --- a/test/sql/delta_kernel_rs/timestamp_ntz.test +++ b/test/sql/delta_kernel_rs/timestamp_ntz.test @@ -9,14 +9,14 @@ require delta require-env DELTA_KERNEL_TESTS_PATH query IIIIII -DESCRIBE FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/data-reader-timestamp_ntz') order by id +DESCRIBE FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/data-reader-timestamp_ntz') order by id ---- id INTEGER YES NULL NULL NULL tsNtz TIMESTAMP YES NULL NULL NULL tsNtzPartition TIMESTAMP YES NULL NULL NULL query III -FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/data-reader-timestamp_ntz') order by id +FROM delta_scan('{DELTA_KERNEL_TESTS_PATH}/data-reader-timestamp_ntz') order by id ---- 0 2021-11-18 02:30:00.123456 2021-11-18 02:30:00.123456 1 2013-07-05 17:01:00.123456 2021-11-18 02:30:00.123456 diff --git a/test/sql/generated/file_skipping_all_types.test b/test/sql/generated/file_skipping_all_types.test index 197231ba..270ac412 100644 --- a/test/sql/generated/file_skipping_all_types.test +++ b/test/sql/generated/file_skipping_all_types.test @@ -13,17 +13,17 @@ foreach type float double # using column to skip files query II EXPLAIN ANALYZE SELECT value1, value2, value3 -FROM delta_scan('./data/generated/test_file_skipping/${type}/delta_lake') +FROM delta_scan('./data/generated/test_file_skipping/{type}/delta_lake') WHERE value1 > 0.5 and value2 > 2.5 and value3 < 3.5 ---- -analyzed_plan :.*File Filters:.*value1>0.5.*value2>2.5.*value3<3.5.*Scanning Files: 1/5.* +analyzed_plan :.*File Filters:.*value1 > 0.5.*value2 > 2.5.*value3 < 3.5.*Scanning Files: 1/5.* query III SELECT value1, value2, value3 -FROM delta_scan('./data/generated/test_file_skipping/${type}/delta_lake') +FROM delta_scan('./data/generated/test_file_skipping/{type}/delta_lake') WHERE value1 > 0.5 and value2 > 2.5 and @@ -33,7 +33,7 @@ WHERE query II EXPLAIN ANALYZE SELECT part -FROM delta_scan('./data/generated/test_file_skipping/${type}/delta_lake') +FROM delta_scan('./data/generated/test_file_skipping/{type}/delta_lake') WHERE part > 0.5 ---- analyzed_plan :.* Scanning Files: 4/5.* @@ -46,7 +46,7 @@ EXPLAIN ANALYZE SELECT * FROM delta_scan('./data/generated/test_file_skipping/bool/delta_lake') WHERE value1=false ---- -analyzed_plan :.*File Filters:.*value1=false.*Scanning Files: 1/2.* +analyzed_plan :.*File Filters:.*value1 = false.*Scanning Files: 1/2.* query II EXPLAIN ANALYZE SELECT part @@ -60,17 +60,17 @@ foreach type int tinyint smallint bigint # using column to skip files query II EXPLAIN ANALYZE SELECT value1, value2, value3 -FROM delta_scan('./data/generated/test_file_skipping/${type}/delta_lake') +FROM delta_scan('./data/generated/test_file_skipping/{type}/delta_lake') WHERE value1 > 1 and value2 > 2 and value3 < 4 ---- -analyzed_plan :.*File Filters:.*value1>1.*value2>2.*value3<4.*Scanning Files: 1/5.* +analyzed_plan :.*File Filters:.*value1 > 1.*value2 > 2.*value3 < 4.*Scanning Files: 1/5.* query III SELECT value1, value2, value3 -FROM delta_scan('./data/generated/test_file_skipping/${type}/delta_lake') +FROM delta_scan('./data/generated/test_file_skipping/{type}/delta_lake') WHERE value1 > 1 and value2 > 2 and @@ -80,7 +80,7 @@ WHERE query II EXPLAIN ANALYZE SELECT part -FROM delta_scan('./data/generated/test_file_skipping/${type}/delta_lake') +FROM delta_scan('./data/generated/test_file_skipping/{type}/delta_lake') WHERE part = 0 ---- analyzed_plan :.* Scanning Files: 1/5.* @@ -96,7 +96,7 @@ WHERE value2 = '2' and value3 = '2' ---- -analyzed_plan :.*File Filters:.*value1='2'.*value2='2'.*value3='2'.*Scanning Files: 1/5.* +analyzed_plan :.*File Filters:.*value1 = '2'.*value2 = '2'.*value3 = '2'.*Scanning Files: 1/5.* query III SELECT value1, value2, value3 @@ -124,7 +124,7 @@ FROM delta_scan('./data/generated/test_file_skipping/date/delta_lake') WHERE value1 = '1994-01-01'::DATE ---- -analyzed_plan :.*File Filters:.*value1='1994-01-01'.*Scanning Files: 1/5.* +analyzed_plan :.*File Filters:.*value1 = '1994-01-01'.*Scanning Files: 1/5.* query II SELECT value1, part @@ -148,7 +148,7 @@ EXPLAIN ANALYZE SELECT value1, part FROM delta_scan('./data/generated/test_file_skipping/decimal/delta_lake') WHERE value1 = 3.00::DECIMAL(10,2) ---- -analyzed_plan :.*File Filters:.*value1=3.*Scanning Files: 1/5.* +analyzed_plan :.*File Filters:.*value1 = 3.*Scanning Files: 1/5.* query II SELECT value1, part diff --git a/test/sql/generated/file_skipping_dynamic.test b/test/sql/generated/file_skipping_dynamic.test index fead510a..a661a13b 100644 --- a/test/sql/generated/file_skipping_dynamic.test +++ b/test/sql/generated/file_skipping_dynamic.test @@ -23,7 +23,7 @@ query IIIII SELECT filter_type, filters_before, filters_after, files_before, files_after FROM delta_filter_pushdown_log() ---- -constant [] ['value2=102', 'value3=1002'] 5 1 +constant [] ['(value2 = 102)', '(value3 = 1002)'] 5 1 # Clear logging storage statement ok @@ -41,7 +41,7 @@ query IIIII SELECT filter_type, filters_before, filters_after, files_before, files_after FROM delta_filter_pushdown_log() ---- -constant [] ['value3=1002'] 5 1 +constant [] ['(value3 = 1002)'] 5 1 # Clear logging storage statement ok @@ -59,7 +59,7 @@ query IIIII SELECT filter_type, filters_before, filters_after, files_before, files_after FROM delta_filter_pushdown_log() ---- -dynamic [] ['value2=102'] 5 1 +dynamic [] ['(value2 = 102)'] 5 1 # Clear logging storage statement ok @@ -77,7 +77,7 @@ query IIIII SELECT filter_type, filters_before, filters_after, files_before, files_after FROM delta_filter_pushdown_log() ---- -dynamic [] ['value1=13'] 5 1 +dynamic [] ['(value1 = 13)'] 5 1 # Clear logging storage statement ok @@ -95,7 +95,7 @@ query IIIII SELECT filter_type, filters_before, filters_after, files_before, files_after FROM delta_filter_pushdown_log() ---- -dynamic [] ['part=2'] 5 1 +dynamic [] ['(part = 2)'] 5 1 # Clear logging storage statement ok @@ -115,8 +115,8 @@ SELECT filter_type, filters_before, filters_after, files_before, files_after FROM delta_filter_pushdown_log() ORDER BY filter_type ---- -constant [] ['value1=13'] 5 1 -dynamic ['value1=13'] ['value1=13', 'value2=103'] 1 1 +constant [] ['(value1 = 13)'] 5 1 +dynamic ['(value1 = 13)'] ['(value1 = 13)', '(value2 = 103)'] 1 1 # Clear logging storage statement ok @@ -136,8 +136,8 @@ SELECT filter_type, filters_before, filters_after, files_before, files_after FROM delta_filter_pushdown_log() ORDER BY filter_type ---- -constant [] ['value1=13'] 5 1 -dynamic ['value1=13'] ['value1=13 AND value1=13 AND value1=13'] 1 1 +constant [] ['(value1 = 13)'] 5 1 +dynamic ['(value1 = 13)'] ['((value1 = 13) AND (value1 = 13))'] 1 1 # Clear logging storage statement ok @@ -167,5 +167,5 @@ SELECT filter_type, filters_before, filters_after, files_before, files_after FROM delta_filter_pushdown_log() ORDER BY filter_type ---- -constant [] ['value1=12'] NULL NULL -dynamic [] ['value1=12'] NULL NULL +constant [] ['(value1 = 12)'] NULL NULL +dynamic [] ['(value1 = 12)'] NULL NULL diff --git a/test/sql/generated/late_materialization.test b/test/sql/generated/late_materialization.test index 348fdcf2..7ab1f9ff 100644 --- a/test/sql/generated/late_materialization.test +++ b/test/sql/generated/late_materialization.test @@ -8,9 +8,6 @@ require delta require-env GENERATED_DATA_AVAILABLE -statement ok -pragma enable_verification - statement ok CREATE VIEW lineitem AS FROM delta_scan('data/generated/tpch_sf0_01/lineitem/delta_lake'); @@ -20,9 +17,5 @@ FROM lineitem ORDER BY l_orderkey, l_linenumber DESC LIMIT 5; statement ok ATTACH 'data/generated/tpch_sf0_01/lineitem/delta_lake' as dt (TYPE delta); -# TODO: figure out why the serialization verification breaks here -statement ok -pragma disable_verification - statement ok FROM dt ORDER BY l_orderkey DESC LIMIT 5; diff --git a/test/sql/generated/partitioned_large.test b/test/sql/generated/partitioned_large.test index b8506bfc..b1aa24a1 100644 --- a/test/sql/generated/partitioned_large.test +++ b/test/sql/generated/partitioned_large.test @@ -17,7 +17,7 @@ CREATE VIEW t AS SELECT part::INT as part, sum(i) as value query II EXPLAIN FROM t ---- -physical_plan :.*PARTITIONED_AGGREGATE.* +physical_plan :.*Partitioned Aggregate.* # With a projection and delta constant column query II @@ -54,7 +54,7 @@ CREATE VIEW t2 AS SELECT part::INT as part, sum(i) as value query II EXPLAIN FROM t2 ---- -physical_plan :.*PARTITIONED_AGGREGATE.* +physical_plan :.*Partitioned Aggregate.* # With a projection and delta constant column query II @@ -94,7 +94,7 @@ CREATE VIEW t3 AS SELECT part::INT as part, sum(i) as value query II EXPLAIN FROM t3 ---- -physical_plan :.*PARTITIONED_AGGREGATE.* +physical_plan :.*Partitioned Aggregate.* # Then for ATTACH, with default settings partition info pushdown statement ok @@ -109,5 +109,5 @@ CREATE VIEW t4 AS SELECT part::INT as part, sum(i) as value query II EXPLAIN FROM t4 ---- -physical_plan :.*PARTITIONED_AGGREGATE.* +physical_plan :.*Partitioned Aggregate.* diff --git a/test/sql/generated/tpcds.test_slow b/test/sql/generated/tpcds.test_slow index b6f9d061..b3188c91 100644 --- a/test/sql/generated/tpcds.test_slow +++ b/test/sql/generated/tpcds.test_slow @@ -17,32 +17,32 @@ require-env GENERATED_DATA_AVAILABLE foreach table call_center catalog_page catalog_returns catalog_sales customer customer_demographics customer_address date_dim household_demographics inventory income_band item promotion reason ship_mode store store_returns store_sales time_dim warehouse web_page web_returns web_sales web_site statement ok -create view ${table}_delta as from delta_scan('./data/generated/tpcds_sf0_01/${table}/delta_lake'); +create view {table}_delta as from delta_scan('./data/generated/tpcds_sf0_01/{table}/delta_lake'); statement ok -create view ${table}_parquet as from parquet_scan('./data/generated/tpcds_sf0_01/${table}/parquet/**/*.parquet'); +create view {table}_parquet as from parquet_scan('./data/generated/tpcds_sf0_01/{table}/parquet/**/*.parquet'); # NOTE: switch this to _parquet to easily compare plans while debugging statement ok -create view ${table} as from ${table}_delta +create view {table} as from {table}_delta endloop loop i 1 9 query I -PRAGMA tpcds(${i}) +PRAGMA tpcds({i}) ---- -:duckdb/extension/tpcds/dsdgen/answers/sf0.01/0${i}.csv +:duckdb/extension/tpcds/dsdgen/answers/sf0.01/0{i}.csv endloop loop i 10 99 query I -PRAGMA tpcds(${i}) +PRAGMA tpcds({i}) ---- -:duckdb/extension/tpcds/dsdgen/answers/sf0.01/${i}.csv +:duckdb/extension/tpcds/dsdgen/answers/sf0.01/{i}.csv endloop diff --git a/test/sql/generated/tpch.test_slow b/test/sql/generated/tpch.test_slow index 201c6c85..931fcca4 100644 --- a/test/sql/generated/tpch.test_slow +++ b/test/sql/generated/tpch.test_slow @@ -19,32 +19,32 @@ require-env GENERATED_DATA_AVAILABLE foreach table customer lineitem nation orders part partsupp region supplier statement ok -create view ${table}_delta as from delta_scan('./data/generated/tpch_sf1/${table}/delta_lake'); +create view {table}_delta as from delta_scan('./data/generated/tpch_sf1/{table}/delta_lake'); statement ok -create view ${table}_parquet as from parquet_scan('./data/generated/tpch_sf1/${table}/parquet/**/*.parquet'); +create view {table}_parquet as from parquet_scan('./data/generated/tpch_sf1/{table}/parquet/**/*.parquet'); # NOTE: switch this to _parquet to easily compare plans while debugging statement ok -create view ${table} as from ${table}_delta +create view {table} as from {table}_delta endloop loop i 1 9 query I -PRAGMA tpch(${i}) +PRAGMA tpch({i}) ---- -:duckdb/extension/tpch/dbgen/answers/sf1/q0${i}.csv +:duckdb/extension/tpch/dbgen/answers/sf1/q0{i}.csv endloop loop i 10 23 query I -PRAGMA tpch(${i}) +PRAGMA tpch({i}) ---- -:duckdb/extension/tpch/dbgen/answers/sf1/q${i}.csv +:duckdb/extension/tpch/dbgen/answers/sf1/q{i}.csv endloop @@ -52,13 +52,13 @@ foreach table customer lineitem nation orders part partsupp region supplier # Cleanup statement ok -DROP VIEW ${table} +DROP VIEW {table} statement ok -DROP VIEW ${table}_delta +DROP VIEW {table}_delta statement ok -DROP VIEW ${table}_parquet +DROP VIEW {table}_parquet endloop @@ -68,24 +68,24 @@ endloop foreach table customer lineitem nation orders part partsupp region supplier statement ok -create view ${table} as from delta_scan('./data/generated/tpch_sf0_01/${table}/delta_lake'); +create view {table} as from delta_scan('./data/generated/tpch_sf0_01/{table}/delta_lake'); endloop loop i 1 9 query I -PRAGMA tpch(${i}) +PRAGMA tpch({i}) ---- -:duckdb/extension/tpch/dbgen/answers/sf0.01/q0${i}.csv +:duckdb/extension/tpch/dbgen/answers/sf0.01/q0{i}.csv endloop loop i 10 23 query I -PRAGMA tpch(${i}) +PRAGMA tpch({i}) ---- -:duckdb/extension/tpch/dbgen/answers/sf0.01/q${i}.csv +:duckdb/extension/tpch/dbgen/answers/sf0.01/q{i}.csv endloop diff --git a/test/sql/generated/writing/append/basic_append.test b/test/sql/generated/writing/append/basic_append.test index 2e404799..1f7955e0 100644 --- a/test/sql/generated/writing/append/basic_append.test +++ b/test/sql/generated/writing/append/basic_append.test @@ -10,12 +10,12 @@ require-env GENERATED_DATA_AVAILABLE # Copy test data over to tmp test dir because we'll be inserting stuff statement ok -from copy_dir('data/generated/simple_table', '__TEST_DIR__/late_materialization/simple_table'); -from copy_dir('data/generated/simple_table_partitioned', '__TEST_DIR__/late_materialization/simple_table_partitioned'); -from copy_dir('data/generated/simple_table_column_mapped', '__TEST_DIR__/late_materialization/simple_table_column_mapped'); +from copy_dir('data/generated/simple_table', '{TEMP_DIR}/late_materialization/simple_table'); +from copy_dir('data/generated/simple_table_partitioned', '{TEMP_DIR}/late_materialization/simple_table_partitioned'); +from copy_dir('data/generated/simple_table_column_mapped', '{TEMP_DIR}/late_materialization/simple_table_column_mapped'); statement ok -ATTACH '__TEST_DIR__/late_materialization/simple_table/delta_lake' AS simple_table (TYPE delta); +ATTACH '{TEMP_DIR}/late_materialization/simple_table/delta_lake' AS simple_table (TYPE delta); query II select count(*), sum(i) FROM simple_table; @@ -23,7 +23,7 @@ select count(*), sum(i) FROM simple_table; 10 45 query II -select count(*), sum(i) FROM read_parquet('__TEST_DIR__/late_materialization/simple_table/delta_lake/**/*.parquet', hive_partitioning=0); +select count(*), sum(i) FROM read_parquet('{TEMP_DIR}/late_materialization/simple_table/delta_lake/**/*.parquet', hive_partitioning=0); ---- 10 45 @@ -36,7 +36,7 @@ select count(*), sum(i) FROM simple_table; 11 50 statement ok -ATTACH '__TEST_DIR__/late_materialization/simple_table_partitioned/delta_lake' AS simple_table_partitioned (TYPE delta); +ATTACH '{TEMP_DIR}/late_materialization/simple_table_partitioned/delta_lake' AS simple_table_partitioned (TYPE delta); query III select count(*), sum(i), sum(part) FROM simple_table_partitioned; @@ -45,7 +45,7 @@ select count(*), sum(i), sum(part) FROM simple_table_partitioned; # We write in hive_partitioned format, so this should also pick things up query III -select count(*), sum(i), sum(part) FROM read_parquet('__TEST_DIR__/late_materialization/simple_table_partitioned/delta_lake/**/*.parquet', hive_partitioning=1); +select count(*), sum(i), sum(part) FROM read_parquet('{TEMP_DIR}/late_materialization/simple_table_partitioned/delta_lake/**/*.parquet', hive_partitioning=1); ---- 10 45 5 @@ -59,18 +59,18 @@ select count(*), sum(i), sum(part) FROM simple_table_partitioned; # We write in hive_partitioned format, so this should also pick things up query III -select count(*), sum(i), sum(part) FROM read_parquet('__TEST_DIR__/late_materialization/simple_table_partitioned/delta_lake/**/*.parquet', hive_partitioning=1); +select count(*), sum(i), sum(part) FROM read_parquet('{TEMP_DIR}/late_materialization/simple_table_partitioned/delta_lake/**/*.parquet', hive_partitioning=1); ---- 11 50 6 # The files should not contain the partition values though query IIIIII -DESCRIBE FROM read_parquet('__TEST_DIR__/late_materialization/simple_table_partitioned/delta_lake/**/*.parquet', hive_partitioning=0); +DESCRIBE FROM read_parquet('{TEMP_DIR}/late_materialization/simple_table_partitioned/delta_lake/**/*.parquet', hive_partitioning=0); ---- i BIGINT YES NULL NULL NULL statement ok -ATTACH '__TEST_DIR__/late_materialization/simple_table_column_mapped/delta_lake' AS simple_table_column_mapped (TYPE delta); +ATTACH '{TEMP_DIR}/late_materialization/simple_table_column_mapped/delta_lake' AS simple_table_column_mapped (TYPE delta); query IIII select count(*), sum(i), sum(part), sum(new_col) FROM simple_table_column_mapped; diff --git a/test/sql/generated/writing/append/test_arrow_release.test b/test/sql/generated/writing/append/test_arrow_release.test index 71da9ba3..ecde5ead 100644 --- a/test/sql/generated/writing/append/test_arrow_release.test +++ b/test/sql/generated/writing/append/test_arrow_release.test @@ -13,10 +13,10 @@ statement ok CALL enable_logging(level='trace') statement ok -CALL copy_dir('data/generated/simple_table', '__TEST_DIR__/late_materialization/simple_table'); +CALL copy_dir('data/generated/simple_table', '{TEMP_DIR}/late_materialization/simple_table'); statement ok -ATTACH '__TEST_DIR__/late_materialization/simple_table/delta_lake' AS simple_table (TYPE delta); +ATTACH '{TEMP_DIR}/late_materialization/simple_table/delta_lake' AS simple_table (TYPE delta); loop i 0 10 diff --git a/test/sql/generated/writing/append/tpch_append.test_slow b/test/sql/generated/writing/append/tpch_append.test_slow index 5dc5d0cf..a9236354 100644 --- a/test/sql/generated/writing/append/tpch_append.test_slow +++ b/test/sql/generated/writing/append/tpch_append.test_slow @@ -19,32 +19,32 @@ call dbgen(sf=1, suffix='_mem') # Copy test data over to tmp test dir because we'll be inserting stuff statement ok -from copy_dir('data/generated/tpch_sf0', '__TEST_DIR__/tpch_append/tpch_sf0'); +from copy_dir('data/generated/tpch_sf0', '{TEMP_DIR}/tpch_append/tpch_sf0'); foreach table customer lineitem nation orders part partsupp region supplier statement ok -ATTACH '__TEST_DIR__/tpch_append/tpch_sf0/${table}/delta_lake' AS ${table} (TYPE delta); +ATTACH '{TEMP_DIR}/tpch_append/tpch_sf0/{table}/delta_lake' AS {table} (TYPE delta); statement ok -INSERT INTO ${table} from ${table}_mem +INSERT INTO {table} from {table}_mem endloop loop i 1 9 query I -PRAGMA tpch(${i}) +PRAGMA tpch({i}) ---- -:duckdb/extension/tpch/dbgen/answers/sf1/q0${i}.csv +:duckdb/extension/tpch/dbgen/answers/sf1/q0{i}.csv endloop loop i 10 23 query I -PRAGMA tpch(${i}) +PRAGMA tpch({i}) ---- -:duckdb/extension/tpch/dbgen/answers/sf1/q${i}.csv +:duckdb/extension/tpch/dbgen/answers/sf1/q{i}.csv endloop diff --git a/test/sql/generated/writing/append/transaction_append.test b/test/sql/generated/writing/append/transaction_append.test index 2d34b8ed..5ee08f2c 100644 --- a/test/sql/generated/writing/append/transaction_append.test +++ b/test/sql/generated/writing/append/transaction_append.test @@ -10,10 +10,10 @@ require-env GENERATED_DATA_AVAILABLE # Copy test data over to tmp test dir because we'll be inserting stuff statement ok -from copy_dir('data/generated/simple_table', '__TEST_DIR__/transaction_append/simple_table'); +from copy_dir('data/generated/simple_table', '{TEMP_DIR}/transaction_append/simple_table'); statement ok -ATTACH '__TEST_DIR__/transaction_append/simple_table/delta_lake' AS simple_table (TYPE delta); +ATTACH '{TEMP_DIR}/transaction_append/simple_table/delta_lake' AS simple_table (TYPE delta); query II select count(*), sum(i) FROM simple_table; @@ -21,7 +21,7 @@ select count(*), sum(i) FROM simple_table; 10 45 query I -select count(*) FROM glob('__TEST_DIR__/transaction_append/simple_table/delta_lake/**/*.parquet') +select count(*) FROM glob('{TEMP_DIR}/transaction_append/simple_table/delta_lake/**/*.parquet') ---- 1 @@ -36,7 +36,7 @@ statement ok INSERT INTO simple_table VALUES (5); query I -select count(*) FROM glob('__TEST_DIR__/transaction_append/simple_table/delta_lake/**/*.parquet') +select count(*) FROM glob('{TEMP_DIR}/transaction_append/simple_table/delta_lake/**/*.parquet') ---- 2 @@ -51,7 +51,7 @@ select count(*), sum(i) FROM simple_table; # The file is automatically cleaned up by DuckDB on rollback query I -select count(*) FROM glob('__TEST_DIR__/transaction_append/simple_table/delta_lake/**/*.parquet') +select count(*) FROM glob('{TEMP_DIR}/transaction_append/simple_table/delta_lake/**/*.parquet') ---- 1 @@ -101,7 +101,7 @@ COMMIT # there are now 4 files: of which 1 is uncommitted query I -select count(*) FROM glob('__TEST_DIR__/transaction_append/simple_table/delta_lake/**/*.parquet') +select count(*) FROM glob('{TEMP_DIR}/transaction_append/simple_table/delta_lake/**/*.parquet') ---- 4 @@ -113,7 +113,7 @@ TransactionContext Error: Failed to commit: DeltaKernel GenericError (5): Generi # The uncommitted file was cleaned up on rollback query I -select count(*) FROM glob('__TEST_DIR__/transaction_append/simple_table/delta_lake/**/*.parquet') +select count(*) FROM glob('{TEMP_DIR}/transaction_append/simple_table/delta_lake/**/*.parquet') ---- 3 diff --git a/test/sql/generated/writing/append/write_null_partition.test b/test/sql/generated/writing/append/write_null_partition.test index a0bde81c..dde0290e 100644 --- a/test/sql/generated/writing/append/write_null_partition.test +++ b/test/sql/generated/writing/append/write_null_partition.test @@ -11,10 +11,10 @@ require json require-env GENERATED_DATA_AVAILABLE statement ok -from copy_dir('data/generated/simple_partitioned', '__TEST_DIR__/write_null_partition/simple_partitioned'); +from copy_dir('data/generated/simple_partitioned', '{TEMP_DIR}/write_null_partition/simple_partitioned'); statement ok -ATTACH '__TEST_DIR__/write_null_partition/simple_partitioned/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_null_partition/simple_partitioned/delta_lake' AS t (TYPE delta); # Baseline query II @@ -31,7 +31,7 @@ INSERT INTO t VALUES (99, NULL); # have "" for the null value. query I SELECT json_extract_string(add.partitionValues, '$.part') -FROM read_json('__TEST_DIR__/write_null_partition/simple_partitioned/delta_lake/_delta_log/00000000000000000001.json') +FROM read_json('{TEMP_DIR}/write_null_partition/simple_partitioned/delta_lake/_delta_log/00000000000000000001.json') WHERE add IS NOT NULL; ---- (empty) @@ -50,4 +50,4 @@ SELECT i FROM t WHERE part IS NULL; # Confirm that part=NULL directory is encoded as part=__HIVE_DEFAULT_PARTITION__ statement ok -FROM read_parquet('__TEST_DIR__/write_null_partition/simple_partitioned/delta_lake/part=__HIVE_DEFAULT_PARTITION__/*.parquet'); +FROM read_parquet('{TEMP_DIR}/write_null_partition/simple_partitioned/delta_lake/part=__HIVE_DEFAULT_PARTITION__/*.parquet'); diff --git a/test/sql/generated/writing/append/write_stats.test b/test/sql/generated/writing/append/write_stats.test index f5f0c9ec..6d947980 100644 --- a/test/sql/generated/writing/append/write_stats.test +++ b/test/sql/generated/writing/append/write_stats.test @@ -12,10 +12,10 @@ require-env GENERATED_DATA_AVAILABLE # Copy test data over to tmp test dir because we'll be inserting stuff statement ok -from copy_dir('data/generated/simple_table', '__TEST_DIR__/write_stats/simple_table'); +from copy_dir('data/generated/simple_table', '{TEMP_DIR}/write_stats/simple_table'); statement ok -ATTACH '__TEST_DIR__/write_stats/simple_table/delta_lake' AS simple_table (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats/simple_table/delta_lake' AS simple_table (TYPE delta); query II select count(*), sum(i) FROM simple_table; @@ -31,7 +31,7 @@ select count(*), sum(i) FROM simple_table; 11 50 query I -select add.stats FROM read_json('__TEST_DIR__/write_stats/simple_table/delta_lake/_delta_log/00000000000000000002.json') +select add.stats FROM read_json('{TEMP_DIR}/write_stats/simple_table/delta_lake/_delta_log/00000000000000000002.json') ---- NULL {"numRecords":1,"nullCount":{"i":0},"minValues":{"i":5},"maxValues":{"i":5},"tightBounds":true} @@ -40,7 +40,7 @@ statement ok INSERT INTO simple_table FROM range(0,10) t(i) ; query I -select add.stats FROM read_json('__TEST_DIR__/write_stats/simple_table/delta_lake/_delta_log/00000000000000000003.json') +select add.stats FROM read_json('{TEMP_DIR}/write_stats/simple_table/delta_lake/_delta_log/00000000000000000003.json') ---- NULL {"numRecords":10,"nullCount":{"i":0},"minValues":{"i":0},"maxValues":{"i":9},"tightBounds":true} diff --git a/test/sql/generated/writing/append/write_stats_list.test b/test/sql/generated/writing/append/write_stats_list.test index 6c2d1220..99067659 100644 --- a/test/sql/generated/writing/append/write_stats_list.test +++ b/test/sql/generated/writing/append/write_stats_list.test @@ -11,10 +11,10 @@ require json require-env GENERATED_DATA_AVAILABLE statement ok -from copy_dir('data/generated/simple_table_list', '__TEST_DIR__/write_stats_list/simple_table_list'); +from copy_dir('data/generated/simple_table_list', '{TEMP_DIR}/write_stats_list/simple_table_list'); statement ok -ATTACH '__TEST_DIR__/write_stats_list/simple_table_list/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats_list/simple_table_list/delta_lake' AS t (TYPE delta); statement ok INSERT INTO t VALUES (10, [10, 11, 12]); @@ -24,7 +24,7 @@ query II SELECT json_extract(add.stats, '$.nullCount.items') IS NULL, json_extract(add.stats, '$.minValues.items') IS NULL -FROM read_json('__TEST_DIR__/write_stats_list/simple_table_list/delta_lake/_delta_log/00000000000000000002.json') +FROM read_json('{TEMP_DIR}/write_stats_list/simple_table_list/delta_lake/_delta_log/00000000000000000002.json') WHERE add IS NOT NULL ---- true true diff --git a/test/sql/generated/writing/append/write_stats_nested.test b/test/sql/generated/writing/append/write_stats_nested.test index d68fce68..104269be 100644 --- a/test/sql/generated/writing/append/write_stats_nested.test +++ b/test/sql/generated/writing/append/write_stats_nested.test @@ -17,10 +17,10 @@ require-env GENERATED_DATA_AVAILABLE #------------------------------------------------------------------------------ statement ok -from copy_dir('data/generated/simple_table_nested', '__TEST_DIR__/write_stats_nested/simple_table_nested'); +from copy_dir('data/generated/simple_table_nested', '{TEMP_DIR}/write_stats_nested/simple_table_nested'); statement ok -ATTACH '__TEST_DIR__/write_stats_nested/simple_table_nested/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats_nested/simple_table_nested/delta_lake' AS t (TYPE delta); query II SELECT count(*), sum(i.a.b) FROM t; @@ -33,7 +33,7 @@ INSERT INTO t VALUES ({'a': {'b': 10}}); # Full stats JSON is deterministic for a single-leaf nested column query I SELECT add.stats -FROM read_json('__TEST_DIR__/write_stats_nested/simple_table_nested/delta_lake/_delta_log/00000000000000000002.json') +FROM read_json('{TEMP_DIR}/write_stats_nested/simple_table_nested/delta_lake/_delta_log/00000000000000000002.json') WHERE add IS NOT NULL ---- {"numRecords":1,"nullCount":{"i":{"a":{"b":0}}},"minValues":{"i":{"a":{"b":10}}},"maxValues":{"i":{"a":{"b":10}}},"tightBounds":true} @@ -70,10 +70,10 @@ DETACH t; #------------------------------------------------------------------------------ statement ok -from copy_dir('data/generated/simple_partitioned_with_structs', '__TEST_DIR__/write_stats_nested/simple_partitioned_with_structs'); +from copy_dir('data/generated/simple_partitioned_with_structs', '{TEMP_DIR}/write_stats_nested/simple_partitioned_with_structs'); statement ok -ATTACH '__TEST_DIR__/write_stats_nested/simple_partitioned_with_structs/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats_nested/simple_partitioned_with_structs/delta_lake' AS t (TYPE delta); statement ok INSERT INTO t VALUES ({'i': 100, 'j': 100}, 99); @@ -86,7 +86,7 @@ SELECT json_extract_string(add.stats, '$.minValues.value.i'), json_extract_string(add.stats, '$.maxValues.value.j'), json_extract_string(add.stats, '$.tightBounds') -FROM read_json('__TEST_DIR__/write_stats_nested/simple_partitioned_with_structs/delta_lake/_delta_log/00000000000000000001.json') +FROM read_json('{TEMP_DIR}/write_stats_nested/simple_partitioned_with_structs/delta_lake/_delta_log/00000000000000000001.json') WHERE add IS NOT NULL ---- 1 0 100 100 true diff --git a/test/sql/generated/writing/append/write_stats_primitives.test b/test/sql/generated/writing/append/write_stats_primitives.test index b7ccf572..9abbca42 100644 --- a/test/sql/generated/writing/append/write_stats_primitives.test +++ b/test/sql/generated/writing/append/write_stats_primitives.test @@ -20,10 +20,10 @@ require-env GENERATED_DATA_AVAILABLE foreach type int tinyint smallint bigint statement ok -from copy_dir('data/generated/test_file_skipping/${type}', '__TEST_DIR__/write_stats_all_types/${type}'); +from copy_dir('data/generated/test_file_skipping/{type}', '{TEMP_DIR}/write_stats_all_types/{type}'); statement ok -ATTACH '__TEST_DIR__/write_stats_all_types/${type}/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats_all_types/{type}/delta_lake' AS t (TYPE delta); statement ok INSERT INTO t (value1, value2, value3, part) VALUES (5, 5, 5, 5); @@ -36,7 +36,7 @@ SELECT json_extract_string(add.stats, '$.minValues.value1'), json_extract_string(add.stats, '$.maxValues.value1'), json_extract_string(add.stats, '$.tightBounds') -FROM read_json('__TEST_DIR__/write_stats_all_types/${type}/delta_lake/_delta_log/00000000000000000001.json') +FROM read_json('{TEMP_DIR}/write_stats_all_types/{type}/delta_lake/_delta_log/00000000000000000001.json') WHERE add IS NOT NULL ---- 1 0 5 5 true @@ -66,10 +66,10 @@ endloop foreach type float double statement ok -from copy_dir('data/generated/test_file_skipping/${type}', '__TEST_DIR__/write_stats_all_types/${type}'); +from copy_dir('data/generated/test_file_skipping/{type}', '{TEMP_DIR}/write_stats_all_types/{type}'); statement ok -ATTACH '__TEST_DIR__/write_stats_all_types/${type}/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats_all_types/{type}/delta_lake' AS t (TYPE delta); statement ok INSERT INTO t (value1, value2, value3, part) VALUES (5.0, 5.0, 5.0, 5.0); @@ -81,7 +81,7 @@ SELECT json_extract_string(add.stats, '$.minValues.value1'), json_extract_string(add.stats, '$.maxValues.value1'), json_extract_string(add.stats, '$.tightBounds') -FROM read_json('__TEST_DIR__/write_stats_all_types/${type}/delta_lake/_delta_log/00000000000000000001.json') +FROM read_json('{TEMP_DIR}/write_stats_all_types/{type}/delta_lake/_delta_log/00000000000000000001.json') WHERE add IS NOT NULL ---- 1 0 5.0 5.0 true @@ -107,10 +107,10 @@ endloop #------------------------------------------------------------------------------ statement ok -from copy_dir('data/generated/test_file_skipping/varchar', '__TEST_DIR__/write_stats_all_types/varchar'); +from copy_dir('data/generated/test_file_skipping/varchar', '{TEMP_DIR}/write_stats_all_types/varchar'); statement ok -ATTACH '__TEST_DIR__/write_stats_all_types/varchar/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats_all_types/varchar/delta_lake' AS t (TYPE delta); statement ok INSERT INTO t (value1, value2, value3, part) VALUES ('5', '5', '5', '5'); @@ -122,7 +122,7 @@ SELECT json_extract_string(add.stats, '$.minValues.value1'), json_extract_string(add.stats, '$.maxValues.value1'), json_extract_string(add.stats, '$.tightBounds') -FROM read_json('__TEST_DIR__/write_stats_all_types/varchar/delta_lake/_delta_log/00000000000000000001.json') +FROM read_json('{TEMP_DIR}/write_stats_all_types/varchar/delta_lake/_delta_log/00000000000000000001.json') WHERE add IS NOT NULL ---- 1 0 5 5 true @@ -147,10 +147,10 @@ DETACH t; #------------------------------------------------------------------------------ statement ok -from copy_dir('data/generated/test_file_skipping/date', '__TEST_DIR__/write_stats_all_types/date'); +from copy_dir('data/generated/test_file_skipping/date', '{TEMP_DIR}/write_stats_all_types/date'); statement ok -ATTACH '__TEST_DIR__/write_stats_all_types/date/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats_all_types/date/delta_lake' AS t (TYPE delta); statement ok INSERT INTO t (value1, part) VALUES ('1994-01-06'::DATE, '1994-01-06'::DATE); @@ -162,7 +162,7 @@ SELECT json_extract_string(add.stats, '$.minValues.value1'), json_extract_string(add.stats, '$.maxValues.value1'), json_extract_string(add.stats, '$.tightBounds') -FROM read_json('__TEST_DIR__/write_stats_all_types/date/delta_lake/_delta_log/00000000000000000001.json') +FROM read_json('{TEMP_DIR}/write_stats_all_types/date/delta_lake/_delta_log/00000000000000000001.json') WHERE add IS NOT NULL ---- 1 0 1994-01-06 1994-01-06 true @@ -187,10 +187,10 @@ DETACH t; #------------------------------------------------------------------------------ statement ok -from copy_dir('data/generated/test_file_skipping/bool', '__TEST_DIR__/write_stats_all_types/bool'); +from copy_dir('data/generated/test_file_skipping/bool', '{TEMP_DIR}/write_stats_all_types/bool'); statement ok -ATTACH '__TEST_DIR__/write_stats_all_types/bool/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats_all_types/bool/delta_lake' AS t (TYPE delta); statement ok INSERT INTO t (value1, value2, value3, part) VALUES (true, true, true, true); @@ -202,7 +202,7 @@ SELECT json_extract_string(add.stats, '$.minValues.value1'), json_extract_string(add.stats, '$.maxValues.value1'), json_extract_string(add.stats, '$.tightBounds') -FROM read_json('__TEST_DIR__/write_stats_all_types/bool/delta_lake/_delta_log/00000000000000000001.json') +FROM read_json('{TEMP_DIR}/write_stats_all_types/bool/delta_lake/_delta_log/00000000000000000001.json') WHERE add IS NOT NULL ---- 1 0 true true true @@ -221,10 +221,10 @@ DETACH t; #------------------------------------------------------------------------------ statement ok -from copy_dir('data/generated/test_file_skipping/decimal', '__TEST_DIR__/write_stats_all_types/decimal'); +from copy_dir('data/generated/test_file_skipping/decimal', '{TEMP_DIR}/write_stats_all_types/decimal'); statement ok -ATTACH '__TEST_DIR__/write_stats_all_types/decimal/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats_all_types/decimal/delta_lake' AS t (TYPE delta); statement ok INSERT INTO t (value1, part) VALUES (6.0::DECIMAL(10,2), 6.0::DECIMAL(10,2)); @@ -236,7 +236,7 @@ SELECT json_extract_string(add.stats, '$.minValues.value1'), json_extract_string(add.stats, '$.maxValues.value1'), json_extract_string(add.stats, '$.tightBounds') -FROM read_json('__TEST_DIR__/write_stats_all_types/decimal/delta_lake/_delta_log/00000000000000000001.json') +FROM read_json('{TEMP_DIR}/write_stats_all_types/decimal/delta_lake/_delta_log/00000000000000000001.json') WHERE add IS NOT NULL ---- 1 0 6.0 6.0 true @@ -262,10 +262,10 @@ DETACH t; #------------------------------------------------------------------------------ statement ok -from copy_dir('data/generated/test_file_skipping/timestamp', '__TEST_DIR__/write_stats_all_types/timestamp'); +from copy_dir('data/generated/test_file_skipping/timestamp', '{TEMP_DIR}/write_stats_all_types/timestamp'); statement ok -ATTACH '__TEST_DIR__/write_stats_all_types/timestamp/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats_all_types/timestamp/delta_lake' AS t (TYPE delta); statement ok INSERT INTO t VALUES ('2024-01-06 00:00:00'::TIMESTAMP, '2024-01-06 00:00:00'::TIMESTAMP); @@ -287,7 +287,7 @@ SELECT json_extract_string(add.stats, '$.minValues.value1'), json_extract_string(add.stats, '$.maxValues.value1'), json_extract_string(add.stats, '$.tightBounds') -FROM read_json('__TEST_DIR__/write_stats_all_types/timestamp/delta_lake/_delta_log/00000000000000000001.json') +FROM read_json('{TEMP_DIR}/write_stats_all_types/timestamp/delta_lake/_delta_log/00000000000000000001.json') WHERE add IS NOT NULL ---- 1 0 2024-01-06T00:00:00 2024-01-06T00:00:00 true diff --git a/test/sql/generated/writing/idempotent_writes.test b/test/sql/generated/writing/idempotent_writes.test index f88d32a2..8c6ee710 100644 --- a/test/sql/generated/writing/idempotent_writes.test +++ b/test/sql/generated/writing/idempotent_writes.test @@ -10,11 +10,11 @@ require-env GENERATED_DATA_AVAILABLE # Copy data statement ok -from copy_dir('data/generated/simple_table', '__TEST_DIR__/idempotent_writes/simple_table'); +from copy_dir('data/generated/simple_table', '{TEMP_DIR}/idempotent_writes/simple_table'); # Attach some delta table statement ok -ATTACH '__TEST_DIR__/idempotent_writes/simple_table/delta_lake' AS delta_table (TYPE delta); +ATTACH '{TEMP_DIR}/idempotent_writes/simple_table/delta_lake' AS delta_table (TYPE delta); statement ok from delta_table; diff --git a/test/sql/golden_tests/generated.test b/test/sql/golden_tests/generated.test index 5de7fc0d..4ed75f40 100644 --- a/test/sql/golden_tests/generated.test +++ b/test/sql/golden_tests/generated.test @@ -13,519 +13,519 @@ require-env GOLDEN_TABLES_PATH ######## 124-decimal-decode-bug ######## query T rowsort res-124-decimal-decode-bug -from delta_scan('${GOLDEN_TABLES_PATH}/124-decimal-decode-bug/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/124-decimal-decode-bug/delta') query T rowsort 124-decimal-decode-bug -from parquet_scan('${GOLDEN_TABLES_PATH}/124-decimal-decode-bug/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/124-decimal-decode-bug/expected/**/*.parquet') ######## 125-iterator-bug ######## query T rowsort res-125-iterator-bug -from delta_scan('${GOLDEN_TABLES_PATH}/125-iterator-bug/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/125-iterator-bug/delta') query T rowsort 125-iterator-bug -from parquet_scan('${GOLDEN_TABLES_PATH}/125-iterator-bug/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/125-iterator-bug/expected/**/*.parquet') ######## basic-with-inserts-deletes-checkpoint ######## query T rowsort res-basic-with-inserts-deletes-checkpoint -from delta_scan('${GOLDEN_TABLES_PATH}/basic-with-inserts-deletes-checkpoint/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/basic-with-inserts-deletes-checkpoint/delta') query T rowsort basic-with-inserts-deletes-checkpoint -from parquet_scan('${GOLDEN_TABLES_PATH}/basic-with-inserts-deletes-checkpoint/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/basic-with-inserts-deletes-checkpoint/expected/**/*.parquet') ######## basic-with-inserts-merge ######## query T rowsort res-basic-with-inserts-merge -from delta_scan('${GOLDEN_TABLES_PATH}/basic-with-inserts-merge/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/basic-with-inserts-merge/delta') query T rowsort basic-with-inserts-merge -from parquet_scan('${GOLDEN_TABLES_PATH}/basic-with-inserts-merge/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/basic-with-inserts-merge/expected/**/*.parquet') ######## basic-with-inserts-overwrite-restore ######## query T rowsort res-basic-with-inserts-overwrite-restore -from delta_scan('${GOLDEN_TABLES_PATH}/basic-with-inserts-overwrite-restore/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/basic-with-inserts-overwrite-restore/delta') query T rowsort basic-with-inserts-overwrite-restore -from parquet_scan('${GOLDEN_TABLES_PATH}/basic-with-inserts-overwrite-restore/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/basic-with-inserts-overwrite-restore/expected/**/*.parquet') ######## basic-with-inserts-updates ######## query T rowsort res-basic-with-inserts-updates -from delta_scan('${GOLDEN_TABLES_PATH}/basic-with-inserts-updates/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/basic-with-inserts-updates/delta') query T rowsort basic-with-inserts-updates -from parquet_scan('${GOLDEN_TABLES_PATH}/basic-with-inserts-updates/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/basic-with-inserts-updates/expected/**/*.parquet') ######## basic-with-vacuum-protocol-check-feature ######## query T rowsort res-basic-with-vacuum-protocol-check-feature -from delta_scan('${GOLDEN_TABLES_PATH}/basic-with-vacuum-protocol-check-feature/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/basic-with-vacuum-protocol-check-feature/delta') query T rowsort basic-with-vacuum-protocol-check-feature -from parquet_scan('${GOLDEN_TABLES_PATH}/basic-with-vacuum-protocol-check-feature/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/basic-with-vacuum-protocol-check-feature/expected/**/*.parquet') ######## corrupted-last-checkpoint-kernel ######## query T rowsort res-corrupted-last-checkpoint-kernel -from delta_scan('${GOLDEN_TABLES_PATH}/corrupted-last-checkpoint-kernel/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/corrupted-last-checkpoint-kernel/delta') query T rowsort corrupted-last-checkpoint-kernel -from parquet_scan('${GOLDEN_TABLES_PATH}/corrupted-last-checkpoint-kernel/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/corrupted-last-checkpoint-kernel/expected/**/*.parquet') ######## data-reader-array-complex-objects ######## query T rowsort res-data-reader-array-complex-objects -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-array-complex-objects/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-array-complex-objects/delta') query T rowsort data-reader-array-complex-objects -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-array-complex-objects/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-array-complex-objects/expected/**/*.parquet') ######## data-reader-array-primitives ######## query T rowsort res-data-reader-array-primitives -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-array-primitives/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-array-primitives/delta') query T rowsort data-reader-array-primitives -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-array-primitives/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-array-primitives/expected/**/*.parquet') ######## data-reader-date-types-America ######## query T rowsort res-data-reader-date-types-America -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-America/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-America/delta') query T rowsort data-reader-date-types-America -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-America/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-America/expected/**/*.parquet') ######## data-reader-date-types-Asia ######## query T rowsort res-data-reader-date-types-Asia -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-Asia/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-Asia/delta') query T rowsort data-reader-date-types-Asia -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-Asia/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-Asia/expected/**/*.parquet') ######## data-reader-date-types-Etc ######## query T rowsort res-data-reader-date-types-Etc -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-Etc/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-Etc/delta') query T rowsort data-reader-date-types-Etc -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-Etc/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-Etc/expected/**/*.parquet') ######## data-reader-date-types-Iceland ######## query T rowsort res-data-reader-date-types-Iceland -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-Iceland/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-Iceland/delta') query T rowsort data-reader-date-types-Iceland -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-Iceland/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-Iceland/expected/**/*.parquet') ######## data-reader-date-types-Jst ######## query T rowsort res-data-reader-date-types-Jst -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-Jst/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-Jst/delta') query T rowsort data-reader-date-types-Jst -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-Jst/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-Jst/expected/**/*.parquet') ######## data-reader-date-types-Pst ######## query T rowsort res-data-reader-date-types-Pst -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-Pst/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-Pst/delta') query T rowsort data-reader-date-types-Pst -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-Pst/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-Pst/expected/**/*.parquet') ######## data-reader-date-types-utc ######## query T rowsort res-data-reader-date-types-utc -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-utc/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-utc/delta') query T rowsort data-reader-date-types-utc -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-date-types-utc/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-date-types-utc/expected/**/*.parquet') ######## data-reader-escaped-chars ######## query T rowsort res-data-reader-escaped-chars -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-escaped-chars/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-escaped-chars/delta') query T rowsort data-reader-escaped-chars -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-escaped-chars/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-escaped-chars/expected/**/*.parquet') ######## data-reader-map ######## query T rowsort res-data-reader-map -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-map/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-map/delta') query T rowsort data-reader-map -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-map/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-map/expected/**/*.parquet') ######## data-reader-nested-struct ######## query T rowsort res-data-reader-nested-struct -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-nested-struct/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-nested-struct/delta') query T rowsort data-reader-nested-struct -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-nested-struct/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-nested-struct/expected/**/*.parquet') ######## data-reader-nullable-field-invalid-schema-key ######## query T rowsort res-data-reader-nullable-field-invalid-schema-key -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-nullable-field-invalid-schema-key/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-nullable-field-invalid-schema-key/delta') query T rowsort data-reader-nullable-field-invalid-schema-key -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-nullable-field-invalid-schema-key/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-nullable-field-invalid-schema-key/expected/**/*.parquet') ######## data-reader-primitives ######## query T rowsort res-data-reader-primitives -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-primitives/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-primitives/delta') query T rowsort data-reader-primitives -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-primitives/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-primitives/expected/**/*.parquet') ######## data-reader-timestamp_ntz ######## query T rowsort res-data-reader-timestamp_ntz -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz/delta') query T rowsort data-reader-timestamp_ntz -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz/expected/**/*.parquet') ######## data-reader-timestamp_ntz-id-mode ######## query T rowsort res-data-reader-timestamp_ntz-id-mode -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz-id-mode/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz-id-mode/delta') query T rowsort data-reader-timestamp_ntz-id-mode -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz-id-mode/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz-id-mode/expected/**/*.parquet') ######## data-reader-timestamp_ntz-name-mode ######## query T rowsort res-data-reader-timestamp_ntz-name-mode -from delta_scan('${GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz-name-mode/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz-name-mode/delta') query T rowsort data-reader-timestamp_ntz-name-mode -from parquet_scan('${GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz-name-mode/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-reader-timestamp_ntz-name-mode/expected/**/*.parquet') ######## data-skipping-basic-stats-all-types ######## query T rowsort res-data-skipping-basic-stats-all-types -from delta_scan('${GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types/delta') query T rowsort data-skipping-basic-stats-all-types -from parquet_scan('${GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types/expected/**/*.parquet') ######## data-skipping-basic-stats-all-types-checkpoint ######## query T rowsort res-data-skipping-basic-stats-all-types-checkpoint -from delta_scan('${GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types-checkpoint/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types-checkpoint/delta') query T rowsort data-skipping-basic-stats-all-types-checkpoint -from parquet_scan('${GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types-checkpoint/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types-checkpoint/expected/**/*.parquet') ######## data-skipping-basic-stats-all-types-columnmapping-name ######## query T rowsort res-data-skipping-basic-stats-all-types-columnmapping-name -from delta_scan('${GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types-columnmapping-name/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types-columnmapping-name/delta') query T rowsort data-skipping-basic-stats-all-types-columnmapping-name -from parquet_scan('${GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types-columnmapping-name/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-skipping-basic-stats-all-types-columnmapping-name/expected/**/*.parquet') ######## data-skipping-change-stats-collected-across-versions ######## query T rowsort res-data-skipping-change-stats-collected-across-versions -from delta_scan('${GOLDEN_TABLES_PATH}/data-skipping-change-stats-collected-across-versions/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-skipping-change-stats-collected-across-versions/delta') query T rowsort data-skipping-change-stats-collected-across-versions -from parquet_scan('${GOLDEN_TABLES_PATH}/data-skipping-change-stats-collected-across-versions/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-skipping-change-stats-collected-across-versions/expected/**/*.parquet') ######## data-skipping-partition-and-data-column ######## query T rowsort res-data-skipping-partition-and-data-column -from delta_scan('${GOLDEN_TABLES_PATH}/data-skipping-partition-and-data-column/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/data-skipping-partition-and-data-column/delta') query T rowsort data-skipping-partition-and-data-column -from parquet_scan('${GOLDEN_TABLES_PATH}/data-skipping-partition-and-data-column/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/data-skipping-partition-and-data-column/expected/**/*.parquet') ######## decimal-various-scale-precision ######## query T rowsort res-decimal-various-scale-precision -from delta_scan('${GOLDEN_TABLES_PATH}/decimal-various-scale-precision/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/decimal-various-scale-precision/delta') query T rowsort decimal-various-scale-precision -from parquet_scan('${GOLDEN_TABLES_PATH}/decimal-various-scale-precision/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/decimal-various-scale-precision/expected/**/*.parquet') ######## deltalog-getChanges ######## query T rowsort res-deltalog-getChanges -from delta_scan('${GOLDEN_TABLES_PATH}/deltalog-getChanges/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/deltalog-getChanges/delta') query T rowsort deltalog-getChanges -from parquet_scan('${GOLDEN_TABLES_PATH}/deltalog-getChanges/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/deltalog-getChanges/expected/**/*.parquet') ######## dv-partitioned-with-checkpoint ######## query T rowsort res-dv-partitioned-with-checkpoint -from delta_scan('${GOLDEN_TABLES_PATH}/dv-partitioned-with-checkpoint/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/dv-partitioned-with-checkpoint/delta') query T rowsort dv-partitioned-with-checkpoint -from parquet_scan('${GOLDEN_TABLES_PATH}/dv-partitioned-with-checkpoint/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/dv-partitioned-with-checkpoint/expected/**/*.parquet') ######## dv-with-columnmapping ######## query T rowsort res-dv-with-columnmapping -from delta_scan('${GOLDEN_TABLES_PATH}/dv-with-columnmapping/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/dv-with-columnmapping/delta') query T rowsort dv-with-columnmapping -from parquet_scan('${GOLDEN_TABLES_PATH}/dv-with-columnmapping/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/dv-with-columnmapping/expected/**/*.parquet') ######## kernel-timestamp-int96 ######## query T rowsort res-kernel-timestamp-int96 -from delta_scan('${GOLDEN_TABLES_PATH}/kernel-timestamp-int96/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/kernel-timestamp-int96/delta') query T rowsort kernel-timestamp-int96 -from parquet_scan('${GOLDEN_TABLES_PATH}/kernel-timestamp-int96/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/kernel-timestamp-int96/expected/**/*.parquet') ######## kernel-timestamp-pst ######## query T rowsort res-kernel-timestamp-pst -from delta_scan('${GOLDEN_TABLES_PATH}/kernel-timestamp-pst/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/kernel-timestamp-pst/delta') query T rowsort kernel-timestamp-pst -from parquet_scan('${GOLDEN_TABLES_PATH}/kernel-timestamp-pst/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/kernel-timestamp-pst/expected/**/*.parquet') ######## kernel-timestamp-timestamp_micros ######## query T rowsort res-kernel-timestamp-timestamp_micros -from delta_scan('${GOLDEN_TABLES_PATH}/kernel-timestamp-timestamp_micros/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/kernel-timestamp-timestamp_micros/delta') query T rowsort kernel-timestamp-timestamp_micros -from parquet_scan('${GOLDEN_TABLES_PATH}/kernel-timestamp-timestamp_micros/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/kernel-timestamp-timestamp_micros/expected/**/*.parquet') ######## kernel-timestamp-timestamp_millis ######## query T rowsort res-kernel-timestamp-timestamp_millis -from delta_scan('${GOLDEN_TABLES_PATH}/kernel-timestamp-timestamp_millis/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/kernel-timestamp-timestamp_millis/delta') query T rowsort kernel-timestamp-timestamp_millis -from parquet_scan('${GOLDEN_TABLES_PATH}/kernel-timestamp-timestamp_millis/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/kernel-timestamp-timestamp_millis/expected/**/*.parquet') ######## log-replay-dv-key-cases ######## query T rowsort res-log-replay-dv-key-cases -from delta_scan('${GOLDEN_TABLES_PATH}/log-replay-dv-key-cases/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/log-replay-dv-key-cases/delta') query T rowsort log-replay-dv-key-cases -from parquet_scan('${GOLDEN_TABLES_PATH}/log-replay-dv-key-cases/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/log-replay-dv-key-cases/expected/**/*.parquet') ######## log-replay-latest-metadata-protocol ######## query T rowsort res-log-replay-latest-metadata-protocol -from delta_scan('${GOLDEN_TABLES_PATH}/log-replay-latest-metadata-protocol/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/log-replay-latest-metadata-protocol/delta') query T rowsort log-replay-latest-metadata-protocol -from parquet_scan('${GOLDEN_TABLES_PATH}/log-replay-latest-metadata-protocol/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/log-replay-latest-metadata-protocol/expected/**/*.parquet') ######## log-replay-special-characters ######## query T rowsort res-log-replay-special-characters -from delta_scan('${GOLDEN_TABLES_PATH}/log-replay-special-characters/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/log-replay-special-characters/delta') query T rowsort log-replay-special-characters -from parquet_scan('${GOLDEN_TABLES_PATH}/log-replay-special-characters/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/log-replay-special-characters/expected/**/*.parquet') ######## log-replay-special-characters-a ######## query T rowsort res-log-replay-special-characters-a -from delta_scan('${GOLDEN_TABLES_PATH}/log-replay-special-characters-a/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/log-replay-special-characters-a/delta') query T rowsort log-replay-special-characters-a -from parquet_scan('${GOLDEN_TABLES_PATH}/log-replay-special-characters-a/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/log-replay-special-characters-a/expected/**/*.parquet') ######## multi-part-checkpoint ######## query T rowsort res-multi-part-checkpoint -from delta_scan('${GOLDEN_TABLES_PATH}/multi-part-checkpoint/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/multi-part-checkpoint/delta') query T rowsort multi-part-checkpoint -from parquet_scan('${GOLDEN_TABLES_PATH}/multi-part-checkpoint/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/multi-part-checkpoint/expected/**/*.parquet') ######## only-checkpoint-files ######## query T rowsort res-only-checkpoint-files -from delta_scan('${GOLDEN_TABLES_PATH}/only-checkpoint-files/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/only-checkpoint-files/delta') query T rowsort only-checkpoint-files -from parquet_scan('${GOLDEN_TABLES_PATH}/only-checkpoint-files/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/only-checkpoint-files/expected/**/*.parquet') ######## snapshot-data0 ######## query T rowsort res-snapshot-data0 -from delta_scan('${GOLDEN_TABLES_PATH}/snapshot-data0/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/snapshot-data0/delta') query T rowsort snapshot-data0 -from parquet_scan('${GOLDEN_TABLES_PATH}/snapshot-data0/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/snapshot-data0/expected/**/*.parquet') ######## snapshot-data1 ######## query T rowsort res-snapshot-data1 -from delta_scan('${GOLDEN_TABLES_PATH}/snapshot-data1/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/snapshot-data1/delta') query T rowsort snapshot-data1 -from parquet_scan('${GOLDEN_TABLES_PATH}/snapshot-data1/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/snapshot-data1/expected/**/*.parquet') ######## snapshot-data2 ######## query T rowsort res-snapshot-data2 -from delta_scan('${GOLDEN_TABLES_PATH}/snapshot-data2/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/snapshot-data2/delta') query T rowsort snapshot-data2 -from parquet_scan('${GOLDEN_TABLES_PATH}/snapshot-data2/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/snapshot-data2/expected/**/*.parquet') ######## snapshot-data2-deleted ######## query T rowsort res-snapshot-data2-deleted -from delta_scan('${GOLDEN_TABLES_PATH}/snapshot-data2-deleted/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/snapshot-data2-deleted/delta') query T rowsort snapshot-data2-deleted -from parquet_scan('${GOLDEN_TABLES_PATH}/snapshot-data2-deleted/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/snapshot-data2-deleted/expected/**/*.parquet') ######## snapshot-data3 ######## query T rowsort res-snapshot-data3 -from delta_scan('${GOLDEN_TABLES_PATH}/snapshot-data3/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/snapshot-data3/delta') query T rowsort snapshot-data3 -from parquet_scan('${GOLDEN_TABLES_PATH}/snapshot-data3/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/snapshot-data3/expected/**/*.parquet') ######## snapshot-repartitioned ######## query T rowsort res-snapshot-repartitioned -from delta_scan('${GOLDEN_TABLES_PATH}/snapshot-repartitioned/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/snapshot-repartitioned/delta') query T rowsort snapshot-repartitioned -from parquet_scan('${GOLDEN_TABLES_PATH}/snapshot-repartitioned/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/snapshot-repartitioned/expected/**/*.parquet') ######## snapshot-vacuumed ######## query T rowsort res-snapshot-vacuumed -from delta_scan('${GOLDEN_TABLES_PATH}/snapshot-vacuumed/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/snapshot-vacuumed/delta') query T rowsort snapshot-vacuumed -from parquet_scan('${GOLDEN_TABLES_PATH}/snapshot-vacuumed/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/snapshot-vacuumed/expected/**/*.parquet') ######## time-travel-partition-changes-a ######## query T rowsort res-time-travel-partition-changes-a -from delta_scan('${GOLDEN_TABLES_PATH}/time-travel-partition-changes-a/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/time-travel-partition-changes-a/delta') query T rowsort time-travel-partition-changes-a -from parquet_scan('${GOLDEN_TABLES_PATH}/time-travel-partition-changes-a/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/time-travel-partition-changes-a/expected/**/*.parquet') ######## time-travel-partition-changes-b ######## query T rowsort res-time-travel-partition-changes-b -from delta_scan('${GOLDEN_TABLES_PATH}/time-travel-partition-changes-b/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/time-travel-partition-changes-b/delta') query T rowsort time-travel-partition-changes-b -from parquet_scan('${GOLDEN_TABLES_PATH}/time-travel-partition-changes-b/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/time-travel-partition-changes-b/expected/**/*.parquet') ######## time-travel-schema-changes-a ######## query T rowsort res-time-travel-schema-changes-a -from delta_scan('${GOLDEN_TABLES_PATH}/time-travel-schema-changes-a/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/time-travel-schema-changes-a/delta') query T rowsort time-travel-schema-changes-a -from parquet_scan('${GOLDEN_TABLES_PATH}/time-travel-schema-changes-a/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/time-travel-schema-changes-a/expected/**/*.parquet') ######## time-travel-schema-changes-b ######## query T rowsort res-time-travel-schema-changes-b -from delta_scan('${GOLDEN_TABLES_PATH}/time-travel-schema-changes-b/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/time-travel-schema-changes-b/delta') query T rowsort time-travel-schema-changes-b -from parquet_scan('${GOLDEN_TABLES_PATH}/time-travel-schema-changes-b/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/time-travel-schema-changes-b/expected/**/*.parquet') ######## time-travel-start ######## query T rowsort res-time-travel-start -from delta_scan('${GOLDEN_TABLES_PATH}/time-travel-start/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/time-travel-start/delta') query T rowsort time-travel-start -from parquet_scan('${GOLDEN_TABLES_PATH}/time-travel-start/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/time-travel-start/expected/**/*.parquet') ######## time-travel-start-start20 ######## query T rowsort res-time-travel-start-start20 -from delta_scan('${GOLDEN_TABLES_PATH}/time-travel-start-start20/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/time-travel-start-start20/delta') query T rowsort time-travel-start-start20 -from parquet_scan('${GOLDEN_TABLES_PATH}/time-travel-start-start20/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/time-travel-start-start20/expected/**/*.parquet') ######## time-travel-start-start20-start40 ######## query T rowsort res-time-travel-start-start20-start40 -from delta_scan('${GOLDEN_TABLES_PATH}/time-travel-start-start20-start40/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/time-travel-start-start20-start40/delta') query T rowsort time-travel-start-start20-start40 -from parquet_scan('${GOLDEN_TABLES_PATH}/time-travel-start-start20-start40/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/time-travel-start-start20-start40/expected/**/*.parquet') ######## v2-checkpoint-json ######## query T rowsort res-v2-checkpoint-json -from delta_scan('${GOLDEN_TABLES_PATH}/v2-checkpoint-json/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/v2-checkpoint-json/delta') query T rowsort v2-checkpoint-json -from parquet_scan('${GOLDEN_TABLES_PATH}/v2-checkpoint-json/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/v2-checkpoint-json/expected/**/*.parquet') ######## v2-checkpoint-parquet ######## query T rowsort res-v2-checkpoint-parquet -from delta_scan('${GOLDEN_TABLES_PATH}/v2-checkpoint-parquet/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/v2-checkpoint-parquet/delta') query T rowsort v2-checkpoint-parquet -from parquet_scan('${GOLDEN_TABLES_PATH}/v2-checkpoint-parquet/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/v2-checkpoint-parquet/expected/**/*.parquet') ######## basic-decimal-table ######## query T rowsort res-basic-decimal-table -from delta_scan('${GOLDEN_TABLES_PATH}/basic-decimal-table/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/basic-decimal-table/delta') query T rowsort basic-decimal-table -from parquet_scan('${GOLDEN_TABLES_PATH}/basic-decimal-table/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/basic-decimal-table/expected/**/*.parquet') ######## basic-decimal-table-legacy ######## query T rowsort res-basic-decimal-table-legacy -from delta_scan('${GOLDEN_TABLES_PATH}/basic-decimal-table-legacy/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/basic-decimal-table-legacy/delta') query T rowsort basic-decimal-table-legacy -from parquet_scan('${GOLDEN_TABLES_PATH}/basic-decimal-table-legacy/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/basic-decimal-table-legacy/expected/**/*.parquet') ######## table-with-columnmapping-mode-name ######## query T rowsort res-table-with-columnmapping-mode-name -from delta_scan('${GOLDEN_TABLES_PATH}/table-with-columnmapping-mode-name/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/table-with-columnmapping-mode-name/delta') query T rowsort table-with-columnmapping-mode-name -from parquet_scan('${GOLDEN_TABLES_PATH}/table-with-columnmapping-mode-name/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/table-with-columnmapping-mode-name/expected/**/*.parquet') ######## table-with-columnmapping-mode-id ######## query T rowsort res-table-with-columnmapping-mode-id -from delta_scan('${GOLDEN_TABLES_PATH}/table-with-columnmapping-mode-id/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/table-with-columnmapping-mode-id/delta') query T rowsort table-with-columnmapping-mode-id -from parquet_scan('${GOLDEN_TABLES_PATH}/table-with-columnmapping-mode-id/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/table-with-columnmapping-mode-id/expected/**/*.parquet') ######## deltalog-invalid-protocol-version ######## statement error -from delta_scan('${GOLDEN_TABLES_PATH}/deltalog-invalid-protocol-version/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/deltalog-invalid-protocol-version/delta') ---- IO Error: DeltaKernel @@ -533,7 +533,7 @@ IO Error: DeltaKernel ######## deltalog-state-reconstruction-from-checkpoint-missing-metadata ######## statement error -from delta_scan('${GOLDEN_TABLES_PATH}/deltalog-state-reconstruction-from-checkpoint-missing-metadata/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/deltalog-state-reconstruction-from-checkpoint-missing-metadata/delta') ---- IO Error: DeltaKernel @@ -541,7 +541,7 @@ IO Error: DeltaKernel ######## deltalog-state-reconstruction-from-checkpoint-missing-protocol ######## statement error -from delta_scan('${GOLDEN_TABLES_PATH}/deltalog-state-reconstruction-from-checkpoint-missing-protocol/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/deltalog-state-reconstruction-from-checkpoint-missing-protocol/delta') ---- IO Error: DeltaKernel @@ -549,7 +549,7 @@ IO Error: DeltaKernel ######## deltalog-state-reconstruction-without-metadata ######## statement error -from delta_scan('${GOLDEN_TABLES_PATH}/deltalog-state-reconstruction-without-metadata/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/deltalog-state-reconstruction-without-metadata/delta') ---- IO Error: DeltaKernel @@ -557,7 +557,7 @@ IO Error: DeltaKernel ######## deltalog-state-reconstruction-without-protocol ######## statement error -from delta_scan('${GOLDEN_TABLES_PATH}/deltalog-state-reconstruction-without-protocol/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/deltalog-state-reconstruction-without-protocol/delta') ---- IO Error: DeltaKernel @@ -565,7 +565,7 @@ IO Error: DeltaKernel ######## no-delta-log-folder ######## statement error -from delta_scan('${GOLDEN_TABLES_PATH}/no-delta-log-folder/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/no-delta-log-folder/delta') ---- IO Error: DeltaKernel @@ -573,7 +573,7 @@ IO Error: DeltaKernel ######## versions-not-contiguous ######## statement error -from delta_scan('${GOLDEN_TABLES_PATH}/versions-not-contiguous/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/versions-not-contiguous/delta') ---- IO Error: DeltaKernel diff --git a/test/sql/golden_tests/generated.test_slow b/test/sql/golden_tests/generated.test_slow index 0a56bbbf..ca0f9649 100644 --- a/test/sql/golden_tests/generated.test_slow +++ b/test/sql/golden_tests/generated.test_slow @@ -13,39 +13,39 @@ require-env GOLDEN_TABLES_PATH ######## parquet-decimal-dictionaries ######## query T rowsort res-parquet-decimal-dictionaries -from delta_scan('${GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries/delta') query T rowsort parquet-decimal-dictionaries -from parquet_scan('${GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries/expected/**/*.parquet') ######## parquet-decimal-dictionaries ######## query T rowsort res-parquet-decimal-dictionaries -from delta_scan('${GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries/delta') query T rowsort parquet-decimal-dictionaries -from parquet_scan('${GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries/expected/**/*.parquet') ######## parquet-decimal-dictionaries-v1 ######## query T rowsort res-parquet-decimal-dictionaries-v1 -from delta_scan('${GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries-v1/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries-v1/delta') query T rowsort parquet-decimal-dictionaries-v1 -from parquet_scan('${GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries-v1/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries-v1/expected/**/*.parquet') ######## parquet-decimal-dictionaries-v2 ######## query T rowsort res-parquet-decimal-dictionaries-v2 -from delta_scan('${GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries-v2/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries-v2/delta') query T rowsort parquet-decimal-dictionaries-v2 -from parquet_scan('${GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries-v2/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/parquet-decimal-dictionaries-v2/expected/**/*.parquet') ######## parquet-decimal-type ######## query T rowsort res-parquet-decimal-type -from delta_scan('${GOLDEN_TABLES_PATH}/parquet-decimal-type/delta') +from delta_scan('{GOLDEN_TABLES_PATH}/parquet-decimal-type/delta') query T rowsort parquet-decimal-type -from parquet_scan('${GOLDEN_TABLES_PATH}/parquet-decimal-type/expected/**/*.parquet') +from parquet_scan('{GOLDEN_TABLES_PATH}/parquet-decimal-type/expected/**/*.parquet') diff --git a/test/sql/inlined/variant/basic.test b/test/sql/inlined/variant/basic.test index a142da9a..9717fc36 100644 --- a/test/sql/inlined/variant/basic.test +++ b/test/sql/inlined/variant/basic.test @@ -8,10 +8,10 @@ require delta require notwindows statement ok -from copy_dir('data/inlined/variant/basic', '__TEST_DIR__/data/inlined/variant/basic'); +from copy_dir('data/inlined/variant/basic', '{TEMP_DIR}/data/inlined/variant/basic'); statement ok -ATTACH '__TEST_DIR__/data/inlined/variant/basic' AS delta_table (TYPE delta); +ATTACH '{TEMP_DIR}/data/inlined/variant/basic' AS delta_table (TYPE delta); query IIIIII DESCRIBE delta_table; @@ -28,7 +28,7 @@ select * from delta_table; query I select stats(data) from delta_table; ---- -shredding_state: INCONSISTENT[Has Null: true, Has No Null: true] +{'has_no_null': true, 'has_null': true, 'shredding_state': INCONSISTENT} query I select variant_typeof(data) from delta_table; diff --git a/test/sql/inlined/variant/write_stats_no_variant_stats.test b/test/sql/inlined/variant/write_stats_no_variant_stats.test index d82a222b..cd810d64 100644 --- a/test/sql/inlined/variant/write_stats_no_variant_stats.test +++ b/test/sql/inlined/variant/write_stats_no_variant_stats.test @@ -11,10 +11,10 @@ require json require notwindows statement ok -from copy_dir('data/inlined/variant/basic', '__TEST_DIR__/write_stats_variant'); +from copy_dir('data/inlined/variant/basic', '{TEMP_DIR}/write_stats_variant'); statement ok -ATTACH '__TEST_DIR__/write_stats_variant' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/write_stats_variant' AS t (TYPE delta); statement ok INSERT INTO t VALUES (100, {'x': 1}), (200, NULL); @@ -24,7 +24,7 @@ query II SELECT json_extract(add.stats, '$.nullCount.data') IS NULL, json_extract(add.stats, '$.minValues.data') IS NULL -FROM read_json('__TEST_DIR__/write_stats_variant/_delta_log/00000000000000000002.json') +FROM read_json('{TEMP_DIR}/write_stats_variant/_delta_log/00000000000000000002.json') WHERE add IS NOT NULL ---- true true diff --git a/test/sql/issues/null_stats_conversion__issue_297.test b/test/sql/issues/null_stats_conversion__issue_297.test index 5a8666fa..b7796032 100644 --- a/test/sql/issues/null_stats_conversion__issue_297.test +++ b/test/sql/issues/null_stats_conversion__issue_297.test @@ -13,10 +13,10 @@ require json require notwindows # paths to JSON fail, skip windows statement ok -from copy_dir('data/inlined/simple_table/delta_lake', '__TEST_DIR__/null_stats'); +from copy_dir('data/inlined/simple_table/delta_lake', '{TEMP_DIR}/null_stats'); statement ok -ATTACH '__TEST_DIR__/null_stats' AS null_stats (TYPE delta, READ_ONLY false); +ATTACH '{TEMP_DIR}/null_stats' AS null_stats (TYPE delta, READ_ONLY false); # Insert non-null (=42), then NULL to get both real min/max and null stats in 2 transactions. statement ok @@ -38,7 +38,7 @@ SELECT json_extract_string(add.stats, '$.minValues.i'), json_extract_string(add.stats, '$.maxValues.i'), json_extract_string(add.stats, '$.nullCount.i') -FROM read_json('__TEST_DIR__/null_stats/_delta_log/00000000000000000001.json') +FROM read_json('{TEMP_DIR}/null_stats/_delta_log/00000000000000000001.json') WHERE add IS NOT NULL ---- 42 42 0 @@ -49,7 +49,7 @@ SELECT json_extract_string(add.stats, '$.minValues.i'), json_extract_string(add.stats, '$.maxValues.i'), json_extract_string(add.stats, '$.nullCount.i') -FROM read_json('__TEST_DIR__/null_stats/_delta_log/00000000000000000002.json') +FROM read_json('{TEMP_DIR}/null_stats/_delta_log/00000000000000000002.json') WHERE add IS NOT NULL ---- NULL NULL 1 diff --git a/test/sql/issues/partition_equality_filter__issue_303.test b/test/sql/issues/partition_equality_filter__issue_303.test index 167d2fdb..1a2472ae 100644 --- a/test/sql/issues/partition_equality_filter__issue_303.test +++ b/test/sql/issues/partition_equality_filter__issue_303.test @@ -18,7 +18,7 @@ require notwindows # statement ok -FROM copy_dir('data/inlined/issue_303_partitioned/delta_lake', '__TEST_DIR__/issue_303'); +FROM copy_dir('data/inlined/issue_303_partitioned/delta_lake', '{TEMP_DIR}/issue_303'); # ----------------------------------------------------------------------------- # delta_scan @@ -26,31 +26,31 @@ FROM copy_dir('data/inlined/issue_303_partitioned/delta_lake', '__TEST_DIR__/iss # Baseline: full scan must be correct (distinguishes from snapshot_incomplete). query I -SELECT count(*) FROM delta_scan('__TEST_DIR__/issue_303') +SELECT count(*) FROM delta_scan('{TEMP_DIR}/issue_303') ---- 16 # Equality — must return 16, not 0. query I -SELECT count(*) FROM delta_scan('__TEST_DIR__/issue_303') WHERE part = 42 +SELECT count(*) FROM delta_scan('{TEMP_DIR}/issue_303') WHERE part = 42 ---- 16 # Single-value IN must behave the same as =. query I -SELECT count(*) FROM delta_scan('__TEST_DIR__/issue_303') WHERE part IN (42) +SELECT count(*) FROM delta_scan('{TEMP_DIR}/issue_303') WHERE part IN (42) ---- 16 # Multi-value IN (with a non-existent value) as contrast — worked even when = didn't. query I -SELECT count(*) FROM delta_scan('__TEST_DIR__/issue_303') WHERE part IN (31, 42) +SELECT count(*) FROM delta_scan('{TEMP_DIR}/issue_303') WHERE part IN (31, 42) ---- 16 # Range predicate too query I -SELECT count(*) FROM delta_scan('__TEST_DIR__/issue_303') WHERE part > 40 +SELECT count(*) FROM delta_scan('{TEMP_DIR}/issue_303') WHERE part > 40 ---- 16 @@ -59,7 +59,7 @@ SELECT count(*) FROM delta_scan('__TEST_DIR__/issue_303') WHERE part > 40 # statement ok -ATTACH '__TEST_DIR__/issue_303' AS issue_303 (TYPE delta) +ATTACH '{TEMP_DIR}/issue_303' AS issue_303 (TYPE delta) query I SELECT count(*) FROM issue_303 diff --git a/test/sql/main/writing/checkpoint.test b/test/sql/main/writing/checkpoint.test index 27dc96de..aba4e62d 100644 --- a/test/sql/main/writing/checkpoint.test +++ b/test/sql/main/writing/checkpoint.test @@ -12,14 +12,14 @@ require notwindows # Copy test data to a writable tmp dir statement ok -CALL copy_dir('data/inlined/simple_table', '__TEST_DIR__/checkpoint/simple_table'); +CALL copy_dir('data/inlined/simple_table', '{TEMP_DIR}/checkpoint/simple_table'); statement ok -ATTACH '__TEST_DIR__/checkpoint/simple_table/delta_lake' AS checkpoint_tbl (TYPE delta); +ATTACH '{TEMP_DIR}/checkpoint/simple_table/delta_lake' AS checkpoint_tbl (TYPE delta); # Pre-checkpoint - Assert: 0 _last_checkpoint file, and 0 *.checkpoint*.parquet file query I -SELECT count(*) FROM glob('__TEST_DIR__/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); +SELECT count(*) FROM glob('{TEMP_DIR}/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); ---- 0 @@ -30,7 +30,7 @@ CHECKPOINT checkpoint_tbl # Post-checkpoint: Expect: 1 _last_checkpoint file, and 1 *.checkpoint*.parquet file # hardcode the name here since it should be safe with this canned data. query I sorted -SELECT parse_filename(file) FROM glob('__TEST_DIR__/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); +SELECT parse_filename(file) FROM glob('{TEMP_DIR}/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); ---- 00000000000000000000.checkpoint.parquet _last_checkpoint @@ -41,7 +41,7 @@ CHECKPOINT checkpoint_tbl # Expect: 1 _last_checkpoint file, and 1 *.checkpoint*.parquet file query I sorted -SELECT count(*) FROM glob('__TEST_DIR__/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); +SELECT count(*) FROM glob('{TEMP_DIR}/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); ---- 2 @@ -51,7 +51,7 @@ FORCE CHECKPOINT checkpoint_tbl # Expect: 1 _last_checkpoint file, and 1 *.checkpoint*.parquet file query I sorted -SELECT count(*) FROM glob('__TEST_DIR__/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); +SELECT count(*) FROM glob('{TEMP_DIR}/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); ---- 2 @@ -70,7 +70,7 @@ CHECKPOINT checkpoint_tbl # Expect: 1 _last_checkpoint file, and 2 *.checkpoint*.parquet files query I sorted -SELECT count(*) FROM glob('__TEST_DIR__/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); +SELECT count(*) FROM glob('{TEMP_DIR}/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); ---- 3 @@ -94,7 +94,7 @@ CHECKPOINT checkpoint_tbl # Expect: 1 _last_checkpoint file, and 3 *.checkpoint*.parquet files query I sorted -SELECT count(*) FROM glob('__TEST_DIR__/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); +SELECT count(*) FROM glob('{TEMP_DIR}/checkpoint/simple_table/delta_lake/_delta_log/*checkpoint*'); ---- 4 @@ -105,10 +105,10 @@ DETACH checkpoint_tbl # Pinned version — checkpoint should work with version=0 statement ok -CALL copy_dir('data/inlined/simple_table', '__TEST_DIR__/checkpoint/simple_table_pinned'); +CALL copy_dir('data/inlined/simple_table', '{TEMP_DIR}/checkpoint/simple_table_pinned'); statement ok -ATTACH '__TEST_DIR__/checkpoint/simple_table_pinned/delta_lake' AS checkpoint_pinned (TYPE delta, version 0); +ATTACH '{TEMP_DIR}/checkpoint/simple_table_pinned/delta_lake' AS checkpoint_pinned (TYPE delta, version 0); statement ok CHECKPOINT checkpoint_pinned @@ -116,7 +116,7 @@ CHECKPOINT checkpoint_pinned # Post-checkpoint: Expect: 1 _last_checkpoint file, and 1 *.checkpoint*.parquet file # hardcode the name here since it should be safe with this canned data. query I sorted -SELECT parse_filename(file) FROM glob('__TEST_DIR__/checkpoint/simple_table_pinned/delta_lake/_delta_log/*checkpoint*'); +SELECT parse_filename(file) FROM glob('{TEMP_DIR}/checkpoint/simple_table_pinned/delta_lake/_delta_log/*checkpoint*'); ---- 00000000000000000000.checkpoint.parquet _last_checkpoint @@ -133,10 +133,10 @@ DETACH checkpoint_pinned # Read-only — checkpoint behavior left to kernel (even if NOOP) statement ok -CALL copy_dir('data/inlined/simple_table', '__TEST_DIR__/checkpoint/simple_table_ro'); +CALL copy_dir('data/inlined/simple_table', '{TEMP_DIR}/checkpoint/simple_table_ro'); statement ok -ATTACH '__TEST_DIR__/checkpoint/simple_table_ro/delta_lake' AS checkpoint_ro (TYPE delta, READ_ONLY); +ATTACH '{TEMP_DIR}/checkpoint/simple_table_ro/delta_lake' AS checkpoint_ro (TYPE delta, READ_ONLY); # do not assert anything on RO checkpoint outcomes diff --git a/test/sql/main/writing/incremental_snapshot.test b/test/sql/main/writing/incremental_snapshot.test index 5522a48a..27a02dc7 100644 --- a/test/sql/main/writing/incremental_snapshot.test +++ b/test/sql/main/writing/incremental_snapshot.test @@ -1,5 +1,5 @@ # name: test/sql/main/writing/incremental_snapshot.test -# description: Test that incremental snapshot loading does not re-read previously loaded log files +# description: Incremental snapshot loading reads only checkpoint-bounded commit files # group: [writing] require parquet @@ -9,113 +9,135 @@ require delta require notwindows # ----------------------------------------------------------------------------- -# setup +# setup: table with checkpoints at v2 and v4, plus a trailing commit v5 +# +# Log layout produced below: +# 0000.json 0001.json (v0, v1) +# 0002.checkpoint.parquet 0002.json (v2 + checkpoint@v2) +# 0003.json (v3) +# 0004.checkpoint.parquet 0004.json (v4 + checkpoint@v4) +# 0005.json (v5, trailing) # # simple_table copy gives us a v0 with 10 rows statement ok -FROM copy_dir('data/inlined/simple_table', '__TEST_DIR__/main/writing/incremental_snapshot'); +FROM copy_dir('data/inlined/simple_table', '{TEMP_DIR}/main/writing/incremental_snapshot'); statement ok -ATTACH '__TEST_DIR__/main/writing/incremental_snapshot/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/main/writing/incremental_snapshot/delta_lake' AS t (TYPE delta); -# 2 inserts makes v1 and v2 +# v1, v2, then a checkpoint at v2 statement ok INSERT INTO t VALUES (10); statement ok INSERT INTO t VALUES (11); +statement ok +CHECKPOINT t; + +# v3, v4, then a checkpoint at v4 +statement ok +INSERT INTO t VALUES (12); + +statement ok +INSERT INTO t VALUES (13); + +statement ok +CHECKPOINT t; + +# v5, trailing commit after the last checkpoint +statement ok +INSERT INTO t VALUES (14); + statement ok DETACH t; -# Enable DeltaKernel logging to track metadata/log file reads +# Enable DeltaKernel trace logging so we can observe which commit files each read touches. statement ok -CALL enable_logging(level = 'trace'); +CALL enable_logging(level='trace', storage='memory'); statement ok SET delta_kernel_logging=true; - # ----------------------------------------------------------------------------- -# test: Variant 1: separate ATTACH calls per version +# test: step forward v2 -> v4 -> v5, asserting checkpoint-bounded commit reads +# +# The kernel emits a "Provisionally selecting ... NNNN.json" trace only for JSON commits +# in the resolved log segment; a checkpoint bounds that segment. So a read AT a checkpoint +# version references NO commit json, and a read past a checkpoint references only the +# trailing commit(s) after it. Pre-checkpoint commits must never be re-read. # -# Cold load at v1: reads log entries 0–1 statement ok -ATTACH '__TEST_DIR__/main/writing/incremental_snapshot/delta_lake' AS t1 (TYPE delta, VERSION 1); - -query I -SELECT count() FROM t1; ----- -11 +ATTACH '{TEMP_DIR}/main/writing/incremental_snapshot/delta_lake' AS t3 (TYPE delta); -# Clear logs to track the next op; we'll track kernel logs for version file read refs, -# (e.g. 00000000000000000000.json) +# --- v2: served entirely from checkpoint@v2; no commit json read --- statement ok CALL truncate_duckdb_logs(); -# Incremental load at v2 via separate ATTACH: should reuse v1 snapshot, only reading log entry 2. -statement ok -ATTACH '__TEST_DIR__/main/writing/incremental_snapshot/delta_lake' AS t2 (TYPE delta, VERSION 2); - query I -SELECT count() FROM t2; +SELECT count() FROM t3 AT (VERSION => 2); ---- 12 -# v0/v1 log files must NOT be re-read by the incremental path. +# commits 0..2 are covered by 0002.checkpoint.parquet and must NOT be read query I SELECT count() FROM duckdb_logs WHERE type = 'DeltaKernel' -AND message LIKE '%00000000000000000000.json%'; ----- -0 - -query I -SELECT count() FROM duckdb_logs -WHERE type = 'DeltaKernel' -AND message LIKE '%00000000000000000001.json%'; +AND (message LIKE '%00000000000000000000.json%' + OR message LIKE '%00000000000000000001.json%' + OR message LIKE '%00000000000000000002.json%'); ---- 0 +# --- v4: served entirely from checkpoint@v4; no commit json read --- statement ok CALL truncate_duckdb_logs(); -# ----------------------------------------------------------------------------- -# test: Variant 2: AT (VERSION => n) on a plain ATTACH -# - -statement ok -ATTACH '__TEST_DIR__/main/writing/incremental_snapshot/delta_lake' AS t3 (TYPE delta); +query I +SELECT count() FROM t3 AT (VERSION => 4); +---- +14 -# Cold read at v1 via AT clause query I -SELECT count() FROM t3 AT (VERSION => 1); +SELECT count() FROM duckdb_logs +WHERE type = 'DeltaKernel' +AND (message LIKE '%00000000000000000000.json%' + OR message LIKE '%00000000000000000001.json%' + OR message LIKE '%00000000000000000002.json%' + OR message LIKE '%00000000000000000003.json%' + OR message LIKE '%00000000000000000004.json%'); ---- -11 +0 +# --- v5: checkpoint@v4 plus the single trailing commit 0005.json --- statement ok CALL truncate_duckdb_logs(); -# Incremental read at v2 via AT clause: should reuse v1 snapshot query I -SELECT count() FROM t3 AT (VERSION => 2); +SELECT count() FROM t3 AT (VERSION => 5); ---- -12 +15 -# Same filename-based assertions as Variant 1. +# the trailing commit after checkpoint@v4 IS read. This also guards the negative assertions +# above from passing vacuously on a build whose kernel emits no trace at all. query I -SELECT count() FROM duckdb_logs +SELECT count() > 0 FROM duckdb_logs WHERE type = 'DeltaKernel' -AND message LIKE '%00000000000000000000.json%'; +AND message LIKE '%00000000000000000005.json%'; ---- -0 +true +# nothing at or before checkpoint@v4 is re-read to reach v5 query I SELECT count() FROM duckdb_logs WHERE type = 'DeltaKernel' -AND message LIKE '%00000000000000000001.json%'; +AND (message LIKE '%00000000000000000000.json%' + OR message LIKE '%00000000000000000001.json%' + OR message LIKE '%00000000000000000002.json%' + OR message LIKE '%00000000000000000003.json%' + OR message LIKE '%00000000000000000004.json%'); ---- 0 @@ -123,30 +145,29 @@ statement ok CALL truncate_duckdb_logs(); # ----------------------------------------------------------------------------- -# test: Variant 3: backward time travel (fresh builder fallback) +# test: backward time travel falls back to a fresh (non-incremental) builder # statement ok -ATTACH '__TEST_DIR__/main/writing/incremental_snapshot/delta_lake' AS t4 (TYPE delta); +ATTACH '{TEMP_DIR}/main/writing/incremental_snapshot/delta_lake' AS t4 (TYPE delta); -# Load HEAD to populate the schema-level cached_table at v2 (12 rows) +# Load HEAD (v5, 15 rows) to populate the schema-level cached_table query I SELECT count() FROM t4; ---- -12 +15 statement ok CALL truncate_duckdb_logs(); -# Request v1 (< v2): kernel rejects builder_from for backward travel, so -# InitializeSnapshot falls back to a fresh builder. Verify data is correct. +# Request v1 (< cached v5): the kernel rejects builder_from for backward travel, so +# InitializeSnapshot falls back to a fresh builder. Data must still be correct. query I SELECT count() FROM t4 AT (VERSION => 1); ---- 11 -# Check duckdb delta's own logs for 'incremental=false', to confirm -# non-incremental load +# Confirm the non-incremental fallback via delta's own boundary log query I SELECT count() > 0 FROM duckdb_logs WHERE type = 'delta.DeltaMultiFileList' diff --git a/test/sql/main/writing/non_nullable.test b/test/sql/main/writing/non_nullable.test index 09ee08d5..967aa54e 100644 --- a/test/sql/main/writing/non_nullable.test +++ b/test/sql/main/writing/non_nullable.test @@ -10,13 +10,13 @@ require notwindows # Copy test data over to tmp test dir because we'll be inserting stuff statement ok -CALL copy_dir('data/inlined/null_constraints_lists', '__TEST_DIR__/non_nullable/null_constraints_lists'); +CALL copy_dir('data/inlined/null_constraints_lists', '{TEMP_DIR}/non_nullable/null_constraints_lists'); statement ok -CALL copy_dir('data/inlined/null_constraints_structs', '__TEST_DIR__/non_nullable/null_constraints_structs'); +CALL copy_dir('data/inlined/null_constraints_structs', '{TEMP_DIR}/non_nullable/null_constraints_structs'); statement ok -ATTACH '__TEST_DIR__/non_nullable/null_constraints_structs/delta_lake' AS delta_table (TYPE delta); +ATTACH '{TEMP_DIR}/non_nullable/null_constraints_structs/delta_lake' AS delta_table (TYPE delta); query I select count(*) FROM delta_table; @@ -67,7 +67,7 @@ statement ok DETACH delta_table; statement ok -ATTACH '__TEST_DIR__/non_nullable/null_constraints_lists/delta_lake' AS delta_table (TYPE delta); +ATTACH '{TEMP_DIR}/non_nullable/null_constraints_lists/delta_lake' AS delta_table (TYPE delta); query I select count(*) FROM delta_table; diff --git a/test/sql/main/writing/transaction_multi_insert.test b/test/sql/main/writing/transaction_multi_insert.test index f05ca1f8..3a329903 100644 --- a/test/sql/main/writing/transaction_multi_insert.test +++ b/test/sql/main/writing/transaction_multi_insert.test @@ -13,10 +13,10 @@ require notwindows # statement ok -FROM copy_dir('data/inlined/simple_table', '__TEST_DIR__/main/writing/transaction_multi_insert'); +FROM copy_dir('data/inlined/simple_table', '{TEMP_DIR}/main/writing/transaction_multi_insert'); statement ok -ATTACH '__TEST_DIR__/main/writing/transaction_multi_insert/delta_lake' AS t (TYPE delta); +ATTACH '{TEMP_DIR}/main/writing/transaction_multi_insert/delta_lake' AS t (TYPE delta); query II SELECT count(*), sum(i) FROM t; @@ -50,7 +50,7 @@ SELECT count(*), sum(i) FROM t; # Each INSERT produced its own parquet file (3 new, 1 original) query I -SELECT count(*) FROM glob('__TEST_DIR__/main/writing/transaction_multi_insert/delta_lake/**/*.parquet') +SELECT count(*) FROM glob('{TEMP_DIR}/main/writing/transaction_multi_insert/delta_lake/**/*.parquet') ---- 4 @@ -78,6 +78,6 @@ SELECT count(*), sum(i) FROM t; # Both staged files cleaned up on rollback query I -SELECT count(*) FROM glob('__TEST_DIR__/main/writing/transaction_multi_insert/delta_lake/**/*.parquet') +SELECT count(*) FROM glob('{TEMP_DIR}/main/writing/transaction_multi_insert/delta_lake/**/*.parquet') ---- 4