diff --git a/src/core/expression/iceberg_transform.cpp b/src/core/expression/iceberg_transform.cpp index 718d68942..b9eb27619 100644 --- a/src/core/expression/iceberg_transform.cpp +++ b/src/core/expression/iceberg_transform.cpp @@ -1,6 +1,7 @@ #include "core/expression/iceberg_transform.hpp" #include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/date.hpp" #include "utf8proc_wrapper.hpp" #include "core/expression/iceberg_hash.hpp" @@ -114,18 +115,47 @@ string IcebergTransform::PartitionValueToString(const Value &partition_value) co return StringUtil::Format("%04d-%02d", year, month); } case IcebergTransformType::YEAR: { - return std::to_string(1970 + partition_value.GetValue()); + return StringUtil::Format("%04d", 1970 + partition_value.GetValue()); } case IcebergTransformType::HOUR: { - int64_t hours = partition_value.GetValue(); - timestamp_t ts(hours * Interval::MICROS_PER_HOUR); - return Timestamp::ToString(ts); + // yyyy-MM-dd-HH (UTC), not a full timestamp + int32_t hours = partition_value.GetValue(); + int64_t day_ordinal = IcebergFloorDiv(hours, 24); + int32_t hour_of_day = static_cast(hours - day_ordinal * 24); + int32_t year, month, day; + Date::Convert(date_t(static_cast(day_ordinal)), year, month, day); + return StringUtil::Format("%04d-%02d-%02d-%02d", year, month, day, hour_of_day); } default: return partition_value.ToString(); } } +Value IcebergTransform::PartitionStringToValue(const string &partition_string) const { + switch (type) { + case IcebergTransformType::DAY: { + return Value::INTEGER(Date::FromString(partition_string).days); + } + case IcebergTransformType::MONTH: { + auto dash = partition_string.find('-', 1); + int32_t year = std::stoi(partition_string.substr(0, dash)); + int32_t month = std::stoi(partition_string.substr(dash + 1)); + return Value::INTEGER((year - 1970) * 12 + (month - 1)); + } + case IcebergTransformType::YEAR: { + return Value::INTEGER(std::stoi(partition_string) - 1970); + } + case IcebergTransformType::HOUR: { + auto last_dash = partition_string.rfind('-'); + int32_t hour_of_day = std::stoi(partition_string.substr(last_dash + 1)); + date_t d = Date::FromString(partition_string.substr(0, last_dash)); + return Value::INTEGER(static_cast(static_cast(d.days) * 24 + hour_of_day)); + } + default: + return Value(partition_string); + } +} + void IcebergTransform::SetBucketOrTruncateValue(idx_t value) { switch (type) { case IcebergTransformType::BUCKET: diff --git a/src/execution/operator/iceberg_insert.cpp b/src/execution/operator/iceberg_insert.cpp index f814defbc..480937b6d 100644 --- a/src/execution/operator/iceberg_insert.cpp +++ b/src/execution/operator/iceberg_insert.cpp @@ -246,7 +246,8 @@ void IcebergInsertGlobalState::AddFiles(DataChunk &chunk, const string &table_na IcebergPartitionInfo info; info.field_id = partition_field.partition_field_id; if (!struct_val[1].IsNull()) { - info.value = Value(StringValue::Get(struct_val[1])); + // Restore the raw ordinal the manifest stores (temporal routing values are human-readable). + info.value = partition_field.transform.PartitionStringToValue(StringValue::Get(struct_val[1])); } else { info.value = Value(); } @@ -585,6 +586,26 @@ static unique_ptr GetDateDiffFunction(ClientContext &context, const return function; } +// Format the temporal ordinal as the human-readable routing value (hence the hive directory); +// partition assignment is unchanged and AddFiles restores the ordinal for the manifest. +static unique_ptr GetTemporalHumanExpression(ClientContext &context, const IcebergCopyInput ©_input, + const string &date_part, uint64_t source_id) { + auto ordinal_expr = GetDateDiffFunction(context, copy_input, date_part, source_id); + + vector> children; + children.push_back(make_uniq(Value(date_part))); + children.push_back(std::move(ordinal_expr)); + + ErrorData error; + FunctionBinder binder(context); + auto function = + binder.BindScalarFunction(DEFAULT_SCHEMA, "iceberg_partition_to_human", std::move(children), error, false); + if (!function) { + error.Throw(); + } + return function; +} + //! Get an iceberg_bucket(N, col) expression for bucket partition transforms static unique_ptr GetBucketExpression(ClientContext &context, const IcebergCopyInput ©_input, const IcebergPartitionSpecField &field) { @@ -635,13 +656,13 @@ static unique_ptr GetPartitionExpression(ClientContext &context, con return CreateColumnReference(copy_input, col_type, col_idx); } case IcebergTransformType::YEAR: - return GetDateDiffFunction(context, copy_input, "year", field.source_id); + return GetTemporalHumanExpression(context, copy_input, "year", field.source_id); case IcebergTransformType::MONTH: - return GetDateDiffFunction(context, copy_input, "month", field.source_id); + return GetTemporalHumanExpression(context, copy_input, "month", field.source_id); case IcebergTransformType::DAY: - return GetDateDiffFunction(context, copy_input, "day", field.source_id); + return GetTemporalHumanExpression(context, copy_input, "day", field.source_id); case IcebergTransformType::HOUR: - return GetDateDiffFunction(context, copy_input, "hour", field.source_id); + return GetTemporalHumanExpression(context, copy_input, "hour", field.source_id); case IcebergTransformType::BUCKET: return GetBucketExpression(context, copy_input, field); case IcebergTransformType::TRUNCATE: diff --git a/src/function/iceberg_functions.cpp b/src/function/iceberg_functions.cpp index 248dd6267..6a4e5d311 100644 --- a/src/function/iceberg_functions.cpp +++ b/src/function/iceberg_functions.cpp @@ -33,6 +33,7 @@ vector IcebergFunctions::GetScalarFunctions() { functions.push_back(GetIcebergBucketFunction()); functions.push_back(GetIcebergTruncateFunction()); + functions.push_back(GetIcebergPartitionToHumanFunction()); functions.push_back(GetVerifyEqualityDeletesFunction()); return functions; diff --git a/src/function/iceberg_scalar_functions.cpp b/src/function/iceberg_scalar_functions.cpp index c91373332..865c265e4 100644 --- a/src/function/iceberg_scalar_functions.cpp +++ b/src/function/iceberg_scalar_functions.cpp @@ -10,6 +10,7 @@ #include "function/iceberg_functions.hpp" #include "core/expression/iceberg_hash.hpp" +#include "core/expression/iceberg_transform.hpp" #include "duckdb.hpp" #include "duckdb/common/vector_operations/binary_executor.hpp" @@ -331,4 +332,28 @@ ScalarFunctionSet IcebergFunctions::GetIcebergTruncateFunction() { return set; } +//===--------------------------------------------------------------------===// +// iceberg_partition_to_human(transform, ordinal) -> VARCHAR +// Format a temporal partition ordinal as the human-readable path segment +// (e.g. 'month', 634 -> "2022-11"), via Transform.toHumanString. +//===--------------------------------------------------------------------===// + +static void IcebergPartitionToHuman(DataChunk &input, ExpressionState &state, Vector &result) { + BinaryExecutor::Execute( + input.data[0], input.data[1], result, input.size(), + [&result](string_t transform_name, int64_t ordinal) -> string_t { + IcebergTransform transform(transform_name.GetString()); + auto human = transform.PartitionValueToString(Value::INTEGER(static_cast(ordinal))); + return StringVector::AddString(result, human); + }); +} + +ScalarFunctionSet IcebergFunctions::GetIcebergPartitionToHumanFunction() { + ScalarFunctionSet set("iceberg_partition_to_human"); + // (transform_name, ordinal) -> VARCHAR + set.AddFunction( + ScalarFunction({LogicalType::VARCHAR, LogicalType::BIGINT}, LogicalType::VARCHAR, IcebergPartitionToHuman)); + return set; +} + } // namespace duckdb diff --git a/src/include/core/expression/iceberg_transform.hpp b/src/include/core/expression/iceberg_transform.hpp index 31ee323b7..5fab9752d 100644 --- a/src/include/core/expression/iceberg_transform.hpp +++ b/src/include/core/expression/iceberg_transform.hpp @@ -44,6 +44,7 @@ struct IcebergTransform { LogicalType GetSerializedType(const LogicalType &input) const; LogicalType GetBoundsType(const LogicalType &input) const; string PartitionValueToString(const Value &partition_value) const; + Value PartitionStringToValue(const string &partition_string) const; void SetBucketOrTruncateValue(idx_t value); private: diff --git a/src/include/function/iceberg_functions.hpp b/src/include/function/iceberg_functions.hpp index 0d03f56e5..60731e713 100644 --- a/src/include/function/iceberg_functions.hpp +++ b/src/include/function/iceberg_functions.hpp @@ -24,6 +24,7 @@ class IcebergFunctions { private: static ScalarFunctionSet GetIcebergBucketFunction(); static ScalarFunctionSet GetIcebergTruncateFunction(); + static ScalarFunctionSet GetIcebergPartitionToHumanFunction(); static ScalarFunctionSet GetVerifyEqualityDeletesFunction(); static TableFunctionSet GetIcebergSnapshotsFunction(); diff --git a/test/sql/local/catalog_test_config_setup/catalog_agnostic/insert/partitions/temporal/test_human_readable_partition_paths.test b/test/sql/local/catalog_test_config_setup/catalog_agnostic/insert/partitions/temporal/test_human_readable_partition_paths.test new file mode 100644 index 000000000..8e40d5da2 --- /dev/null +++ b/test/sql/local/catalog_test_config_setup/catalog_agnostic/insert/partitions/temporal/test_human_readable_partition_paths.test @@ -0,0 +1,125 @@ +# name: test/sql/local/catalog_test_config_setup/catalog_agnostic/insert/partitions/temporal/test_human_readable_partition_paths.test +# description: Temporal partition writes use the Iceberg human-readable path segment, not the raw ordinal +# group: [temporal] + +# Partition directories use Transform.toHumanString: month -> "2022-11", year -> "2022", +# day -> "2022-11-15", hour -> "2022-11-15-13" (not the raw integer ordinal). The manifest +# partition value is unchanged (the INSERT would fail the BIGINT cast if the ordinal were +# lost), so existing / other-engine files in the same logical partition are unaffected. + +require-env CATALOG_TEST_CONFIG_SETUP + +require avro + +require parquet + +require iceberg + +require httpfs + +# Do not ignore 'HTTP' error messages! +set ignore_error_messages + +statement ok +create schema if not exists my_datalake.default; + +# ---- month(ts) -> "2022-11" (ordinal 634) ---- + +statement ok +drop table if exists my_datalake.default.hrpp_month; + +statement ok +CREATE TABLE my_datalake.default.hrpp_month (id int, ts TIMESTAMP) +PARTITIONED BY (month(ts)); + +statement ok +INSERT INTO my_datalake.default.hrpp_month VALUES (1, TIMESTAMP '2022-11-15 10:30:00'); + +# Directory segment is the human-readable month, not the raw ordinal 634. +query I +select count(*) from iceberg_metadata(my_datalake.default.hrpp_month) where file_path like '%=2022-11/%'; +---- +1 + +query I +select count(*) from iceberg_metadata(my_datalake.default.hrpp_month) where file_path like '%=634/%'; +---- +0 + +# Data round-trips and partition pruning still resolves against the manifest ordinal. +query II +select id, ts from my_datalake.default.hrpp_month +where ts >= TIMESTAMP '2022-11-01' and ts < TIMESTAMP '2022-12-01'; +---- +1 2022-11-15 10:30:00 + +# ---- year(dt) -> "2022" (ordinal 52) ---- + +statement ok +drop table if exists my_datalake.default.hrpp_year; + +statement ok +CREATE TABLE my_datalake.default.hrpp_year (id int, dt DATE) +PARTITIONED BY (year(dt)); + +statement ok +INSERT INTO my_datalake.default.hrpp_year VALUES (1, DATE '2022-06-15'); + +query I +select count(*) from iceberg_metadata(my_datalake.default.hrpp_year) where file_path like '%=2022/%'; +---- +1 + +query I +select count(*) from iceberg_metadata(my_datalake.default.hrpp_year) where file_path like '%=52/%'; +---- +0 + +query II +select id, dt from my_datalake.default.hrpp_year; +---- +1 2022-06-15 + +# ---- day(ts) -> "2022-11-15" ---- + +statement ok +drop table if exists my_datalake.default.hrpp_day; + +statement ok +CREATE TABLE my_datalake.default.hrpp_day (id int, ts TIMESTAMP) +PARTITIONED BY (day(ts)); + +statement ok +INSERT INTO my_datalake.default.hrpp_day VALUES (1, TIMESTAMP '2022-11-15 10:30:00'); + +query I +select count(*) from iceberg_metadata(my_datalake.default.hrpp_day) where file_path like '%=2022-11-15/%'; +---- +1 + +query II +select id, ts from my_datalake.default.hrpp_day; +---- +1 2022-11-15 10:30:00 + +# ---- hour(ts) -> "2022-11-15-13" (yyyy-MM-dd-HH, not a full timestamp) ---- + +statement ok +drop table if exists my_datalake.default.hrpp_hour; + +statement ok +CREATE TABLE my_datalake.default.hrpp_hour (id int, ts TIMESTAMP) +PARTITIONED BY (hour(ts)); + +statement ok +INSERT INTO my_datalake.default.hrpp_hour VALUES (1, TIMESTAMP '2022-11-15 13:30:00'); + +query I +select count(*) from iceberg_metadata(my_datalake.default.hrpp_hour) where file_path like '%=2022-11-15-13/%'; +---- +1 + +query II +select id, ts from my_datalake.default.hrpp_hour; +---- +1 2022-11-15 13:30:00