Write human-readable partition paths for temporal transforms

src/core/expression/iceberg_transform.cpp

@@ -1,6 +1,7 @@
 #include "core/expression/iceberg_transform.hpp"
 
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/common/types/date.hpp"
 #include "utf8proc_wrapper.hpp"
 #include "core/expression/iceberg_hash.hpp"
 
@@ -114,18 +115,47 @@ string IcebergTransform::PartitionValueToString(const Value &partition_value) co
 		return StringUtil::Format("%04d-%02d", year, month);
 	}
 	case IcebergTransformType::YEAR: {
-		return std::to_string(1970 + partition_value.GetValue<int32_t>());
+		return StringUtil::Format("%04d", 1970 + partition_value.GetValue<int32_t>());
 	}
 	case IcebergTransformType::HOUR: {
-		int64_t hours = partition_value.GetValue<int32_t>();
-		timestamp_t ts(hours * Interval::MICROS_PER_HOUR);
-		return Timestamp::ToString(ts);
+		// yyyy-MM-dd-HH (UTC), not a full timestamp
+		int32_t hours = partition_value.GetValue<int32_t>();
+		int64_t day_ordinal = IcebergFloorDiv(hours, 24);
+		int32_t hour_of_day = static_cast<int32_t>(hours - day_ordinal * 24);
+		int32_t year, month, day;
+		Date::Convert(date_t(static_cast<int32_t>(day_ordinal)), year, month, day);
+		return StringUtil::Format("%04d-%02d-%02d-%02d", year, month, day, hour_of_day);
 	}
 	default:
 		return partition_value.ToString();
 	}
 }
 
+Value IcebergTransform::PartitionStringToValue(const string &partition_string) const {
+	switch (type) {
+	case IcebergTransformType::DAY: {
+		return Value::INTEGER(Date::FromString(partition_string).days);
+	}
+	case IcebergTransformType::MONTH: {
+		auto dash = partition_string.find('-', 1);
+		int32_t year = std::stoi(partition_string.substr(0, dash));
+		int32_t month = std::stoi(partition_string.substr(dash + 1));
+		return Value::INTEGER((year - 1970) * 12 + (month - 1));
+	}
+	case IcebergTransformType::YEAR: {
+		return Value::INTEGER(std::stoi(partition_string) - 1970);
+	}
+	case IcebergTransformType::HOUR: {
+		auto last_dash = partition_string.rfind('-');
+		int32_t hour_of_day = std::stoi(partition_string.substr(last_dash + 1));
+		date_t d = Date::FromString(partition_string.substr(0, last_dash));
+		return Value::INTEGER(static_cast<int32_t>(static_cast<int64_t>(d.days) * 24 + hour_of_day));
+	}
+	default:
+		return Value(partition_string);
+	}
+}
+
 void IcebergTransform::SetBucketOrTruncateValue(idx_t value) {
 	switch (type) {
 	case IcebergTransformType::BUCKET:

src/execution/operator/iceberg_insert.cpp

@@ -246,7 +246,8 @@ void IcebergInsertGlobalState::AddFiles(DataChunk &chunk, const string &table_na
 				IcebergPartitionInfo info;
 				info.field_id = partition_field.partition_field_id;
 				if (!struct_val[1].IsNull()) {
-					info.value = Value(StringValue::Get(struct_val[1]));
+					// Restore the raw ordinal the manifest stores (temporal routing values are human-readable).
+					info.value = partition_field.transform.PartitionStringToValue(StringValue::Get(struct_val[1]));
 				} else {
 					info.value = Value();
 				}
@@ -585,6 +586,26 @@ static unique_ptr<Expression> GetDateDiffFunction(ClientContext &context, const
 	return function;
 }
 
+// Format the temporal ordinal as the human-readable routing value (hence the hive directory);
+// partition assignment is unchanged and AddFiles restores the ordinal for the manifest.
+static unique_ptr<Expression> GetTemporalHumanExpression(ClientContext &context, const IcebergCopyInput &copy_input,
+                                                         const string &date_part, uint64_t source_id) {
+	auto ordinal_expr = GetDateDiffFunction(context, copy_input, date_part, source_id);
+
+	vector<unique_ptr<Expression>> children;
+	children.push_back(make_uniq<BoundConstantExpression>(Value(date_part)));
+	children.push_back(std::move(ordinal_expr));
+
+	ErrorData error;
+	FunctionBinder binder(context);
+	auto function =
+	    binder.BindScalarFunction(DEFAULT_SCHEMA, "iceberg_partition_to_human", std::move(children), error, false);
+	if (!function) {
+		error.Throw();
+	}
+	return function;
+}
+
 //! Get an iceberg_bucket(N, col) expression for bucket partition transforms
 static unique_ptr<Expression> GetBucketExpression(ClientContext &context, const IcebergCopyInput &copy_input,
                                                   const IcebergPartitionSpecField &field) {
@@ -635,13 +656,13 @@ static unique_ptr<Expression> GetPartitionExpression(ClientContext &context, con
 		return CreateColumnReference(copy_input, col_type, col_idx);
 	}
 	case IcebergTransformType::YEAR:
-		return GetDateDiffFunction(context, copy_input, "year", field.source_id);
+		return GetTemporalHumanExpression(context, copy_input, "year", field.source_id);
 	case IcebergTransformType::MONTH:
-		return GetDateDiffFunction(context, copy_input, "month", field.source_id);
+		return GetTemporalHumanExpression(context, copy_input, "month", field.source_id);
 	case IcebergTransformType::DAY:
-		return GetDateDiffFunction(context, copy_input, "day", field.source_id);
+		return GetTemporalHumanExpression(context, copy_input, "day", field.source_id);
 	case IcebergTransformType::HOUR:
-		return GetDateDiffFunction(context, copy_input, "hour", field.source_id);
+		return GetTemporalHumanExpression(context, copy_input, "hour", field.source_id);
 	case IcebergTransformType::BUCKET:
 		return GetBucketExpression(context, copy_input, field);
 	case IcebergTransformType::TRUNCATE:

src/function/iceberg_functions.cpp

@@ -33,6 +33,7 @@ vector<ScalarFunctionSet> IcebergFunctions::GetScalarFunctions() {
 
 	functions.push_back(GetIcebergBucketFunction());
 	functions.push_back(GetIcebergTruncateFunction());
+	functions.push_back(GetIcebergPartitionToHumanFunction());
 	functions.push_back(GetVerifyEqualityDeletesFunction());
 
 	return functions;

src/function/iceberg_scalar_functions.cpp

@@ -10,6 +10,7 @@
 
 #include "function/iceberg_functions.hpp"
 #include "core/expression/iceberg_hash.hpp"
+#include "core/expression/iceberg_transform.hpp"
 
 #include "duckdb.hpp"
 #include "duckdb/common/vector_operations/binary_executor.hpp"
@@ -331,4 +332,28 @@ ScalarFunctionSet IcebergFunctions::GetIcebergTruncateFunction() {
 	return set;
 }
 
+//===--------------------------------------------------------------------===//
+// iceberg_partition_to_human(transform, ordinal) -> VARCHAR
+// Format a temporal partition ordinal as the human-readable path segment
+// (e.g. 'month', 634 -> "2022-11"), via Transform.toHumanString.
+//===--------------------------------------------------------------------===//
+
+static void IcebergPartitionToHuman(DataChunk &input, ExpressionState &state, Vector &result) {
+	BinaryExecutor::Execute<string_t, int64_t, string_t>(
+	    input.data[0], input.data[1], result, input.size(),
+	    [&result](string_t transform_name, int64_t ordinal) -> string_t {
+		    IcebergTransform transform(transform_name.GetString());
+		    auto human = transform.PartitionValueToString(Value::INTEGER(static_cast<int32_t>(ordinal)));
+		    return StringVector::AddString(result, human);
+	    });
+}
+
+ScalarFunctionSet IcebergFunctions::GetIcebergPartitionToHumanFunction() {
+	ScalarFunctionSet set("iceberg_partition_to_human");
+	// (transform_name, ordinal) -> VARCHAR
+	set.AddFunction(
+	    ScalarFunction({LogicalType::VARCHAR, LogicalType::BIGINT}, LogicalType::VARCHAR, IcebergPartitionToHuman));
+	return set;
+}
+
 } // namespace duckdb

src/include/core/expression/iceberg_transform.hpp

@@ -44,6 +44,7 @@ struct IcebergTransform {
 	LogicalType GetSerializedType(const LogicalType &input) const;
 	LogicalType GetBoundsType(const LogicalType &input) const;
 	string PartitionValueToString(const Value &partition_value) const;
+	Value PartitionStringToValue(const string &partition_string) const;
 	void SetBucketOrTruncateValue(idx_t value);
 
 private:

src/include/function/iceberg_functions.hpp

@@ -24,6 +24,7 @@ class IcebergFunctions {
 private:
 	static ScalarFunctionSet GetIcebergBucketFunction();
 	static ScalarFunctionSet GetIcebergTruncateFunction();
+	static ScalarFunctionSet GetIcebergPartitionToHumanFunction();
 	static ScalarFunctionSet GetVerifyEqualityDeletesFunction();
 
 	static TableFunctionSet GetIcebergSnapshotsFunction();

test/sql/local/catalog_test_config_setup/catalog_agnostic/insert/partitions/temporal/test_human_readable_partition_paths.test

@@ -0,0 +1,125 @@
+# name: test/sql/local/catalog_test_config_setup/catalog_agnostic/insert/partitions/temporal/test_human_readable_partition_paths.test
+# description: Temporal partition writes use the Iceberg human-readable path segment, not the raw ordinal
+# group: [temporal]
+
+# Partition directories use Transform.toHumanString: month -> "2022-11", year -> "2022",
+# day -> "2022-11-15", hour -> "2022-11-15-13" (not the raw integer ordinal). The manifest
+# partition value is unchanged (the INSERT would fail the BIGINT cast if the ordinal were
+# lost), so existing / other-engine files in the same logical partition are unaffected.
+
+require-env CATALOG_TEST_CONFIG_SETUP
+
+require avro
+
+require parquet
+
+require iceberg
+
+require httpfs
+
+# Do not ignore 'HTTP' error messages!
+set ignore_error_messages
+
+statement ok
+create schema if not exists my_datalake.default;
+
+# ---- month(ts) -> "2022-11" (ordinal 634) ----
+
+statement ok
+drop table if exists my_datalake.default.hrpp_month;
+
+statement ok
+CREATE TABLE my_datalake.default.hrpp_month (id int, ts TIMESTAMP)
+PARTITIONED BY (month(ts));
+
+statement ok
+INSERT INTO my_datalake.default.hrpp_month VALUES (1, TIMESTAMP '2022-11-15 10:30:00');
+
+# Directory segment is the human-readable month, not the raw ordinal 634.
+query I
+select count(*) from iceberg_metadata(my_datalake.default.hrpp_month) where file_path like '%=2022-11/%';
+----
+1
+
+query I
+select count(*) from iceberg_metadata(my_datalake.default.hrpp_month) where file_path like '%=634/%';
+----
+0
+
+# Data round-trips and partition pruning still resolves against the manifest ordinal.
+query II
+select id, ts from my_datalake.default.hrpp_month
+where ts >= TIMESTAMP '2022-11-01' and ts < TIMESTAMP '2022-12-01';
+----
+1	2022-11-15 10:30:00
+
+# ---- year(dt) -> "2022" (ordinal 52) ----
+
+statement ok
+drop table if exists my_datalake.default.hrpp_year;
+
+statement ok
+CREATE TABLE my_datalake.default.hrpp_year (id int, dt DATE)
+PARTITIONED BY (year(dt));
+
+statement ok
+INSERT INTO my_datalake.default.hrpp_year VALUES (1, DATE '2022-06-15');
+
+query I
+select count(*) from iceberg_metadata(my_datalake.default.hrpp_year) where file_path like '%=2022/%';
+----
+1
+
+query I
+select count(*) from iceberg_metadata(my_datalake.default.hrpp_year) where file_path like '%=52/%';
+----
+0
+
+query II
+select id, dt from my_datalake.default.hrpp_year;
+----
+1	2022-06-15
+
+# ---- day(ts) -> "2022-11-15" ----
+
+statement ok
+drop table if exists my_datalake.default.hrpp_day;
+
+statement ok
+CREATE TABLE my_datalake.default.hrpp_day (id int, ts TIMESTAMP)
+PARTITIONED BY (day(ts));
+
+statement ok
+INSERT INTO my_datalake.default.hrpp_day VALUES (1, TIMESTAMP '2022-11-15 10:30:00');
+
+query I
+select count(*) from iceberg_metadata(my_datalake.default.hrpp_day) where file_path like '%=2022-11-15/%';
+----
+1
+
+query II
+select id, ts from my_datalake.default.hrpp_day;
+----
+1	2022-11-15 10:30:00
+
+# ---- hour(ts) -> "2022-11-15-13" (yyyy-MM-dd-HH, not a full timestamp) ----
+
+statement ok
+drop table if exists my_datalake.default.hrpp_hour;
+
+statement ok
+CREATE TABLE my_datalake.default.hrpp_hour (id int, ts TIMESTAMP)
+PARTITIONED BY (hour(ts));
+
+statement ok
+INSERT INTO my_datalake.default.hrpp_hour VALUES (1, TIMESTAMP '2022-11-15 13:30:00');
+
+query I
+select count(*) from iceberg_metadata(my_datalake.default.hrpp_hour) where file_path like '%=2022-11-15-13/%';
+----
+1
+
+query II
+select id, ts from my_datalake.default.hrpp_hour;
+----
+1	2022-11-15 13:30:00