diff --git a/catalogs/catalog-lakehouse-hudi/src/main/java/org/apache/gravitino/catalog/lakehouse/hudi/HudiCatalogOperations.java b/catalogs/catalog-lakehouse-hudi/src/main/java/org/apache/gravitino/catalog/lakehouse/hudi/HudiCatalogOperations.java index f471dbbdcea..80b48171e76 100644 --- a/catalogs/catalog-lakehouse-hudi/src/main/java/org/apache/gravitino/catalog/lakehouse/hudi/HudiCatalogOperations.java +++ b/catalogs/catalog-lakehouse-hudi/src/main/java/org/apache/gravitino/catalog/lakehouse/hudi/HudiCatalogOperations.java @@ -49,6 +49,8 @@ import org.apache.gravitino.rel.expressions.sorts.SortOrder; import org.apache.gravitino.rel.expressions.transforms.Transform; import org.apache.gravitino.rel.indexes.Index; +import org.apache.gravitino.rel.types.Type; +import org.apache.gravitino.rel.types.Types; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -237,6 +239,9 @@ public Table createTable( SortOrder[] sortOrders, Index[] indexes) throws NoSuchSchemaException, TableAlreadyExistsException { + for (Column column : columns) { + validateWritableType(column.dataType()); + } return hudiCatalogBackendOps.createTable( ident, columns, comment, properties, partitions, distribution, sortOrders, indexes); } @@ -266,4 +271,52 @@ public Table alterTable(NameIdentifier ident, TableChange... changes) public boolean dropTable(NameIdentifier ident) { return hudiCatalogBackendOps.dropTable(ident); } + + private static void validateWritableType(Type type) { + if (type instanceof Types.GeographyType) { + throw new IllegalArgumentException( + "Unsupported Gravitino type for Hudi: geography. " + + "Hudi 0.15 does not define a Geography type that preserves CRS and edge " + + "algorithm semantics."); + } + if (type instanceof Types.GeometryType) { + throw new IllegalArgumentException( + "Unsupported Gravitino type for Hudi: geometry. " + + "Hudi 0.15 does not define a Geometry type that preserves CRS semantics."); + } + if (type instanceof Types.NullType) { + throw new IllegalArgumentException( + "Unsupported Gravitino type for Hudi: unknown. " + + "Hudi 0.15 table columns require a concrete value type."); + } + if (type instanceof Types.VariantType) { + throw new IllegalArgumentException( + "Unsupported Gravitino type for Hudi: variant. " + + "Hudi 0.15 does not define a Variant column type."); + } + if (type instanceof Types.TimestampType) { + Types.TimestampType timestampType = (Types.TimestampType) type; + if (timestampType.hasPrecisionSet() && timestampType.precision() > 6) { + throw new IllegalArgumentException( + "Unsupported Gravitino type for Hudi: " + + type.simpleString() + + ". Hudi 0.15 cannot preserve timestamp precision greater than 6."); + } + } + if (type instanceof Types.ListType) { + validateWritableType(((Types.ListType) type).elementType()); + } else if (type instanceof Types.MapType) { + Types.MapType mapType = (Types.MapType) type; + validateWritableType(mapType.keyType()); + validateWritableType(mapType.valueType()); + } else if (type instanceof Types.StructType) { + for (Types.StructType.Field field : ((Types.StructType) type).fields()) { + validateWritableType(field.type()); + } + } else if (type instanceof Types.UnionType) { + for (Type unionType : ((Types.UnionType) type).types()) { + validateWritableType(unionType); + } + } + } } diff --git a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/TestHudiCatalogOperations.java b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/TestHudiCatalogOperations.java index 8fb4128b889..be2e47730d6 100644 --- a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/TestHudiCatalogOperations.java +++ b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/TestHudiCatalogOperations.java @@ -21,6 +21,7 @@ import static org.apache.gravitino.catalog.lakehouse.hudi.HudiCatalogPropertiesMetadata.CATALOG_BACKEND; import com.google.common.collect.ImmutableMap; +import java.util.Collections; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.catalog.lakehouse.hudi.backend.hms.HudiHMSBackendOps; import org.apache.gravitino.catalog.lakehouse.hudi.ops.InMemoryBackendOps; @@ -28,6 +29,13 @@ import org.apache.gravitino.connector.PropertiesMetadata; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NoSuchTableException; +import org.apache.gravitino.rel.Column; +import org.apache.gravitino.rel.expressions.distributions.Distributions; +import org.apache.gravitino.rel.expressions.sorts.SortOrders; +import org.apache.gravitino.rel.expressions.transforms.Transforms; +import org.apache.gravitino.rel.indexes.Indexes; +import org.apache.gravitino.rel.types.Type; +import org.apache.gravitino.rel.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -137,4 +145,77 @@ public void testLoadTable() throws Exception { () -> ops.loadTable(NameIdentifier.of("metalake", "catalog", "table"))); } } + + @Test + public void testCreateTableRejectsNanosecondTimestampsBeforeBackendCall() throws Exception { + assertCreateRejected( + Types.TimestampType.withoutTimeZone(9), + "Unsupported Gravitino type for Hudi: timestamp(9). " + + "Hudi 0.15 cannot preserve timestamp precision greater than 6."); + assertCreateRejected( + Types.TimestampType.withTimeZone(9), + "Unsupported Gravitino type for Hudi: timestamp_tz(9). " + + "Hudi 0.15 cannot preserve timestamp precision greater than 6."); + } + + @Test + public void testCreateTableRejectsVariantBeforeBackendCall() throws Exception { + assertCreateRejected( + Types.VariantType.get(), + "Unsupported Gravitino type for Hudi: variant. " + + "Hudi 0.15 does not define a Variant column type."); + } + + @Test + public void testCreateTableRejectsUnknownBeforeBackendCall() throws Exception { + assertCreateRejected( + Types.NullType.get(), + "Unsupported Gravitino type for Hudi: unknown. " + + "Hudi 0.15 table columns require a concrete value type."); + } + + @Test + public void testCreateTableRejectsGeometryBeforeBackendCall() throws Exception { + assertCreateRejected( + Types.GeometryType.of("EPSG:3857"), + "Unsupported Gravitino type for Hudi: geometry. " + + "Hudi 0.15 does not define a Geometry type that preserves CRS semantics."); + } + + @Test + public void testCreateTableRejectsGeographyBeforeBackendCall() throws Exception { + assertCreateRejected( + Types.GeographyType.of("EPSG:4326", "SPHERICAL"), + "Unsupported Gravitino type for Hudi: geography. " + + "Hudi 0.15 does not define a Geography type that preserves CRS and edge " + + "algorithm semantics."); + } + + private static void assertCreateRejected(Type type, String expectedMessage) throws Exception { + try (InMemoryBackendOps backend = new InMemoryBackendOps()) { + HudiCatalogOperations ops = new HudiCatalogOperations(); + ops.hudiCatalogBackendOps = backend; + NameIdentifier ident = NameIdentifier.of("metalake", "catalog", "schema", "table"); + Column[] columns = { + HudiColumn.builder().withName("v3_column").withType(type).withNullable(true).build() + }; + + IllegalArgumentException error = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + ops.createTable( + ident, + columns, + null, + Collections.emptyMap(), + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + SortOrders.NONE, + Indexes.EMPTY_INDEXES)); + + Assertions.assertEquals(expectedMessage, error.getMessage()); + Assertions.assertFalse(backend.tableExists(ident)); + } + } } diff --git a/docs/lakehouse-hudi-catalog.md b/docs/lakehouse-hudi-catalog.md index 0b78a1dcbf5..9182264e515 100644 --- a/docs/lakehouse-hudi-catalog.md +++ b/docs/lakehouse-hudi-catalog.md @@ -121,6 +121,18 @@ The following table shows the mapping between Gravitino and [Apache Hudi column | `map` | `map` | | `struct` | `struct` | +#### V3 Type Write Compatibility + +The current Gravitino Hudi catalog is read-only. It nevertheless validates V3 column types before delegating a create request so that an incompatible type returns a deterministic invalid-argument error without invoking the backend. + +| Gravitino Type | Hudi 0.15 Write Outcome | +|--------------------------------------|-------------------------| +| `timestamp(9)` / `timestamp_tz(9)` | Rejected before the backend is called. Hudi 0.15 supports timestamp logical types only through microsecond precision. | +| `variant` | Rejected before the backend is called. Hudi 0.15 does not define a Variant column type. | +| `unknown` | Rejected before the backend is called. Hudi 0.15 table columns require a concrete value type. | +| `geometry(crs)` | Rejected before the backend is called. Hudi 0.15 does not define a Geometry type that preserves CRS semantics. | +| `geography(crs, algorithm)` | Rejected before the backend is called. Hudi 0.15 does not define a Geography type that preserves CRS and edge-algorithm semantics. | + ### Table Operations Only support read operations: listTable, loadTable, and tableExists.