Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@
import org.apache.gravitino.rel.expressions.sorts.SortOrder;
import org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.indexes.Index;
import org.apache.gravitino.rel.types.Type;
import org.apache.gravitino.rel.types.Types;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -237,6 +239,9 @@ public Table createTable(
SortOrder[] sortOrders,
Index[] indexes)
throws NoSuchSchemaException, TableAlreadyExistsException {
for (Column column : columns) {
validateWritableType(column.dataType());
}
return hudiCatalogBackendOps.createTable(
ident, columns, comment, properties, partitions, distribution, sortOrders, indexes);
}
Expand Down Expand Up @@ -266,4 +271,52 @@ public Table alterTable(NameIdentifier ident, TableChange... changes)
public boolean dropTable(NameIdentifier ident) {
return hudiCatalogBackendOps.dropTable(ident);
}

private static void validateWritableType(Type type) {
if (type instanceof Types.GeographyType) {
throw new IllegalArgumentException(
"Unsupported Gravitino type for Hudi: geography. "
+ "Hudi 0.15 does not define a Geography type that preserves CRS and edge "
+ "algorithm semantics.");
}
if (type instanceof Types.GeometryType) {
throw new IllegalArgumentException(
"Unsupported Gravitino type for Hudi: geometry. "
+ "Hudi 0.15 does not define a Geometry type that preserves CRS semantics.");
}
if (type instanceof Types.NullType) {
throw new IllegalArgumentException(
"Unsupported Gravitino type for Hudi: unknown. "
+ "Hudi 0.15 table columns require a concrete value type.");
}
if (type instanceof Types.VariantType) {
throw new IllegalArgumentException(
"Unsupported Gravitino type for Hudi: variant. "
+ "Hudi 0.15 does not define a Variant column type.");
}
if (type instanceof Types.TimestampType) {
Types.TimestampType timestampType = (Types.TimestampType) type;
if (timestampType.hasPrecisionSet() && timestampType.precision() > 6) {
throw new IllegalArgumentException(
"Unsupported Gravitino type for Hudi: "
+ type.simpleString()
+ ". Hudi 0.15 cannot preserve timestamp precision greater than 6.");
}
}
if (type instanceof Types.ListType) {
validateWritableType(((Types.ListType) type).elementType());
} else if (type instanceof Types.MapType) {
Types.MapType mapType = (Types.MapType) type;
validateWritableType(mapType.keyType());
validateWritableType(mapType.valueType());
} else if (type instanceof Types.StructType) {
for (Types.StructType.Field field : ((Types.StructType) type).fields()) {
validateWritableType(field.type());
}
} else if (type instanceof Types.UnionType) {
for (Type unionType : ((Types.UnionType) type).types()) {
validateWritableType(unionType);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,21 @@
import static org.apache.gravitino.catalog.lakehouse.hudi.HudiCatalogPropertiesMetadata.CATALOG_BACKEND;

import com.google.common.collect.ImmutableMap;
import java.util.Collections;
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.catalog.lakehouse.hudi.backend.hms.HudiHMSBackendOps;
import org.apache.gravitino.catalog.lakehouse.hudi.ops.InMemoryBackendOps;
import org.apache.gravitino.connector.HasPropertyMetadata;
import org.apache.gravitino.connector.PropertiesMetadata;
import org.apache.gravitino.exceptions.NoSuchSchemaException;
import org.apache.gravitino.exceptions.NoSuchTableException;
import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.expressions.distributions.Distributions;
import org.apache.gravitino.rel.expressions.sorts.SortOrders;
import org.apache.gravitino.rel.expressions.transforms.Transforms;
import org.apache.gravitino.rel.indexes.Indexes;
import org.apache.gravitino.rel.types.Type;
import org.apache.gravitino.rel.types.Types;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -137,4 +145,77 @@ public void testLoadTable() throws Exception {
() -> ops.loadTable(NameIdentifier.of("metalake", "catalog", "table")));
}
}

@Test
public void testCreateTableRejectsNanosecondTimestampsBeforeBackendCall() throws Exception {
assertCreateRejected(
Types.TimestampType.withoutTimeZone(9),
"Unsupported Gravitino type for Hudi: timestamp(9). "
+ "Hudi 0.15 cannot preserve timestamp precision greater than 6.");
assertCreateRejected(
Types.TimestampType.withTimeZone(9),
"Unsupported Gravitino type for Hudi: timestamp_tz(9). "
+ "Hudi 0.15 cannot preserve timestamp precision greater than 6.");
}

@Test
public void testCreateTableRejectsVariantBeforeBackendCall() throws Exception {
assertCreateRejected(
Types.VariantType.get(),
"Unsupported Gravitino type for Hudi: variant. "
+ "Hudi 0.15 does not define a Variant column type.");
}

@Test
public void testCreateTableRejectsUnknownBeforeBackendCall() throws Exception {
assertCreateRejected(
Types.NullType.get(),
"Unsupported Gravitino type for Hudi: unknown. "
+ "Hudi 0.15 table columns require a concrete value type.");
}

@Test
public void testCreateTableRejectsGeometryBeforeBackendCall() throws Exception {
assertCreateRejected(
Types.GeometryType.of("EPSG:3857"),
"Unsupported Gravitino type for Hudi: geometry. "
+ "Hudi 0.15 does not define a Geometry type that preserves CRS semantics.");
}

@Test
public void testCreateTableRejectsGeographyBeforeBackendCall() throws Exception {
assertCreateRejected(
Types.GeographyType.of("EPSG:4326", "SPHERICAL"),
"Unsupported Gravitino type for Hudi: geography. "
+ "Hudi 0.15 does not define a Geography type that preserves CRS and edge "
+ "algorithm semantics.");
}

private static void assertCreateRejected(Type type, String expectedMessage) throws Exception {
try (InMemoryBackendOps backend = new InMemoryBackendOps()) {
HudiCatalogOperations ops = new HudiCatalogOperations();
ops.hudiCatalogBackendOps = backend;
NameIdentifier ident = NameIdentifier.of("metalake", "catalog", "schema", "table");
Column[] columns = {
HudiColumn.builder().withName("v3_column").withType(type).withNullable(true).build()
};

IllegalArgumentException error =
Assertions.assertThrows(
IllegalArgumentException.class,
() ->
ops.createTable(
ident,
columns,
null,
Collections.emptyMap(),
Transforms.EMPTY_TRANSFORM,
Distributions.NONE,
SortOrders.NONE,
Indexes.EMPTY_INDEXES));

Assertions.assertEquals(expectedMessage, error.getMessage());
Assertions.assertFalse(backend.tableExists(ident));
}
}
}
12 changes: 12 additions & 0 deletions docs/lakehouse-hudi-catalog.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,18 @@ The following table shows the mapping between Gravitino and [Apache Hudi column
| `map` | `map` |
| `struct` | `struct` |

#### V3 Type Write Compatibility

The current Gravitino Hudi catalog is read-only. It nevertheless validates V3 column types before delegating a create request so that an incompatible type returns a deterministic invalid-argument error without invoking the backend.

| Gravitino Type | Hudi 0.15 Write Outcome |
|--------------------------------------|-------------------------|
| `timestamp(9)` / `timestamp_tz(9)` | Rejected before the backend is called. Hudi 0.15 supports timestamp logical types only through microsecond precision. |
| `variant` | Rejected before the backend is called. Hudi 0.15 does not define a Variant column type. |
| `unknown` | Rejected before the backend is called. Hudi 0.15 table columns require a concrete value type. |
| `geometry(crs)` | Rejected before the backend is called. Hudi 0.15 does not define a Geometry type that preserves CRS semantics. |
| `geography(crs, algorithm)` | Rejected before the backend is called. Hudi 0.15 does not define a Geography type that preserves CRS and edge-algorithm semantics. |

### Table Operations

Only support read operations: listTable, loadTable, and tableExists.
Expand Down
Loading