diff --git a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/converter/DorisTypeConverter.java b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/converter/DorisTypeConverter.java index df1fd341da9..8df2b4d332f 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/converter/DorisTypeConverter.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/converter/DorisTypeConverter.java @@ -24,6 +24,8 @@ /** Type converter for Apache Doris. */ public class DorisTypeConverter extends JdbcTypeConverter { + private static final int MAX_DATETIME_PRECISION = 6; + static final String BOOLEAN = "boolean"; static final String TINYINT = "tinyint"; static final String SMALLINT = "smallint"; @@ -188,6 +190,12 @@ public String fromGravitino(Type type) { return DATEV2; } else if (type instanceof Types.TimestampType) { Types.TimestampType timestampType = (Types.TimestampType) type; + if (timestampType.hasTimeZone()) { + throw unsupportedType(type, "Doris DATETIME does not store time-zone information"); + } + if (timestampType.hasPrecisionSet() && timestampType.precision() > MAX_DATETIME_PRECISION) { + throw unsupportedType(type, "fractional-second precision must be between 0 and 6"); + } return timestampType.hasPrecisionSet() ? String.format("%s(%d)", DATETIME, timestampType.precision()) : DATETIME; @@ -212,10 +220,31 @@ public String fromGravitino(Type type) { return STRING; } else if (type instanceof Types.BinaryType) { return BINARY; + } else if (type instanceof Types.VariantType) { + throw unsupportedType( + type, + "MySQL JDBC metadata reports Doris VARIANT as UNKNOWN, so the catalog cannot preserve " + + "the type on round-trip"); + } else if (type instanceof Types.NullType) { + throw unsupportedType(type, "the null-only placeholder has no Doris column type"); + } else if (type instanceof Types.GeometryType) { + throw unsupportedType( + type, + "Doris GEO uses String/Varchar storage and cannot preserve Geometry CRS metadata as a " + + "column type"); + } else if (type instanceof Types.GeographyType) { + throw unsupportedType( + type, + "Doris GEO has no Geography column type that preserves CRS and edge-algorithm metadata"); } else if (type instanceof Types.ExternalType) { return ((Types.ExternalType) type).catalogString(); } throw new IllegalArgumentException( String.format("Couldn't convert Gravitino type %s to Doris type", type.simpleString())); } + + private static IllegalArgumentException unsupportedType(Type type, String reason) { + return new IllegalArgumentException( + String.format("Doris does not support Gravitino type %s: %s", type.simpleString(), reason)); + } } diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/converter/TestDorisTypeConverter.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/converter/TestDorisTypeConverter.java index 3958ef0e469..ba301027d6b 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/converter/TestDorisTypeConverter.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/converter/TestDorisTypeConverter.java @@ -115,6 +115,82 @@ public void testFromGravitinoType() { () -> DORIS_TYPE_CONVERTER.fromGravitino(Types.UnparsedType.of(USER_DEFINED_TYPE))); } + @Test + public void testRejectNanosecondTimestampTypes() { + checkGravitinoTypeToJdbcType("datetime(6)", Types.TimestampType.withoutTimeZone(6)); + + IllegalArgumentException timestampException = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> DORIS_TYPE_CONVERTER.fromGravitino(Types.TimestampType.withoutTimeZone(9))); + Assertions.assertTrue( + timestampException + .getMessage() + .contains("fractional-second precision must be between 0 and 6")); + + IllegalArgumentException timestampTzException = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> DORIS_TYPE_CONVERTER.fromGravitino(Types.TimestampType.withTimeZone(9))); + Assertions.assertTrue( + timestampTzException + .getMessage() + .contains("Doris DATETIME does not store time-zone information")); + } + + @Test + public void testRejectVariantType() { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> DORIS_TYPE_CONVERTER.fromGravitino(Types.VariantType.get())); + Assertions.assertTrue( + exception + .getMessage() + .contains( + "MySQL JDBC metadata reports Doris VARIANT as UNKNOWN, so the catalog cannot " + + "preserve the type on round-trip")); + } + + @Test + public void testRejectNullType() { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> DORIS_TYPE_CONVERTER.fromGravitino(Types.NullType.get())); + Assertions.assertTrue( + exception.getMessage().contains("the null-only placeholder has no Doris column type")); + } + + @Test + public void testRejectGeometryType() { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> DORIS_TYPE_CONVERTER.fromGravitino(Types.GeometryType.of("SRID:3857"))); + Assertions.assertTrue( + exception + .getMessage() + .contains( + "Doris GEO uses String/Varchar storage and cannot preserve Geometry CRS metadata " + + "as a column type")); + } + + @Test + public void testRejectGeographyType() { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + DORIS_TYPE_CONVERTER.fromGravitino(Types.GeographyType.of("EPSG:4326", "karney"))); + Assertions.assertTrue( + exception + .getMessage() + .contains( + "Doris GEO has no Geography column type that preserves CRS and edge-algorithm " + + "metadata")); + } + @Test public void testExternalTypeRoundTrip() { // ExternalType round-trip: fromGravitino(ExternalType) → toGravitino diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index e474c10fa76..a9341bc17c1 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -77,6 +77,7 @@ import org.apache.gravitino.rel.partitions.Partition; import org.apache.gravitino.rel.partitions.Partitions; import org.apache.gravitino.rel.partitions.RangePartition; +import org.apache.gravitino.rel.types.Type; import org.apache.gravitino.rel.types.Types; import org.apache.gravitino.utils.RandomNameUtils; import org.awaitility.Awaitility; @@ -950,6 +951,48 @@ void testTableWithTimeStampColumn() { } } + @Test + void testRejectNanosecondTimestampsWithoutSideEffects() { + assertUnsupportedV3TypeDoesNotCreateTable( + "test_timestamp_ns", + Types.TimestampType.withoutTimeZone(9), + "fractional-second precision must be between 0 and 6"); + assertUnsupportedV3TypeDoesNotCreateTable( + "test_timestamp_tz_ns", + Types.TimestampType.withTimeZone(9), + "Doris DATETIME does not store time-zone information"); + } + + @Test + void testRejectVariantWithoutSideEffects() { + assertUnsupportedV3TypeDoesNotCreateTable( + "test_variant", + Types.VariantType.get(), + "MySQL JDBC metadata reports Doris VARIANT as UNKNOWN"); + } + + @Test + void testRejectNullTypeWithoutSideEffects() { + assertUnsupportedV3TypeDoesNotCreateTable( + "test_null", Types.NullType.get(), "the null-only placeholder has no Doris column type"); + } + + @Test + void testRejectGeometryWithoutSideEffects() { + assertUnsupportedV3TypeDoesNotCreateTable( + "test_geometry", + Types.GeometryType.of("SRID:3857"), + "Doris GEO uses String/Varchar storage and cannot preserve Geometry CRS metadata"); + } + + @Test + void testRejectGeographyWithoutSideEffects() { + assertUnsupportedV3TypeDoesNotCreateTable( + "test_geography", + Types.GeographyType.of("EPSG:4326", "karney"), + "Doris GEO has no Geography column type that preserves CRS and edge-algorithm metadata"); + } + @Test void testNonPartitionedTable() { // create a non-partitioned table @@ -1239,4 +1282,34 @@ void testMultiColumnListPartitionRoundTrip() { assertPartition(Partitions.list("p1", p1Values, Collections.emptyMap()), partitions.get("p1")); assertPartition(Partitions.list("p2", p2Values, Collections.emptyMap()), partitions.get("p2")); } + + private void assertUnsupportedV3TypeDoesNotCreateTable( + String tablePrefix, Type type, String expectedMessage) { + NameIdentifier tableIdentifier = + NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName(tablePrefix)); + Column[] columns = { + Column.of("id", Types.IntegerType.get(), "distribution column"), + Column.of("v3_column", type, "V3 column") + }; + Distribution distribution = Distributions.hash(1, NamedReference.field("id")); + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + catalog + .asTableCatalog() + .createTable( + tableIdentifier, + columns, + null, + Collections.emptyMap(), + Transforms.EMPTY_TRANSFORM, + distribution, + null, + Indexes.EMPTY_INDEXES)); + + assertTrue(exception.getMessage().contains(expectedMessage), exception::getMessage); + assertFalse(catalog.asTableCatalog().tableExists(tableIdentifier)); + } } diff --git a/docs/jdbc-doris-catalog.md b/docs/jdbc-doris-catalog.md index 474ceee62f8..d830b7fbe74 100644 --- a/docs/jdbc-doris-catalog.md +++ b/docs/jdbc-doris-catalog.md @@ -126,7 +126,7 @@ Refer to | `Double` | `Double` | | `Decimal` | `Decimal` | | `Date` | `Date`/`DateV2` | -| `Timestamp[(p)]` | `Datetime[(p)]` | +| `Timestamp[(p)]` | `Datetime[(p)]`, p in [0, 6] | | `VarChar` | `VarChar` | | `FixedChar` | `Char` | | `String` | `String` | @@ -139,7 +139,26 @@ Refer to | `ExternalType("bitmap")` | `Bitmap` | | `ExternalType("hll")` | `HLL` | -Doris doesn't support Gravitino `Fixed` `Timestamp_tz` `IntervalDay` `IntervalYear` `Union` `UUID` type. +Apache Doris `DATETIME` supports fractional-second precision from 0 through 6 and does not store +time-zone information. The catalog rejects `Timestamp` precision from 7 through 9 and all +`Timestamp_tz` types before executing DDL. + +Although Doris 2.1 and later have a native `VARIANT` type, MySQL JDBC metadata reports it as +`UNKNOWN`. The catalog therefore rejects Gravitino `Variant` before executing DDL because a +create/get round-trip cannot preserve the type family. `ExternalType("variant")` remains the raw +Doris type escape hatch. + +Gravitino `Null` (the V3 unknown type) is rejected before executing DDL because it is a null-only +placeholder, not a Doris column type. + +Gravitino `Geometry` is rejected before executing DDL. Doris GEO is not a column type; it stores +geospatial values in `String`/`VarChar`, which cannot preserve Gravitino CRS metadata as part of the +type. + +Gravitino `Geography` is also rejected before executing DDL because Doris has no geography column +type that preserves both its CRS and edge-algorithm metadata. + +Doris doesn't support Gravitino `Fixed` `Timestamp_tz` `IntervalDay` `IntervalYear` `Union` `UUID` `Variant` `Null` `Geometry` `Geography` type. The data types other than those listed above are mapped to Gravitino's **[Unparsed Type](./manage-relational-metadata-using-gravitino.md#unparsed-type)** that represents an unresolvable data type since 0.5.0. :::note