Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@

/** Type converter for Apache Doris. */
public class DorisTypeConverter extends JdbcTypeConverter {
private static final int MAX_DATETIME_PRECISION = 6;

static final String BOOLEAN = "boolean";
static final String TINYINT = "tinyint";
static final String SMALLINT = "smallint";
Expand Down Expand Up @@ -188,6 +190,12 @@ public String fromGravitino(Type type) {
return DATEV2;
} else if (type instanceof Types.TimestampType) {
Types.TimestampType timestampType = (Types.TimestampType) type;
if (timestampType.hasTimeZone()) {
throw unsupportedType(type, "Doris DATETIME does not store time-zone information");
}
if (timestampType.hasPrecisionSet() && timestampType.precision() > MAX_DATETIME_PRECISION) {
throw unsupportedType(type, "fractional-second precision must be between 0 and 6");
}
return timestampType.hasPrecisionSet()
? String.format("%s(%d)", DATETIME, timestampType.precision())
: DATETIME;
Expand All @@ -212,10 +220,31 @@ public String fromGravitino(Type type) {
return STRING;
} else if (type instanceof Types.BinaryType) {
return BINARY;
} else if (type instanceof Types.VariantType) {
throw unsupportedType(
type,
"MySQL JDBC metadata reports Doris VARIANT as UNKNOWN, so the catalog cannot preserve "
+ "the type on round-trip");
} else if (type instanceof Types.NullType) {
throw unsupportedType(type, "the null-only placeholder has no Doris column type");
} else if (type instanceof Types.GeometryType) {
throw unsupportedType(
type,
"Doris GEO uses String/Varchar storage and cannot preserve Geometry CRS metadata as a "
+ "column type");
} else if (type instanceof Types.GeographyType) {
throw unsupportedType(
type,
"Doris GEO has no Geography column type that preserves CRS and edge-algorithm metadata");
} else if (type instanceof Types.ExternalType) {
return ((Types.ExternalType) type).catalogString();
}
throw new IllegalArgumentException(
String.format("Couldn't convert Gravitino type %s to Doris type", type.simpleString()));
}

private static IllegalArgumentException unsupportedType(Type type, String reason) {
return new IllegalArgumentException(
String.format("Doris does not support Gravitino type %s: %s", type.simpleString(), reason));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,82 @@ public void testFromGravitinoType() {
() -> DORIS_TYPE_CONVERTER.fromGravitino(Types.UnparsedType.of(USER_DEFINED_TYPE)));
}

@Test
public void testRejectNanosecondTimestampTypes() {
checkGravitinoTypeToJdbcType("datetime(6)", Types.TimestampType.withoutTimeZone(6));

IllegalArgumentException timestampException =
Assertions.assertThrows(
IllegalArgumentException.class,
() -> DORIS_TYPE_CONVERTER.fromGravitino(Types.TimestampType.withoutTimeZone(9)));
Assertions.assertTrue(
timestampException
.getMessage()
.contains("fractional-second precision must be between 0 and 6"));

IllegalArgumentException timestampTzException =
Assertions.assertThrows(
IllegalArgumentException.class,
() -> DORIS_TYPE_CONVERTER.fromGravitino(Types.TimestampType.withTimeZone(9)));
Assertions.assertTrue(
timestampTzException
.getMessage()
.contains("Doris DATETIME does not store time-zone information"));
}

@Test
public void testRejectVariantType() {
IllegalArgumentException exception =
Assertions.assertThrows(
IllegalArgumentException.class,
() -> DORIS_TYPE_CONVERTER.fromGravitino(Types.VariantType.get()));
Assertions.assertTrue(
exception
.getMessage()
.contains(
"MySQL JDBC metadata reports Doris VARIANT as UNKNOWN, so the catalog cannot "
+ "preserve the type on round-trip"));
}

@Test
public void testRejectNullType() {
IllegalArgumentException exception =
Assertions.assertThrows(
IllegalArgumentException.class,
() -> DORIS_TYPE_CONVERTER.fromGravitino(Types.NullType.get()));
Assertions.assertTrue(
exception.getMessage().contains("the null-only placeholder has no Doris column type"));
}

@Test
public void testRejectGeometryType() {
IllegalArgumentException exception =
Assertions.assertThrows(
IllegalArgumentException.class,
() -> DORIS_TYPE_CONVERTER.fromGravitino(Types.GeometryType.of("SRID:3857")));
Assertions.assertTrue(
exception
.getMessage()
.contains(
"Doris GEO uses String/Varchar storage and cannot preserve Geometry CRS metadata "
+ "as a column type"));
}

@Test
public void testRejectGeographyType() {
IllegalArgumentException exception =
Assertions.assertThrows(
IllegalArgumentException.class,
() ->
DORIS_TYPE_CONVERTER.fromGravitino(Types.GeographyType.of("EPSG:4326", "karney")));
Assertions.assertTrue(
exception
.getMessage()
.contains(
"Doris GEO has no Geography column type that preserves CRS and edge-algorithm "
+ "metadata"));
}

@Test
public void testExternalTypeRoundTrip() {
// ExternalType round-trip: fromGravitino(ExternalType) → toGravitino
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
import org.apache.gravitino.rel.partitions.Partition;
import org.apache.gravitino.rel.partitions.Partitions;
import org.apache.gravitino.rel.partitions.RangePartition;
import org.apache.gravitino.rel.types.Type;
import org.apache.gravitino.rel.types.Types;
import org.apache.gravitino.utils.RandomNameUtils;
import org.awaitility.Awaitility;
Expand Down Expand Up @@ -950,6 +951,48 @@ void testTableWithTimeStampColumn() {
}
}

@Test
void testRejectNanosecondTimestampsWithoutSideEffects() {
assertUnsupportedV3TypeDoesNotCreateTable(
"test_timestamp_ns",
Types.TimestampType.withoutTimeZone(9),
"fractional-second precision must be between 0 and 6");
assertUnsupportedV3TypeDoesNotCreateTable(
"test_timestamp_tz_ns",
Types.TimestampType.withTimeZone(9),
"Doris DATETIME does not store time-zone information");
}

@Test
void testRejectVariantWithoutSideEffects() {
assertUnsupportedV3TypeDoesNotCreateTable(
"test_variant",
Types.VariantType.get(),
"MySQL JDBC metadata reports Doris VARIANT as UNKNOWN");
}

@Test
void testRejectNullTypeWithoutSideEffects() {
assertUnsupportedV3TypeDoesNotCreateTable(
"test_null", Types.NullType.get(), "the null-only placeholder has no Doris column type");
}

@Test
void testRejectGeometryWithoutSideEffects() {
assertUnsupportedV3TypeDoesNotCreateTable(
"test_geometry",
Types.GeometryType.of("SRID:3857"),
"Doris GEO uses String/Varchar storage and cannot preserve Geometry CRS metadata");
}

@Test
void testRejectGeographyWithoutSideEffects() {
assertUnsupportedV3TypeDoesNotCreateTable(
"test_geography",
Types.GeographyType.of("EPSG:4326", "karney"),
"Doris GEO has no Geography column type that preserves CRS and edge-algorithm metadata");
}

@Test
void testNonPartitionedTable() {
// create a non-partitioned table
Expand Down Expand Up @@ -1239,4 +1282,34 @@ void testMultiColumnListPartitionRoundTrip() {
assertPartition(Partitions.list("p1", p1Values, Collections.emptyMap()), partitions.get("p1"));
assertPartition(Partitions.list("p2", p2Values, Collections.emptyMap()), partitions.get("p2"));
}

private void assertUnsupportedV3TypeDoesNotCreateTable(
String tablePrefix, Type type, String expectedMessage) {
NameIdentifier tableIdentifier =
NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName(tablePrefix));
Column[] columns = {
Column.of("id", Types.IntegerType.get(), "distribution column"),
Column.of("v3_column", type, "V3 column")
};
Distribution distribution = Distributions.hash(1, NamedReference.field("id"));

IllegalArgumentException exception =
assertThrows(
IllegalArgumentException.class,
() ->
catalog
.asTableCatalog()
.createTable(
tableIdentifier,
columns,
null,
Collections.emptyMap(),
Transforms.EMPTY_TRANSFORM,
distribution,
null,
Indexes.EMPTY_INDEXES));

assertTrue(exception.getMessage().contains(expectedMessage), exception::getMessage);
assertFalse(catalog.asTableCatalog().tableExists(tableIdentifier));
}
}
23 changes: 21 additions & 2 deletions docs/jdbc-doris-catalog.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ Refer to
| `Double` | `Double` |
| `Decimal` | `Decimal` |
| `Date` | `Date`/`DateV2` |
| `Timestamp[(p)]` | `Datetime[(p)]` |
| `Timestamp[(p)]` | `Datetime[(p)]`, p in [0, 6] |
| `VarChar` | `VarChar` |
| `FixedChar` | `Char` |
| `String` | `String` |
Expand All @@ -139,7 +139,26 @@ Refer to
| `ExternalType("bitmap")` | `Bitmap` |
| `ExternalType("hll")` | `HLL` |

Doris doesn't support Gravitino `Fixed` `Timestamp_tz` `IntervalDay` `IntervalYear` `Union` `UUID` type.
Apache Doris `DATETIME` supports fractional-second precision from 0 through 6 and does not store
time-zone information. The catalog rejects `Timestamp` precision from 7 through 9 and all
`Timestamp_tz` types before executing DDL.

Although Doris 2.1 and later have a native `VARIANT` type, MySQL JDBC metadata reports it as
`UNKNOWN`. The catalog therefore rejects Gravitino `Variant` before executing DDL because a
create/get round-trip cannot preserve the type family. `ExternalType("variant")` remains the raw
Doris type escape hatch.

Gravitino `Null` (the V3 unknown type) is rejected before executing DDL because it is a null-only
placeholder, not a Doris column type.

Gravitino `Geometry` is rejected before executing DDL. Doris GEO is not a column type; it stores
geospatial values in `String`/`VarChar`, which cannot preserve Gravitino CRS metadata as part of the
type.

Gravitino `Geography` is also rejected before executing DDL because Doris has no geography column
type that preserves both its CRS and edge-algorithm metadata.

Doris doesn't support Gravitino `Fixed` `Timestamp_tz` `IntervalDay` `IntervalYear` `Union` `UUID` `Variant` `Null` `Geometry` `Geography` type.
The data types other than those listed above are mapped to Gravitino's **[Unparsed Type](./manage-relational-metadata-using-gravitino.md#unparsed-type)** that represents an unresolvable data type since 0.5.0.

:::note
Expand Down
Loading