diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java b/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java index f5161b12f76..f2ad3254589 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java @@ -44,6 +44,7 @@ public class PostgreSqlTypeConverter extends JdbcTypeConverter { @VisibleForTesting static final String ARRAY_TOKEN = "[]"; @VisibleForTesting static final int DEFAULT_NUMERIC_PRECISION = 38; @VisibleForTesting static final int DEFAULT_NUMERIC_SCALE = 18; + @VisibleForTesting static final int MAX_TIMESTAMP_PRECISION = 6; @Override public Type toGravitino(JdbcTypeBean typeBean) { @@ -127,6 +128,12 @@ public String fromGravitino(Type type) { return type.simpleString(); } else if (type instanceof Types.TimestampType) { Types.TimestampType timestampType = (Types.TimestampType) type; + if (timestampType.hasPrecisionSet() && timestampType.precision() > MAX_TIMESTAMP_PRECISION) { + throw new IllegalArgumentException( + String.format( + "PostgreSQL supports timestamp precision up to %d, but cannot convert Gravitino type %s", + MAX_TIMESTAMP_PRECISION, type.simpleString())); + } String baseType = timestampType.hasTimeZone() ? TIMESTAMP_TZ : TIMESTAMP; return timestampType.hasPrecisionSet() ? String.format("%s(%d)", baseType, timestampType.precision()) @@ -146,6 +153,22 @@ public String fromGravitino(Type type) { return BYTEA; } else if (type instanceof Types.UUIDType) { return UUID; + } else if (type instanceof Types.NullType) { + throw new IllegalArgumentException( + "PostgreSQL table columns cannot represent Gravitino Unknown (NullType); cannot convert Gravitino type null"); + } else if (type instanceof Types.VariantType) { + throw new IllegalArgumentException( + "PostgreSQL JSON and JSONB do not preserve Gravitino Variant semantics; cannot convert Gravitino type variant"); + } else if (type instanceof Types.GeometryType) { + throw new IllegalArgumentException( + String.format( + "PostgreSQL PostGIS geometry does not preserve Gravitino Geometry CRS semantics; cannot convert Gravitino type %s", + type.simpleString())); + } else if (type instanceof Types.GeographyType) { + throw new IllegalArgumentException( + String.format( + "PostgreSQL PostGIS geography does not preserve Gravitino Geography CRS and edge-algorithm semantics; cannot convert Gravitino type %s", + type.simpleString())); } else if (type instanceof Types.ListType) { return fromGravitinoArrayType((ListType) type); } else if (type instanceof Types.ExternalType) { diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java index e4139441b79..a4532949579 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java @@ -35,6 +35,7 @@ import static org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.INT_4; import static org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.INT_8; import static org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.JDBC_ARRAY_PREFIX; +import static org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.MAX_TIMESTAMP_PRECISION; import static org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.NUMERIC; import static org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.UUID; @@ -91,6 +92,11 @@ public void testToGravitinoType() { checkJdbcTypeToGravitinoType(Types.UUIDType.get(), UUID, null, null, 0); checkJdbcTypeToGravitinoType( Types.ExternalType.of(USER_DEFINED_TYPE), USER_DEFINED_TYPE, null, null, 0); + checkJdbcTypeToGravitinoType(Types.ExternalType.of("json"), "json", null, null, 0); + checkJdbcTypeToGravitinoType(Types.ExternalType.of("jsonb"), "jsonb", null, null, 0); + checkJdbcTypeToGravitinoType(Types.ExternalType.of("unknown"), "unknown", null, null, 0); + checkJdbcTypeToGravitinoType(Types.ExternalType.of("geometry"), "geometry", null, null, 0); + checkJdbcTypeToGravitinoType(Types.ExternalType.of("geography"), "geography", null, null, 0); } @Test @@ -137,6 +143,80 @@ public void testFromGravitinoType() { () -> POSTGRE_SQL_TYPE_CONVERTER.fromGravitino(Types.UnparsedType.of(USER_DEFINED_TYPE))); } + @Test + public void testRejectNanosecondTimestampTypes() { + Type[] nanosecondTypes = { + Types.TimestampType.withoutTimeZone(9), Types.TimestampType.withTimeZone(9) + }; + + for (Type type : nanosecondTypes) { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, () -> POSTGRE_SQL_TYPE_CONVERTER.fromGravitino(type)); + Assertions.assertTrue( + exception + .getMessage() + .contains( + "PostgreSQL supports timestamp precision up to " + MAX_TIMESTAMP_PRECISION)); + } + } + + @Test + public void testRejectVariantType() { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> POSTGRE_SQL_TYPE_CONVERTER.fromGravitino(Types.VariantType.get())); + + Assertions.assertTrue( + exception + .getMessage() + .contains("PostgreSQL JSON and JSONB do not preserve Gravitino Variant semantics")); + } + + @Test + public void testRejectUnknownType() { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> POSTGRE_SQL_TYPE_CONVERTER.fromGravitino(Types.NullType.get())); + + Assertions.assertTrue( + exception + .getMessage() + .contains("PostgreSQL table columns cannot represent Gravitino Unknown (NullType)")); + } + + @Test + public void testRejectGeometryType() { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> POSTGRE_SQL_TYPE_CONVERTER.fromGravitino(Types.GeometryType.of("EPSG:3857"))); + + Assertions.assertTrue( + exception + .getMessage() + .contains( + "PostgreSQL PostGIS geometry does not preserve Gravitino Geometry CRS semantics")); + } + + @Test + public void testRejectGeographyType() { + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + POSTGRE_SQL_TYPE_CONVERTER.fromGravitino( + Types.GeographyType.of("EPSG:4326", "karney"))); + + Assertions.assertTrue( + exception + .getMessage() + .contains( + "PostgreSQL PostGIS geography does not preserve Gravitino Geography CRS and edge-algorithm semantics")); + } + protected void checkGravitinoTypeToJdbcType(String jdbcTypeName, Type gravitinoType) { Assertions.assertEquals(jdbcTypeName, POSTGRE_SQL_TYPE_CONVERTER.fromGravitino(gravitinoType)); } diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java index 68216e674b2..e0a75d45fe5 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java @@ -73,6 +73,7 @@ import org.apache.gravitino.rel.indexes.Index; import org.apache.gravitino.rel.indexes.Indexes; import org.apache.gravitino.rel.types.Decimal; +import org.apache.gravitino.rel.types.Type; import org.apache.gravitino.rel.types.Types; import org.apache.gravitino.rel.types.Types.IntegerType; import org.apache.gravitino.utils.RandomNameUtils; @@ -1973,4 +1974,64 @@ void testTimeTypePrecision() { } } } + + @Test + void testRejectNanosecondTimestampTypesWithoutSideEffects() { + assertCreateRejectedWithoutSideEffects( + "timestamp_ns", Types.TimestampType.withoutTimeZone(9), "timestamp precision up to 6"); + assertCreateRejectedWithoutSideEffects( + "timestamptz_ns", Types.TimestampType.withTimeZone(9), "timestamp precision up to 6"); + } + + @Test + void testRejectVariantWithoutSideEffects() { + assertCreateRejectedWithoutSideEffects( + "variant", + Types.VariantType.get(), + "PostgreSQL JSON and JSONB do not preserve Gravitino Variant semantics"); + } + + @Test + void testRejectUnknownWithoutSideEffects() { + assertCreateRejectedWithoutSideEffects( + "unknown", + Types.NullType.get(), + "PostgreSQL table columns cannot represent Gravitino Unknown (NullType)"); + } + + @Test + void testRejectGeometryWithoutSideEffects() { + assertCreateRejectedWithoutSideEffects( + "geometry", + Types.GeometryType.of("EPSG:3857"), + "PostgreSQL PostGIS geometry does not preserve Gravitino Geometry CRS semantics"); + } + + @Test + void testRejectGeographyWithoutSideEffects() { + assertCreateRejectedWithoutSideEffects( + "geography", + Types.GeographyType.of("EPSG:4326", "karney"), + "PostgreSQL PostGIS geography does not preserve Gravitino Geography CRS and edge-algorithm semantics"); + } + + private void assertCreateRejectedWithoutSideEffects( + String typeName, Type type, String expectedMessage) { + String rejectedTableName = GravitinoITUtils.genRandomName("rejected_" + typeName); + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, rejectedTableName); + TableCatalog tableCatalog = catalog.asTableCatalog(); + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + tableCatalog.createTable( + tableIdentifier, + new Column[] {Column.of("unsupported_col", type)}, + null, + ImmutableMap.of())); + + Assertions.assertTrue(exception.getMessage().contains(expectedMessage)); + Assertions.assertFalse(tableCatalog.tableExists(tableIdentifier)); + } } diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperationsSqlGeneration.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperationsSqlGeneration.java index 12f2e1834e0..c56222426b5 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperationsSqlGeneration.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperationsSqlGeneration.java @@ -89,4 +89,94 @@ public void testCreateTableWithNonEmptyStringDefaultValue() { sql.contains("DEFAULT " + converter.fromGravitino(col1.defaultValue())), "Should contain DEFAULT value but was: " + sql); } + + @Test + public void testCreateTableRejectsNanosecondTimestampTypesBeforeSqlGeneration() { + TestablePostgreSqlTableOperations ops = new TestablePostgreSqlTableOperations(); + Types.TimestampType[] nanosecondTypes = { + Types.TimestampType.withoutTimeZone(9), Types.TimestampType.withTimeZone(9) + }; + + for (Types.TimestampType type : nanosecondTypes) { + JdbcColumn column = JdbcColumn.builder().withName("nanosecond_col").withType(type).build(); + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> ops.createTableSql("test_table", new JdbcColumn[] {column})); + Assertions.assertTrue( + exception.getMessage().contains("PostgreSQL supports timestamp precision up to 6")); + } + } + + @Test + public void testCreateTableRejectsVariantBeforeSqlGeneration() { + TestablePostgreSqlTableOperations ops = new TestablePostgreSqlTableOperations(); + JdbcColumn column = + JdbcColumn.builder().withName("variant_col").withType(Types.VariantType.get()).build(); + + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> ops.createTableSql("test_table", new JdbcColumn[] {column})); + Assertions.assertTrue( + exception + .getMessage() + .contains("PostgreSQL JSON and JSONB do not preserve Gravitino Variant semantics")); + } + + @Test + public void testCreateTableRejectsUnknownBeforeSqlGeneration() { + TestablePostgreSqlTableOperations ops = new TestablePostgreSqlTableOperations(); + JdbcColumn column = + JdbcColumn.builder().withName("unknown_col").withType(Types.NullType.get()).build(); + + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> ops.createTableSql("test_table", new JdbcColumn[] {column})); + Assertions.assertTrue( + exception + .getMessage() + .contains("PostgreSQL table columns cannot represent Gravitino Unknown (NullType)")); + } + + @Test + public void testCreateTableRejectsGeometryBeforeSqlGeneration() { + TestablePostgreSqlTableOperations ops = new TestablePostgreSqlTableOperations(); + JdbcColumn column = + JdbcColumn.builder() + .withName("geometry_col") + .withType(Types.GeometryType.of("EPSG:3857")) + .build(); + + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> ops.createTableSql("test_table", new JdbcColumn[] {column})); + Assertions.assertTrue( + exception + .getMessage() + .contains( + "PostgreSQL PostGIS geometry does not preserve Gravitino Geometry CRS semantics")); + } + + @Test + public void testCreateTableRejectsGeographyBeforeSqlGeneration() { + TestablePostgreSqlTableOperations ops = new TestablePostgreSqlTableOperations(); + JdbcColumn column = + JdbcColumn.builder() + .withName("geography_col") + .withType(Types.GeographyType.of("EPSG:4326", "karney")) + .build(); + + IllegalArgumentException exception = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> ops.createTableSql("test_table", new JdbcColumn[] {column})); + Assertions.assertTrue( + exception + .getMessage() + .contains( + "PostgreSQL PostGIS geography does not preserve Gravitino Geography CRS and edge-algorithm semantics")); + } } diff --git a/docs/jdbc-postgresql-catalog.md b/docs/jdbc-postgresql-catalog.md index 8235f1b0693..a21de41c4a6 100644 --- a/docs/jdbc-postgresql-catalog.md +++ b/docs/jdbc-postgresql-catalog.md @@ -115,6 +115,16 @@ Refer to [Manage Relational Metadata Using Gravitino](./manage-relational-metada | `UUID` | `Uuid` | | `List` | `Array` | +#### V3 Type Compatibility + +| Gravitino Type | PostgreSQL outcome | +|-------------------------------------|-------------------------------------------------------------------------------------------------------------------------| +| `Timestamp(9)` / `Timestamp_tz(9)` | Rejected before DDL. PostgreSQL only supports precision 0 through 6 and cannot preserve nanosecond timestamp precision. | +| `Variant` | Rejected before DDL. Native `json` and `jsonb` columns load as `External` because they do not preserve Variant semantics. | +| `Unknown` | Rejected before DDL. PostgreSQL table columns cannot represent an optional, null-only Unknown logical type. | +| `Geometry` | Rejected before DDL. Native PostGIS `geometry` columns load as `External`; exact CRS semantics are not translated. | +| `Geography` | Rejected before DDL. Native PostGIS `geography` columns load as `External`; exact CRS and edge-algorithm semantics are not translated. | + :::info PostgreSQL doesn't support Gravitino `Fixed` `Struct` `Map` `IntervalDay` `IntervalYear` `Union` type. Meanwhile, the data types other than listed above are mapped to Gravitino **[External Type](./manage-relational-metadata-using-gravitino.md#external-type)** that represents an unresolvable data type since 0.6.0-incubating.