Skip to content

Add IMPORT/EXPORT support for expression indexes #4455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
tinaspark wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Add IMPORT/EXPORT support for expression indexes #4455

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion v1/src/main/java/com/google/cloud/teleport/spanner/ddl/IndexColumn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ public abstract class IndexColumn implements Serializable {
@Nullable
public abstract NullsOrder nullsOrder();

@Nullable
public abstract String expression();
Comment thread
tinaspark marked this conversation as resolved.

public static IndexColumn create(String name, Order order, Dialect dialect) {
return new AutoValue_IndexColumn.Builder().dialect(dialect).name(name).order(order).autoBuild();
}
Expand Down Expand Up @@ -79,7 +82,12 @@ public enum NullsOrder {
}

public void prettyPrint(Appendable appendable) throws IOException {
appendable.append(quoteIdentifier(name(), dialect())).append(" ").append(order().title);
if (expression() == null) {
appendable.append(quoteIdentifier(name(), dialect()));
} else {
appendable.append("(").append(expression()).append(")");
}
appendable.append(" ").append(order().title);
if (nullsOrder() != null) {
appendable.append(" NULLS ").append(nullsOrder().title);
}
Expand Down Expand Up @@ -112,6 +120,8 @@ public abstract static class Builder {

abstract Builder nullsOrder(NullsOrder nullsOrder);

abstract Builder expression(String expression);

abstract IndexColumn autoBuild();
}

Expand Down Expand Up @@ -222,6 +232,15 @@ public IndexColumnsBuilder<T> nullsLast() {
return this;
}

public IndexColumnsBuilder<T> expression(String expression) {
if (indexColumnBuilder == null) {
throw new IllegalArgumentException(
"Builder is missing. Call create method to initiate a builder first.");
}
indexColumnBuilder.expression(expression);
return this;
}

public ImmutableList<IndexColumn> build() {
return columns.build();
}
Expand Down
10 changes: 7 additions & 3 deletions v1/src/main/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,7 @@ private void listIndexColumns(
String indexLocalName = resultSet.getString(4);
String indexType = resultSet.getString(5);
String spannerType = resultSet.getString(6);
String expression = resultSet.getString(7);

if (indexLocalName.equals("PRIMARY_KEY")) {
IndexColumn.IndexColumnsBuilder<Table.Builder> pkBuilder =
Expand Down Expand Up @@ -599,7 +600,10 @@ private void listIndexColumns(
}
IndexColumn.IndexColumnsBuilder<Index.Builder> indexColumnsBuilder =
indexBuilder.columns().create().name(columnName);
boolean isStoring = resultSet.isNull(7);
if (expression != null) {
indexColumnsBuilder.expression(expression);
}
boolean isStoring = resultSet.isNull(8);
if (isStoring) {
indexColumnsBuilder.storing();
} else if (ordering == null) {
Expand Down Expand Up @@ -631,15 +635,15 @@ Statement listIndexColumnsSQL() {
case GOOGLE_STANDARD_SQL:
return Statement.of(
"SELECT t.table_schema, t.table_name, t.column_name, t.column_ordering, t.index_name,"
+ " t.index_type, t.spanner_type, t.ordinal_position "
+ " t.index_type, t.spanner_type, t.expression, t.ordinal_position "
+ "FROM information_schema.index_columns AS t "
Comment thread
tinaspark marked this conversation as resolved.
+ " WHERE t.table_schema NOT IN"
+ " ('INFORMATION_SCHEMA', 'SPANNER_SYS')"
+ "ORDER BY t.table_name, t.index_name, t.ordinal_position");
case POSTGRESQL:
return Statement.of(
"SELECT t.table_schema, t.table_name, t.column_name, t.column_ordering, t.index_name,"
+ " t.index_type, t.spanner_type, t.ordinal_position "
+ " t.index_type, t.spanner_type, t.expression, t.ordinal_position "
+ "FROM information_schema.index_columns AS t "
Comment thread
tinaspark marked this conversation as resolved.
+ "WHERE t.table_schema NOT IN "
+ "('information_schema', 'spanner_sys', 'pg_catalog') "
Expand Down
7 changes: 7 additions & 0 deletions v1/src/test/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,8 @@ public void simple() {
+ " \"spannerIndex_0\" : "
+ " \"CREATE INDEX `UsersByFirstName` ON `Users` (`first_name`)\","
+ " \"spannerIndex_1\" : "
+ " \"CREATE INDEX `UsersExpressionIndex` ON `Users` (concat(`first_name`, `last_name`))\","
+ " \"spannerIndex_2\" : "
+ " \"CREATE VECTOR INDEX `VI` ON `Users` (`Embeddings`) WHERE Embeddings IS NOT NULL OPTIONS (distance_type=\'COSINE\')\","
+ " \"spannerForeignKey_0\" : "
+ " \"ALTER TABLE `Users` ADD CONSTRAINT `fk` FOREIGN KEY (`first_name`) "
Expand Down Expand Up @@ -311,6 +313,7 @@ public void simple() {
+ " CONSTRAINT `ck` CHECK(`first_name` != 'last_name'),"
+ " ) PRIMARY KEY (`id` ASC, `gen_id` ASC, `last_name` DESC)"
+ " CREATE INDEX `UsersByFirstName` ON `Users` (`first_name`)"
+ " CREATE INDEX `UsersExpressionIndex` ON `Users` (concat(`first_name`, `last_name`))"
+ " CREATE VECTOR INDEX `VI` ON `Users` (`Embeddings`) WHERE Embeddings IS NOT NULL"
+ " OPTIONS (distance_type=\'COSINE\')"
+ " ALTER TABLE `Users` ADD CONSTRAINT `fk`"
Expand Down Expand Up @@ -536,6 +539,9 @@ public void pgSimple() {
+ " : \"\\\"last_name\\\" ASC\", \"spannerIndex_0\" :"
+ " \"CREATE INDEX \\\"UsersByFirstName\\\" ON \\\"Users\\\" (\\\"first_name\\\")\", "
+ " \"spannerIndex_1\" : "
+ " \"CREATE INDEX \\\"UsersExpressionIndex\\\" ON \\\"Users\\\" (concat("
+ "\\\"first_name\\\", \\\"last_name\\\"))\", "
+ " \"spannerIndex_2\" : "
+ " \"CREATE INDEX \\\"VI\\\" ON \\\"Users\\\" USING ScaNN (\\\"Embeddings\\\") "
+ "WHERE \\\"Embeddings\\\" IS NOT NULL OPTIONS (distance_type=\'COSINE\')\","
+ " \"spannerForeignKey_0\" : \"ALTER TABLE \\\"Users\\\" ADD CONSTRAINT \\\"fk\\\""
Expand Down Expand Up @@ -602,6 +608,7 @@ public void pgSimple() {
+ " PRIMARY KEY (\"id\", \"gen_id\", \"last_name\")"
+ " )"
+ " CREATE INDEX \"UsersByFirstName\" ON \"Users\" (\"first_name\")"
+ " CREATE INDEX \"UsersExpressionIndex\" ON \"Users\" (concat(\"first_name\", \"last_name\"))"
+ " CREATE INDEX \"VI\" ON \"Users\" USING ScaNN (\"Embeddings\") WHERE \"Embeddings\" IS NOT NULL OPTIONS (distance_type=\'COSINE\')"
+ " ALTER TABLE \"Users\" ADD CONSTRAINT \"fk\" FOREIGN KEY (\"first_name\")"
+ " REFERENCES \"AllowedNames\" (\"first_name\")"
Expand Down
4 changes: 2 additions & 2 deletions v1/src/test/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScannerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ public void testListIndexColumnsSQL() {
googleSQLInfoScanner.listIndexColumnsSQL().getSql(),
equalToCompressingWhiteSpace(
"SELECT t.table_schema, t.table_name, t.column_name, t.column_ordering, t.index_name, "
+ "t.index_type, t.spanner_type, t.ordinal_position "
+ "t.index_type, t.spanner_type, t.expression, t.ordinal_position "
+ "FROM information_schema.index_columns AS t "
+ " WHERE t.table_schema NOT IN"
+ " ('INFORMATION_SCHEMA', 'SPANNER_SYS')"
Expand All @@ -195,7 +195,7 @@ public void testListIndexColumnsSQL() {
postgresSQLInfoScanner.listIndexColumnsSQL().getSql(),
equalToCompressingWhiteSpace(
"SELECT t.table_schema, t.table_name, t.column_name, t.column_ordering, t.index_name, "
+ "t.index_type, t.spanner_type, t.ordinal_position "
+ "t.index_type, t.spanner_type, t.expression, t.ordinal_position "
+ "FROM information_schema.index_columns AS t "
+ "WHERE t.table_schema NOT IN "
+ "('information_schema', 'spanner_sys', 'pg_catalog') "
Expand Down
Loading