diff --git a/bloviate-core/src/main/java/io/bloviate/db/Table.java b/bloviate-core/src/main/java/io/bloviate/db/Table.java index 48768e5..b09b111 100644 --- a/bloviate-core/src/main/java/io/bloviate/db/Table.java +++ b/bloviate-core/src/main/java/io/bloviate/db/Table.java @@ -98,16 +98,82 @@ public String insertString() { * @return a parameterized SQL INSERT statement string */ public String insertString(String identifierQuote) { + return insertString(identifierQuote, null); + } + + /** + * Generates an SQL INSERT statement template whose values may be SQL expressions rather than + * bare placeholders. + * + *

Identical to {@link #insertString(String)} except that each value slot is taken from + * {@code valueExpressions} instead of being a {@code ?}. This exists for columns whose type the + * driver cannot bind and which must therefore be constructed in SQL — BigQuery's + * {@code PARSE_JSON(?)} and {@code ST_GEOGFROMTEXT(?)}, for example. Expressions come from + * {@link io.bloviate.gen.DataGenerator#valueExpression()}, so the generator resolved for a + * column decides its own binding. + * + *

Each expression must contain exactly one {@code ?}. The engine binds one + * parameter per column by position, so an expression with none or several would shift every + * later parameter onto the wrong column — a silent data-corruption failure rather than an + * error — and is rejected here instead. + * + * @param identifierQuote the identifier quote string, as in {@link #insertString(String)} + * @param valueExpressions one expression per {@link #filteredColumns() filtered column}, in + * order; {@code null} means a bare {@code ?} for every column + * @return a parameterized SQL INSERT statement string + * @throws IllegalArgumentException if the list size does not match the filtered column count + * @throws IllegalStateException if any expression does not contain exactly one {@code ?} + * @since 3.2.0 + */ + public String insertString(String identifierQuote, List valueExpressions) { + List columns = filteredColumns(); + + if (valueExpressions != null && valueExpressions.size() != columns.size()) { + throw new IllegalArgumentException(String.format( + "table [%s] has %d fillable columns but %d value expressions were supplied", + name, columns.size(), valueExpressions.size())); + } + StringJoiner nameJoiner = new StringJoiner(","); StringJoiner valueJoiner = new StringJoiner(","); - for (Column column : filteredColumns()) { + for (int i = 0; i < columns.size(); i++) { + Column column = columns.get(i); nameJoiner.add(quote(column.name(), identifierQuote)); - valueJoiner.add("?"); + valueJoiner.add(valueExpressions == null + ? "?" + : validatedExpression(valueExpressions.get(i), column)); } return String.format("insert into %s (%s) values (%s)", qualifiedName(identifierQuote), nameJoiner, valueJoiner); + } + + /** + * Checks that a value expression binds exactly one parameter, so column {@code i}'s generator + * always writes to parameter {@code i + 1}. + */ + private static String validatedExpression(String expression, Column column) { + if (expression == null) { + throw new IllegalStateException(String.format( + "value expression for column [%s.%s] is null; it must contain exactly one '?'", + column.tableName(), column.name())); + } + + int placeholders = 0; + for (int i = 0; i < expression.length(); i++) { + if (expression.charAt(i) == '?') { + placeholders++; + } + } + + if (placeholders != 1) { + throw new IllegalStateException(String.format( + "value expression [%s] for column [%s.%s] contains %d '?' placeholders; it must contain " + + "exactly one, because the engine binds one parameter per column by position", + expression, column.tableName(), column.name(), placeholders)); + } + return expression; } private String qualifiedName(String identifierQuote) { diff --git a/bloviate-core/src/main/java/io/bloviate/db/TableFiller.java b/bloviate-core/src/main/java/io/bloviate/db/TableFiller.java index 3ac69de..288d318 100644 --- a/bloviate-core/src/main/java/io/bloviate/db/TableFiller.java +++ b/bloviate-core/src/main/java/io/bloviate/db/TableFiller.java @@ -30,6 +30,7 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; @@ -102,10 +103,6 @@ public TableFiller(Connection connection, Database database, DatabaseConfigurati @Override public void fill() throws SQLException { - String sql = table.insertString(connection.getMetaData().getIdentifierQuoteString()); - - logger.trace("{}", sql); - // The fill loop is the hot path: it runs once per cell (rowCount * columnCount times). // To keep it allocation- and lookup-free, everything is resolved up front into arrays // indexed by column position, so the inner loop only does positional array reads instead @@ -216,6 +213,19 @@ public void fill() throws SQLException { } } + // Built after generator resolution, not before: a generator decides how its value is bound, + // and a type the driver cannot bind directly must be constructed in SQL instead (BigQuery's + // PARSE_JSON(?) / ST_GEOGFROMTEXT(?)). Nearly always this is the same all-placeholder + // statement as before, since DataGenerator.valueExpression() defaults to "?". + List valueExpressions = new ArrayList<>(columnCount); + for (DataGenerator generator : generators) { + valueExpressions.add(generator.valueExpression()); + } + + String sql = table.insertString(connection.getMetaData().getIdentifierQuoteString(), valueExpressions); + + logger.trace("{}", sql); + int batchSize = databaseConfiguration.batchSize(); long totalRowCount = databaseConfiguration.defaultRowCount(); diff --git a/bloviate-core/src/main/java/io/bloviate/ext/BigQuerySupport.java b/bloviate-core/src/main/java/io/bloviate/ext/BigQuerySupport.java index ee7a2e3..fd5bbf6 100644 --- a/bloviate-core/src/main/java/io/bloviate/ext/BigQuerySupport.java +++ b/bloviate-core/src/main/java/io/bloviate/ext/BigQuerySupport.java @@ -20,7 +20,12 @@ import io.bloviate.db.Database; import io.bloviate.gen.BigDecimalGenerator; import io.bloviate.gen.ByteGenerator; +import io.bloviate.gen.DataGenerator; +import io.bloviate.gen.IntervalGenerator; +import io.bloviate.gen.JsonbGenerator; import io.bloviate.gen.SimpleStringGenerator; +import io.bloviate.gen.SqlExpressionGenerator; +import io.bloviate.gen.WktPointGenerator; import java.sql.Connection; import java.sql.JDBCType; @@ -30,7 +35,8 @@ /** * Google BigQuery-specific {@link DatabaseSupport}, written against the * tbc-bq-jdbc driver - * (vc.tbc:tbc-bq-jdbc, 4.3.0 or later). + * (vc.tbc:tbc-bq-jdbc, 4.3.0 or later; 4.4.0 or later is strongly recommended once + * released — see the value-expression note below). * *

BigQuery is an analytical engine, and it diverges from the OLTP databases Bloviate * otherwise targets in three ways this class has to account for: @@ -47,26 +53,30 @@ * {@code JSON}, {@code GEOGRAPHY} and {@code INTERVAL} back as {@link JDBCType#VARCHAR} and * {@code DATETIME} as {@link JDBCType#TIMESTAMP}, but has no parameter binding that produces * those types, and BigQuery will not implicitly coerce a {@code STRING}/{@code TIMESTAMP} - * parameter into them. Filling those columns needs SQL-side construction - * ({@code PARSE_JSON(?)}, {@code ST_GEOGFROMTEXT(?)}, {@code CAST(? AS DATETIME)}), which is - * not yet wired through the fill engine, so they are rejected here with an actionable - * message rather than generated and rejected by the server mid-batch. + * parameter into them. Their values are therefore generated as text and constructed by the + * server through a {@link io.bloviate.gen.DataGenerator#valueExpression() value expression} + * — see {@link #VALUE_EXPRESSIONS}. This works on any supported driver version, but + * is much faster from 4.4.0: earlier versions only collapse a batch whose {@code VALUES} + * tuple is placeholders-only, so a table with one of these columns degrades to one query job + * per row — correct, but slow enough to matter and expensive on a large fill. *

  • Keys are always {@code NOT ENFORCED}. BigQuery accepts declarative * {@code PRIMARY KEY}/{@code FOREIGN KEY} constraints but never enforces them, and the driver * surfaces them through {@code getPrimaryKeys}/{@code getImportedKeys}. Bloviate's FK-aware * ordering therefore works, and {@link #supportsBulkLoad()} is safe to enable.
    • * * - *

    Supported types: {@code STRING}, {@code BYTES}, {@code INT64}, - * {@code FLOAT64}, {@code NUMERIC}, {@code BIGNUMERIC} (within {@code NUMERIC} range), - * {@code BOOL}, {@code DATE}, {@code TIME}, {@code TIMESTAMP}. + *

    Supported types: every scalar type — {@code STRING}, {@code BYTES}, + * {@code INT64}, {@code FLOAT64}, {@code NUMERIC}, {@code BIGNUMERIC} (within {@code NUMERIC} + * range), {@code BOOL}, {@code DATE}, {@code TIME}, {@code TIMESTAMP}, {@code DATETIME}, + * {@code JSON}, {@code GEOGRAPHY} and {@code INTERVAL}. * - *

    Unsupported types: {@code DATETIME}, {@code JSON}, {@code GEOGRAPHY}, - * {@code INTERVAL}, {@code RANGE}, {@code ARRAY}, {@code STRUCT}. Supply a per-column generator - * through {@code ColumnConfiguration} or - * {@link GeneratorRegistry.Builder#registerColumnNamePattern} to fill them; the driver implements - * {@link Connection#createArrayOf} and {@link Connection#createStruct}, so a hand-written generator - * can write composites today. + *

    Unsupported types: the composite ones — {@code ARRAY}, {@code STRUCT} + * and {@code RANGE}. {@code ARRAY} and {@code STRUCT} need a shape Bloviate has no representation + * for; {@code RANGE} would need two parameters for one column, which the fill engine's + * one-parameter-per-column binding cannot express. Supply a per-column generator through + * {@code ColumnConfiguration} or {@link GeneratorRegistry.Builder#registerColumnNamePattern} to + * fill them; the driver implements {@link Connection#createArrayOf} and + * {@link Connection#createStruct}, so a hand-written generator can write composites today. * *

    Required driver settings: {@code includeStructFields=false} (the default) and * {@code metadataLazyLoad=false} (the default). With struct fields spliced in, {@code getColumns} @@ -112,24 +122,55 @@ public class BigQuerySupport extends AbstractDatabaseSupport { */ public static final int MAX_NUMERIC_SCALE = 9; + /** + * SQL that constructs a value for each type the driver cannot bind, keyed by BigQuery type name. + * + *

    These are the types whose {@code getColumns} shape is indistinguishable from a bindable one + * — {@code JSON}, {@code GEOGRAPHY} and {@code INTERVAL} all read back as + * {@link JDBCType#VARCHAR}, {@code DATETIME} as {@link JDBCType#TIMESTAMP} — but which + * have no parameter type of their own, and which BigQuery will not implicitly coerce into. The + * value is generated as text (or, for {@code DATETIME}, as a timestamp) and turned into the + * column's type by the server. + * + *

    Wrapping costs the batch collapse: tbc-bq-jdbc keeps a batch whose {@code VALUES} tuple is + * not placeholders-only off its NDJSON load-job path, since that path writes bound values + * directly and would drop the wrapping. The DML collapse itself still applies, but only from + * driver 4.4.0 — earlier versions require a placeholders-only tuple and fall back to one + * query job per row, which is correct but very slow. + */ + private static final Map VALUE_EXPRESSIONS = Map.of( + "JSON", "PARSE_JSON(?)", + "GEOGRAPHY", "ST_GEOGFROMTEXT(?)", + "INTERVAL", "CAST(? AS INTERVAL)", + "DATETIME", "CAST(? AS DATETIME)"); + /** Creates the BigQuery support with its default configuration. */ public BigQuerySupport() { } + /** + * Wraps a generator so its text is turned into {@code typeName} by the server. + * + * @param delegate the generator producing the value's text + * @param typeName the BigQuery type name, which must have a {@link #VALUE_EXPRESSIONS} entry + * @return the wrapped generator + */ + private static DataGenerator constructed(DataGenerator delegate, String typeName) { + return SqlExpressionGenerator.of(delegate, VALUE_EXPRESSIONS.get(typeName)); + } + @Override protected void configure(Map registry) { - // STRING, and the three types the driver reads back as VARCHAR but cannot bind. + // STRING, plus the three types the driver reads back as VARCHAR but cannot bind: their + // values are generated as text and constructed server-side (see VALUE_EXPRESSIONS). registry.put(JDBCType.VARCHAR, (column, random) -> switch (baseTypeName(column)) { case "STRING" -> new SimpleStringGenerator.Builder(random) .size(clamp(column.maxSize(), MAX_STRING_LENGTH)) .build(); - case "JSON" -> throw unsupported(column, - "the driver binds strings as STRING and BigQuery will not coerce STRING to JSON"); - case "GEOGRAPHY" -> throw unsupported(column, - "the driver binds strings as STRING and BigQuery will not coerce STRING to GEOGRAPHY"); - case "INTERVAL" -> throw unsupported(column, - "the driver binds strings as STRING and BigQuery will not coerce STRING to INTERVAL"); + case "JSON" -> constructed(new JsonbGenerator.Builder(random).build(), "JSON"); + case "GEOGRAPHY" -> constructed(new WktPointGenerator.Builder(random).build(), "GEOGRAPHY"); + case "INTERVAL" -> constructed(new IntervalGenerator.Builder(random).build(), "INTERVAL"); default -> throw unsupported(column, "no generator is registered for this type"); }); @@ -159,14 +200,13 @@ protected void configure(Map registry) { registry.put(JDBCType.NUMERIC, bigDecimal); registry.put(JDBCType.DECIMAL, bigDecimal); - // TIMESTAMP and DATETIME both arrive as JDBC TIMESTAMP; only TIMESTAMP can be bound. + // TIMESTAMP and DATETIME both arrive as JDBC TIMESTAMP. Only TIMESTAMP can be bound, so a + // DATETIME column takes the same generated instant and casts it server-side. The cast is + // interpreted in UTC, which is deterministic and therefore reproducible. GeneratorFactory inheritedTimestamp = registry.get(JDBCType.TIMESTAMP); registry.put(JDBCType.TIMESTAMP, (column, random) -> { - if ("DATETIME".equals(baseTypeName(column))) { - throw unsupported(column, "the driver has no DATETIME parameter type and BigQuery " - + "will not coerce TIMESTAMP to DATETIME"); - } - return inheritedTimestamp.create(column, random); + DataGenerator timestamp = inheritedTimestamp.create(column, random); + return "DATETIME".equals(baseTypeName(column)) ? constructed(timestamp, "DATETIME") : timestamp; }); // RANGE<...> is the only type the driver maps to OTHER. diff --git a/bloviate-core/src/main/java/io/bloviate/gen/DataGenerator.java b/bloviate-core/src/main/java/io/bloviate/gen/DataGenerator.java index 168e2ff..5846103 100644 --- a/bloviate-core/src/main/java/io/bloviate/gen/DataGenerator.java +++ b/bloviate-core/src/main/java/io/bloviate/gen/DataGenerator.java @@ -84,6 +84,37 @@ default boolean positionable() { return false; } + /** + * The SQL this generator's value occupies inside an {@code INSERT ... VALUES} tuple. + * + *

    The default {@code "?"} binds the generated value directly, which is what almost every + * generator wants. Override it when the value has to be constructed in SQL because the + * driver cannot bind the column's type — BigQuery's {@code JSON} and {@code GEOGRAPHY} + * have no parameter binding and the server will not coerce a {@code STRING} into them, so a + * generator emitting JSON text declares {@code "PARSE_JSON(?)"} and one emitting WKT declares + * {@code "ST_GEOGFROMTEXT(?)"}. + * + *

    This lives on the generator rather than on {@link io.bloviate.ext.DatabaseSupport} because + * the generator is what knows the shape of the text it produces. It also means a generator + * supplied through {@link io.bloviate.ext.GeneratorRegistry} or a + * {@link io.bloviate.ext.GeneratorPlugin} carries its own wrapping with no support involvement. + * + *

    Must contain exactly one {@code ?}. The fill engine binds one parameter + * per column by position, so any other count would silently misalign every subsequent + * parameter; {@link io.bloviate.db.Table#insertString(String, java.util.List)} rejects it rather + * than emit such a statement. To wrap an existing generator without subclassing it, use + * {@link SqlExpressionGenerator}. + * + *

    Note that a wrapped value may cost throughput: a driver that collapses a JDBC batch into a + * multi-row {@code INSERT} may decline to do so for a tuple that is not placeholders-only. + * + * @return the SQL expression for this value, containing exactly one {@code ?} + * @since 3.2.0 + */ + default String valueExpression() { + return "?"; + } + /** * Resets this generator's random source to the given seed for reproducible foreign-key * wraparound. Implementations must replace the underlying {@link java.util.random.RandomGenerator} diff --git a/bloviate-core/src/main/java/io/bloviate/gen/SqlExpressionGenerator.java b/bloviate-core/src/main/java/io/bloviate/gen/SqlExpressionGenerator.java new file mode 100644 index 0000000..cf9bb47 --- /dev/null +++ b/bloviate-core/src/main/java/io/bloviate/gen/SqlExpressionGenerator.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2021 Tim Veil + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.bloviate.gen; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Wraps another {@link DataGenerator} so its value is constructed by a SQL expression rather than + * bound as a bare placeholder, changing nothing else about it. + * + *

    This is how a shared generator is reused for a type whose value the driver cannot bind + * directly. {@link JsonbGenerator} already produces exactly the JSON text a BigQuery {@code JSON} + * column needs, but the driver binds it as a {@code STRING} and BigQuery will not coerce that into + * {@code JSON}; wrapping supplies the missing {@code PARSE_JSON(?)} without giving + * {@code JsonbGenerator} a BigQuery-specific opinion that would then apply to PostgreSQL too: + * + * 

    {@code
+ * SqlExpressionGenerator.of(new JsonbGenerator.Builder(random).build(), "PARSE_JSON(?)")
+ * }
    + * + *

    Use {@link #of(DataGenerator, String)} rather than the constructor. Whether a generator + * implements {@link IndexedDataGenerator} is load-bearing — the fill engine tests for it to + * decide how to position a partitioned fill — so the factory returns a variant that + * implements it exactly when the delegate does. A single class cannot do that, and getting it wrong + * in either direction breaks partitioned fills: claiming the interface without a seekable delegate + * silently skips repositioning, and dropping it from a seekable delegate replays draws instead of + * seeking. + * + * @param the Java type of values produced by the wrapped generator + * @since 3.2.0 + * @see DataGenerator#valueExpression() + */ +public class SqlExpressionGenerator implements DataGenerator { + + private final DataGenerator delegate; + private final String valueExpression; + + /** + * Wraps a generator so its value is written through the given SQL expression. + * + * @param delegate the generator producing the value; must not be null + * @param valueExpression the SQL expression, containing exactly one {@code ?} + * @param the generated value type + * @return a generator equivalent to {@code delegate} but reporting {@code valueExpression}, + * implementing {@link IndexedDataGenerator} exactly when {@code delegate} does + * @throws IllegalArgumentException if the expression does not contain exactly one {@code ?} + */ + public static DataGenerator of(DataGenerator delegate, String valueExpression) { + return delegate instanceof IndexedDataGenerator + ? new IndexedSqlExpressionGenerator<>(delegate, valueExpression) + : new SqlExpressionGenerator<>(delegate, valueExpression); + } + + /** + * Prefer {@link #of(DataGenerator, String)}, which preserves a seekable delegate's + * {@link IndexedDataGenerator} contract. + * + * @param delegate the generator producing the value + * @param valueExpression the SQL expression, containing exactly one {@code ?} + * @throws IllegalArgumentException if the expression does not contain exactly one {@code ?} + */ + protected SqlExpressionGenerator(DataGenerator delegate, String valueExpression) { + if (delegate == null) { + throw new IllegalArgumentException("delegate generator is required"); + } + this.delegate = delegate; + this.valueExpression = validate(valueExpression); + } + + private static String validate(String valueExpression) { + if (valueExpression == null) { + throw new IllegalArgumentException("value expression is required"); + } + int placeholders = 0; + for (int i = 0; i < valueExpression.length(); i++) { + if (valueExpression.charAt(i) == '?') { + placeholders++; + } + } + if (placeholders != 1) { + throw new IllegalArgumentException(String.format( + "value expression [%s] contains %d '?' placeholders; it must contain exactly one, " + + "because the engine binds one parameter per column by position", + valueExpression, placeholders)); + } + return valueExpression; + } + + /** @return the wrapped generator */ + protected final DataGenerator delegate() { + return delegate; + } + + @Override + public final String valueExpression() { + return valueExpression; + } + + @Override + public T generate() { + return delegate.generate(); + } + + @Override + public String generateAsString() { + return delegate.generateAsString(); + } + + /** + * {@inheritDoc} + * + *

    Delegated rather than assumed: wrapping changes only how the value reaches the column, so + * the delegate's positioning contract carries over unchanged. Answering for it would either + * skip repositioning a positionable column or reposition one that cannot survive it. + */ + @Override + public boolean positionable() { + return delegate.positionable(); + } + + @Override + public void reseed(long seed) { + delegate.reseed(seed); + } + + @Override + public void generateAndSet(Connection connection, PreparedStatement statement, int parameterIndex) + throws SQLException { + delegate.generateAndSet(connection, statement, parameterIndex); + } + + @Override + public void set(Connection connection, PreparedStatement statement, int parameterIndex, T value) + throws SQLException { + delegate.set(connection, statement, parameterIndex, value); + } + + @Override + public T get(ResultSet resultSet, int columnIndex) throws SQLException { + return delegate.get(resultSet, columnIndex); + } + + /** + * The variant returned by {@link #of} for a delegate that is itself an + * {@link IndexedDataGenerator}, so a partitioned fill still seeks the delegate's counter to the + * absolute row index instead of replaying draws. + * + * @param the generated value type + */ + private static final class IndexedSqlExpressionGenerator extends SqlExpressionGenerator + implements IndexedDataGenerator { + + private IndexedSqlExpressionGenerator(DataGenerator delegate, String valueExpression) { + super(delegate, valueExpression); + } + + @Override + public void seek(long rowIndex) { + ((IndexedDataGenerator) delegate()).seek(rowIndex); + } + } +} diff --git a/bloviate-core/src/main/java/io/bloviate/gen/WktPointGenerator.java b/bloviate-core/src/main/java/io/bloviate/gen/WktPointGenerator.java new file mode 100644 index 0000000..420e9fc --- /dev/null +++ b/bloviate-core/src/main/java/io/bloviate/gen/WktPointGenerator.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2021 Tim Veil + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.bloviate.gen; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Locale; +import java.util.random.RandomGenerator; + +/** + * Generator for geospatial point columns. Produces a + * Well-Known + * Text point literal — {@code POINT(longitude latitude)} — as a {@link String}. + * + *

    Coordinates are drawn uniformly over the whole globe: longitude in {@code [-180, 180]}, + * latitude in {@code [-90, 90]}, rounded to a configurable number of decimal places. Six places is + * roughly 0.1 m at the equator, which is finer than any generated data needs to be and keeps the + * literal short. + * + *

    WKT is the interchange form every geospatial engine accepts, but usually not as a directly + * bound parameter — BigQuery needs {@code ST_GEOGFROMTEXT(?)} and PostGIS + * {@code ST_GeomFromText(?)}. Pair this with {@link SqlExpressionGenerator} to supply that wrapping + * rather than binding the text straight into a geography column, which the server will reject. + * + *

    Longitude precedes latitude, per WKT (and unlike the "lat, long" convention of mapping UIs). + * + * @since 3.2.0 + */ +public class WktPointGenerator extends AbstractDataGenerator { + + private static final int MAX_LONGITUDE = 180; + private static final int MAX_LATITUDE = 90; + + private final int scale; + + @Override + public String generate() { + // draw order (longitude, then latitude) is part of the reproducibility contract + String longitude = coordinate(MAX_LONGITUDE); + String latitude = coordinate(MAX_LATITUDE); + + return "POINT(" + longitude + " " + latitude + ")"; + } + + private String coordinate(int bound) { + // drawn over [0, 2*bound] and shifted, because the shared random utility rejects a negative + // range; one draw either way, so the seeded sequence is unaffected + double value = randomUtils.nextDouble(0, 2.0 * bound) - bound; + return BigDecimal.valueOf(value).setScale(scale, RoundingMode.HALF_UP).toPlainString(); + } + + @Override + public String get(ResultSet resultSet, int columnIndex) throws SQLException { + return resultSet.getString(columnIndex); + } + + /** Fluent builder for {@link WktPointGenerator}. */ + public static class Builder extends AbstractBuilder { + + private int scale = 6; + + /** + * Creates a builder backed by the given seeded random source. + * + * @param random the random source used to draw generated values + */ + public Builder(RandomGenerator random) { + super(random); + } + + /** + * Sets how many decimal places each coordinate carries. Defaults to {@code 6}, about 0.1 m + * of resolution at the equator. + * + * @param scale the number of decimal places; must not be negative + * @return this builder, for chaining + * @throws IllegalArgumentException if {@code scale} is negative + */ + public Builder scale(int scale) { + if (scale < 0) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "scale cannot be negative: %d", scale)); + } + this.scale = scale; + return this; + } + + @Override + public WktPointGenerator build() { + return new WktPointGenerator(this); + } + } + + private WktPointGenerator(Builder builder) { + super(builder.random); + this.scale = builder.scale; + } +} diff --git a/bloviate-core/src/test/java/io/bloviate/db/BigQueryFillerTest.java b/bloviate-core/src/test/java/io/bloviate/db/BigQueryFillerTest.java index 472d2b2..cb16590 100644 --- a/bloviate-core/src/test/java/io/bloviate/db/BigQueryFillerTest.java +++ b/bloviate-core/src/test/java/io/bloviate/db/BigQueryFillerTest.java @@ -164,5 +164,22 @@ private static void verify(Connection connection, String dataset, String runId) // BIGNUMERIC is bound as NUMERIC, so its values must sit inside NUMERIC's range assertCount(connection, String.format( "select count(*) from %s where abs(c_bignumeric) >= 1e29", standard), 0); + + // The four server-constructed types actually landed as their declared type rather than as + // text. Each of these functions only accepts the real type, so a row that arrived as a + // STRING could not have been written at all -- but assert non-null so a silently skipped + // column cannot pass either. + assertCount(connection, String.format( + "select count(*) from %s where c_datetime is null or c_json is null " + + "or c_geography is null or c_interval is null", standard), 0); + assertCount(connection, String.format( + "select count(*) from %s where extract(year from c_datetime) is null", standard), 0); + assertCount(connection, String.format( + "select count(*) from %s where st_x(c_geography) not between -180 and 180", standard), 0); + assertCount(connection, String.format( + "select count(*) from %s where st_y(c_geography) not between -90 and 90", standard), 0); + // JSON_TYPE errors on a non-JSON argument, so reaching a count at all proves the column + assertCount(connection, String.format( + "select count(*) from %s where json_type(c_json) <> 'object'", standard), 0); } } diff --git a/bloviate-core/src/test/java/io/bloviate/db/TableTest.java b/bloviate-core/src/test/java/io/bloviate/db/TableTest.java index c8f532f..98330c2 100644 --- a/bloviate-core/src/test/java/io/bloviate/db/TableTest.java +++ b/bloviate-core/src/test/java/io/bloviate/db/TableTest.java @@ -19,9 +19,12 @@ import org.junit.jupiter.api.Test; import java.sql.JDBCType; +import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class TableTest { @@ -46,6 +49,61 @@ void noArgInsertStringStaysUnqualifiedWhenSchemaPresent() { assertEquals("insert into orders (id) values (?)", table.insertString()); } + @Test + void insertStringSubstitutesValueExpressions() { + Table table = new Table("events", null, + List.of(column("id", null, false), column("payload", null, false)), List.of()); + + assertEquals("insert into `events` (`id`,`payload`) values (?,PARSE_JSON(?))", + table.insertString("`", List.of("?", "PARSE_JSON(?)"))); + } + + @Test + void insertStringWithNullExpressionsMatchesThePlainForm() { + // the overload is what the engine always calls now, so its no-expression behavior must be + // byte-identical to the form it replaced + Table table = new Table("orders", null, + List.of(column("id", "public", false), column("qty", null, false)), List.of()); + + assertEquals(table.insertString("\""), table.insertString("\"", null)); + } + + @Test + void insertStringRejectsAnExpressionWithoutExactlyOnePlaceholder() { + // one parameter is bound per column by position, so any other count shifts every later + // column's value onto the wrong parameter -- silent corruption rather than an error + Table table = new Table("events", null, + List.of(column("id", null, false), column("payload", null, false)), List.of()); + + for (String bad : List.of("PARSE_JSON('x')", "RANGE(?, ?)")) { + IllegalStateException e = assertThrows(IllegalStateException.class, + () -> table.insertString("`", List.of("?", bad))); + assertTrue(e.getMessage().contains("payload"), e.getMessage()); + } + + assertThrows(IllegalStateException.class, + () -> table.insertString("`", Arrays.asList("?", null))); + } + + @Test + void insertStringRejectsAMismatchedExpressionCount() { + Table table = new Table("events", null, + List.of(column("id", null, false), column("payload", null, false)), List.of()); + + assertThrows(IllegalArgumentException.class, () -> table.insertString("`", List.of("?"))); + } + + @Test + void insertStringCountsExpressionsAgainstFilteredColumns() { + // auto-increment columns are excluded from the INSERT, so expressions align with the + // filtered list rather than every column + Table table = new Table("orders", null, + List.of(column("id", null, true), column("qty", null, false)), List.of()); + + assertEquals("insert into `orders` (`qty`) values (CAST(? AS INT64))", + table.insertString("`", List.of("CAST(? AS INT64)"))); + } + @Test void insertStringQuotesAndSchemaQualifiesIdentifiers() { Table table = new Table("order", null, diff --git a/bloviate-core/src/test/java/io/bloviate/ext/BigQuerySupportTest.java b/bloviate-core/src/test/java/io/bloviate/ext/BigQuerySupportTest.java index 2a59e70..0bb2112 100644 --- a/bloviate-core/src/test/java/io/bloviate/ext/BigQuerySupportTest.java +++ b/bloviate-core/src/test/java/io/bloviate/ext/BigQuerySupportTest.java @@ -16,6 +16,7 @@ package io.bloviate.ext; +import com.fasterxml.jackson.databind.ObjectMapper; import io.bloviate.db.Column; import io.bloviate.gen.BigDecimalGenerator; import io.bloviate.gen.BooleanGenerator; @@ -31,10 +32,12 @@ import java.math.BigDecimal; import java.sql.JDBCType; +import java.sql.Timestamp; import java.util.LinkedHashMap; import java.util.Map; import java.util.Random; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -189,14 +192,6 @@ void toleratesUnreportedNumericPrecisionAndScale() { // ------------------------------------------------------------------------ rejected types - @Test - void rejectsDatetimeWhichIsIndistinguishableFromTimestampByJdbcTypeAlone() { - UnsupportedOperationException e = rejectionFor(JDBCType.TIMESTAMP, "DATETIME"); - - assertTrue(e.getMessage().contains("DATETIME")); - assertTrue(e.getMessage().contains("ColumnConfiguration")); - } - @Test void rejectsTypesWithNoParameterBinding() { Map rejected = new LinkedHashMap<>(); @@ -204,12 +199,75 @@ void rejectsTypesWithNoParameterBinding() { rejected.put(JDBCType.ARRAY, "ARRAY"); rejected.put(JDBCType.STRUCT, "STRUCT"); - for (String typeName : new String[]{"JSON", "GEOGRAPHY", "INTERVAL"}) { - assertRejected(JDBCType.VARCHAR, typeName); - } rejected.forEach(this::assertRejected); } + // ------------------------------------------------------- server-constructed (Phase 2) types + + @Test + void constructsJsonServerSide() { + // the driver binds text as STRING and BigQuery will not coerce that into JSON, so the + // value has to be built by the server + DataGenerator generator = generatorFor(JDBCType.VARCHAR, null, "JSON"); + + assertEquals("PARSE_JSON(?)", generator.valueExpression()); + assertDoesNotThrow(() -> new ObjectMapper().readTree((String) generator.generate())); + } + + @Test + void constructsGeographyFromWellKnownText() { + DataGenerator generator = generatorFor(JDBCType.VARCHAR, null, "GEOGRAPHY"); + + assertEquals("ST_GEOGFROMTEXT(?)", generator.valueExpression()); + + for (int i = 0; i < DRAWS; i++) { + String wkt = (String) generator.generate(); + assertTrue(wkt.startsWith("POINT(") && wkt.endsWith(")"), wkt); + + // WKT orders coordinates longitude-first, so the wider bound comes first + String[] coordinates = wkt.substring(6, wkt.length() - 1).split(" "); + assertEquals(2, coordinates.length, wkt); + assertTrue(Math.abs(Double.parseDouble(coordinates[0])) <= 180, wkt); + assertTrue(Math.abs(Double.parseDouble(coordinates[1])) <= 90, wkt); + } + } + + @Test + void constructsIntervalServerSide() { + DataGenerator generator = generatorFor(JDBCType.VARCHAR, null, "INTERVAL"); + + assertEquals("CAST(? AS INTERVAL)", generator.valueExpression()); + + // BigQuery's interval literal is Y-M D H:M:S, which IntervalGenerator already emits. + // Drawn once and reused, so a failure reports the value that actually failed rather than + // a fresh (possibly valid) one. + String interval = (String) generator.generate(); + assertTrue(interval.matches("-?\\d+-\\d+ -?\\d+ \\d+:\\d+:\\d+"), interval); + } + + @Test + void constructsDatetimeFromTheTimestampGenerator() { + // DATETIME and TIMESTAMP are indistinguishable by JDBC type, so the type name decides; + // both draw the same instant and only DATETIME is cast server-side + DataGenerator datetime = generatorFor(JDBCType.TIMESTAMP, null, "DATETIME"); + DataGenerator timestamp = generatorFor(JDBCType.TIMESTAMP, null, "TIMESTAMP"); + + assertEquals("CAST(? AS DATETIME)", datetime.valueExpression()); + assertEquals("?", timestamp.valueExpression()); + assertInstanceOf(Timestamp.class, datetime.generate()); + } + + @Test + void leavesEveryOtherTypeBoundDirectly() { + // a wrapped tuple costs the driver's load-job path, so nothing should be wrapped that does + // not have to be + assertEquals("?", generatorFor(JDBCType.VARCHAR, 20, "STRING(20)").valueExpression()); + assertEquals("?", generatorFor(JDBCType.VARBINARY, null, "BYTES").valueExpression()); + assertEquals("?", generatorFor(JDBCType.NUMERIC, 38, 9, "NUMERIC").valueExpression()); + assertEquals("?", generatorFor(JDBCType.BIGINT, null, "INT64").valueExpression()); + assertEquals("?", generatorFor(JDBCType.DATE, null, "DATE").valueExpression()); + } + private void assertRejected(JDBCType jdbcType, String typeName) { UnsupportedOperationException e = rejectionFor(jdbcType, typeName); diff --git a/bloviate-core/src/test/java/io/bloviate/gen/SqlExpressionGeneratorTest.java b/bloviate-core/src/test/java/io/bloviate/gen/SqlExpressionGeneratorTest.java new file mode 100644 index 0000000..92ec78a --- /dev/null +++ b/bloviate-core/src/test/java/io/bloviate/gen/SqlExpressionGeneratorTest.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2021 Tim Veil + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.bloviate.gen; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SqlExpressionGeneratorTest { + + private static DataGenerator wrapped(String expression) { + return SqlExpressionGenerator.of(new JsonbGenerator.Builder(new Random(1)).build(), expression); + } + + @Test + void reportsTheExpressionAndDelegatesTheValue() { + DataGenerator wrapped = wrapped("PARSE_JSON(?)"); + + assertEquals("PARSE_JSON(?)", wrapped.valueExpression()); + assertTrue(wrapped.generate().startsWith("{")); + } + + @Test + void wrappingDoesNotChangeGeneratedValues() { + // same seed, one wrapped and one not: wrapping affects how the value reaches the column, + // never the value itself, so seed reproducibility must survive it + DataGenerator plain = new JsonbGenerator.Builder(new Random(42)).build(); + DataGenerator wrapped = SqlExpressionGenerator.of( + new JsonbGenerator.Builder(new Random(42)).build(), "PARSE_JSON(?)"); + + for (int i = 0; i < 20; i++) { + assertEquals(plain.generate(), wrapped.generate()); + } + } + + @Test + void rejectsAnExpressionThatWouldMisalignParameters() { + // the engine binds one parameter per column by position, so anything other than exactly one + // placeholder shifts every later column's value onto the wrong parameter + assertThrows(IllegalArgumentException.class, () -> wrapped("PARSE_JSON('x')")); + assertThrows(IllegalArgumentException.class, () -> wrapped("RANGE(?, ?)")); + assertThrows(IllegalArgumentException.class, () -> wrapped(null)); + assertThrows(IllegalArgumentException.class, + () -> SqlExpressionGenerator.of(null, "PARSE_JSON(?)")); + } + + @Test + void delegatesPositionable() { + // answering for the delegate would either skip repositioning a positionable column or + // reposition one that cannot survive it + assertTrue(wrapped("PARSE_JSON(?)").positionable()); + assertFalse(SqlExpressionGenerator.of(new NotPositionable(), "CAST(? AS DATETIME)").positionable()); + } + + @Test + void delegatesReseed() { + DataGenerator wrapped = wrapped("PARSE_JSON(?)"); + String first = wrapped.generate(); + + wrapped.reseed(99); + String afterReseed = wrapped.generate(); + wrapped.reseed(99); + + assertNotEquals(first, afterReseed); + assertEquals(afterReseed, wrapped.generate()); + } + + // ------------------------------------------------------------------ IndexedDataGenerator + + @Test + void keepsAnIndexedDelegateSeekable() { + // TableFiller tests `instanceof IndexedDataGenerator` to decide how to position a + // partitioned fill, so dropping the interface here would silently replay draws instead + DataGenerator wrapped = SqlExpressionGenerator.of(new Counter(), "CAST(? AS INT64)"); + + assertInstanceOf(IndexedDataGenerator.class, wrapped); + + ((IndexedDataGenerator) wrapped).seek(7); + assertEquals(7, wrapped.generate()); + assertEquals(8, wrapped.generate()); + } + + @Test + void doesNotClaimSeekabilityForAPlainDelegate() { + // the mirror-image failure: claiming the interface without a seekable delegate makes the + // engine skip the repositioning that column actually needs + assertFalse(SqlExpressionGenerator.of(new NotPositionable(), "CAST(? AS DATETIME)") + instanceof IndexedDataGenerator); + } + + @Test + void aSeekedIndexedDelegateMatchesTheSequentialRun() { + // the property partitioned fills depend on: seeking to row N produces exactly what the + // sequential run produces at row N + DataGenerator sequential = SqlExpressionGenerator.of(new Counter(), "CAST(? AS INT64)"); + List expected = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + expected.add(sequential.generate()); + } + + DataGenerator partitioned = SqlExpressionGenerator.of(new Counter(), "CAST(? AS INT64)"); + ((IndexedDataGenerator) partitioned).seek(5); + + for (int i = 5; i < 10; i++) { + assertEquals(expected.get(i), partitioned.generate()); + } + } + + /** A minimal indexed generator: its value is its absolute row index. */ + private static final class Counter extends AbstractDataGenerator implements IndexedDataGenerator { + + private long row; + + private Counter() { + super(new Random(1)); + } + + @Override + public Integer generate() { + return (int) row++; + } + + @Override + public void seek(long rowIndex) { + this.row = rowIndex; + } + + @Override + public Integer get(java.sql.ResultSet resultSet, int columnIndex) { + throw new UnsupportedOperationException(); + } + } + + /** A generator that opts out of positioning, as the datafaker integration does. */ + private static final class NotPositionable extends AbstractDataGenerator { + + private NotPositionable() { + super(new Random(1)); + } + + @Override + public boolean positionable() { + return false; + } + + @Override + public String generate() { + return "x"; + } + + @Override + public String get(java.sql.ResultSet resultSet, int columnIndex) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/bloviate-core/src/test/resources/create_tables.bigquery.sql b/bloviate-core/src/test/resources/create_tables.bigquery.sql index e22d8b9..c177fd1 100644 --- a/bloviate-core/src/test/resources/create_tables.bigquery.sql +++ b/bloviate-core/src/test/resources/create_tables.bigquery.sql @@ -4,9 +4,11 @@ -- concurrent runs never collide). Every table carries a 2-hour expiration so a failed run cannot -- leave billable tables behind. -- --- Only natively bindable types appear here. DatabaseFiller fills every table in the dataset, so a --- single DATETIME/JSON/GEOGRAPHY/ARRAY/STRUCT column would fail the whole fill; rejection of those --- is covered by BigQuerySupportTest instead. +-- Every scalar type appears here, including the four the driver cannot bind directly +-- (DATETIME, JSON, GEOGRAPHY, INTERVAL), which are constructed server-side from generated text. +-- The composite types (ARRAY, STRUCT, RANGE) are deliberately absent: DatabaseFiller fills every +-- table in the dataset, so one unsupported column would fail the whole fill. Their rejection is +-- covered by BigQuerySupportTest instead. -- -- Keys are declared NOT ENFORCED, which is the only form BigQuery accepts. They are never enforced -- at write time, but the driver surfaces them through getPrimaryKeys/getImportedKeys, which is what @@ -43,7 +45,13 @@ CREATE TABLE `${dataset}.standard_types_${suffix}` ( c_bytes_sized BYTES(16), c_date DATE, c_time TIME, - c_timestamp TIMESTAMP + c_timestamp TIMESTAMP, + -- constructed server-side: none of these has a parameter binding, and BigQuery will not + -- coerce a STRING or TIMESTAMP parameter into them + c_datetime DATETIME, + c_json JSON, + c_geography GEOGRAPHY, + c_interval INTERVAL ) OPTIONS(expiration_timestamp = TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 2 HOUR)); -- Any real BigQuery table is partitioned and clustered; this proves the batch/DML path works diff --git a/docs/DATABASE_SUPPORT.md b/docs/DATABASE_SUPPORT.md index 9f6e74b..48a8617 100644 --- a/docs/DATABASE_SUPPORT.md +++ b/docs/DATABASE_SUPPORT.md @@ -12,7 +12,7 @@ explicitly, or let Bloviate detect it from the connection. | MariaDB | `MariaDBSupport` | Extends `MySQLSupport` (MariaDB speaks the MySQL wire protocol) | | H2 | `H2Support` | Standard JDBC types **plus** `UUID` and `JSON` (embedded; no Docker) | | SQLite | `SQLiteSupport` | Standard JDBC types via type affinity (embedded; no Docker) | -| BigQuery | `BigQuerySupport` | Scalar types only — see [BigQuery](#bigquery) for what is excluded | +| BigQuery | `BigQuerySupport` | All scalar types **incl.** `JSON`, `GEOGRAPHY`, `INTERVAL`, `DATETIME`; no composites | | Generic JDBC | `DefaultSupport` | Standard JDBC types only | All of them resolve the cross-database defaults for the common JDBC types (integers, decimals, @@ -49,28 +49,63 @@ graph regardless. Requires the [tbc-bq-jdbc](https://github.com/Two-Bear-Capital/tbc-bq-jdbc) driver (`vc.tbc:tbc-bq-jdbc`, 4.3.0 or later), which is not yet on Maven Central — install it locally with -`./mvnw clean install` in that repo, or use its GitHub Releases jar. +`./mvnw clean install` in that repo, or use its GitHub Releases jar. Version 4.4.0 is strongly +recommended once it is released, for the reason given under [server-constructed +values](#server-constructed-values). ```java String url = "jdbc:bigquery:my-project/my_dataset?authType=ADC"; ``` -**Supported:** `STRING`, `BYTES`, `INT64`, `FLOAT64`, `NUMERIC`, `BIGNUMERIC`, `BOOL`, `DATE`, -`TIME`, `TIMESTAMP`. +**Supported:** every scalar type — `STRING`, `BYTES`, `INT64`, `FLOAT64`, `NUMERIC`, `BIGNUMERIC`, +`BOOL`, `DATE`, `TIME`, `TIMESTAMP`, `DATETIME`, `JSON`, `GEOGRAPHY`, `INTERVAL`. -**Not supported:** `DATETIME`, `JSON`, `GEOGRAPHY`, `INTERVAL`, `RANGE`, `ARRAY`, `STRUCT`. These -fail fast with a message naming the column, before any rows are written. - -The reason is that BigQuery's *write* surface is narrower than its read surface. The driver reads -`JSON`/`GEOGRAPHY`/`INTERVAL` back as `VARCHAR` and `DATETIME` as `TIMESTAMP`, but has no parameter -binding that produces those types, and BigQuery will not implicitly coerce a `STRING` or `TIMESTAMP` -parameter into them. Filling them requires SQL-side construction (`PARSE_JSON(?)`, -`ST_GEOGFROMTEXT(?)`, `CAST(? AS DATETIME)`), which the fill engine does not yet emit. +**Not supported:** the composite types — `ARRAY`, `STRUCT`, `RANGE`. These fail fast with a message +naming the column, before any rows are written. `ARRAY` and `STRUCT` need a shape Bloviate has no +representation for, and `RANGE` would need two parameters for one column, which the engine's +one-parameter-per-column binding cannot express. To fill one anyway, supply a per-column generator through `ColumnConfiguration` or `GeneratorRegistry.registerColumnNamePattern`. The driver does implement `Connection.createArrayOf` and `Connection.createStruct`, so a hand-written generator can write composites today. +### Server-constructed values + +BigQuery's *write* surface is narrower than its read surface. The driver reads `JSON`, `GEOGRAPHY` +and `INTERVAL` back as `VARCHAR` and `DATETIME` as `TIMESTAMP`, but has no parameter binding that +produces any of them, and BigQuery will not implicitly coerce a `STRING` or `TIMESTAMP` parameter +into them. + +Those four are therefore generated as text and turned into the column's type by the server: + +| Type | Generated as | Written as | +|------|--------------|------------| +| `JSON` | a JSON object literal | `PARSE_JSON(?)` | +| `GEOGRAPHY` | a WKT point, `POINT(lon lat)` | `ST_GEOGFROMTEXT(?)` | +| `INTERVAL` | `Y-M D H:M:S` | `CAST(? AS INTERVAL)` | +| `DATETIME` | a timestamp | `CAST(? AS DATETIME)` (interpreted as UTC) | + +This is why 4.4.0 matters. It is not required — these columns fill correctly on 4.3.0 — but earlier +versions collapse a JDBC batch into a multi-row `INSERT` only when the `VALUES` tuple is +placeholders-only, so a table with any of these columns falls back to **one query job per row**. +That is correct, but slow enough to matter and expensive on a large fill. + +At the time of writing 4.4.0 is merged but unreleased, so `tbc-bq-jdbc.version` still pins 4.3.0. + +Two further consequences: + +- A table containing one of these columns never takes the driver's NDJSON load-job path, even above + `batchLoadThreshold`. That path writes bound values directly and never sees the SQL, so it would + drop the wrapping and store the wrong thing; the driver keeps such batches on DML deliberately. +- The mechanism is general, not BigQuery-specific: any generator can declare its own + `DataGenerator.valueExpression()`, and `SqlExpressionGenerator` wraps an existing generator + without subclassing it. A custom PostGIS generator can use the same seam. + +`BIGNUMERIC` is *not* in that table. It has no parameter binding either, but unlike the four above +it does not need one: the driver binds every `BigDecimal` as `NUMERIC`, and a `NUMERIC`-range value +is always valid in a `BIGNUMERIC` column. Values are therefore clamped rather than constructed — +see below. + > `GeneratorRegistry.registerTypeName` is a poor fit here: it matches type names exactly, and > BigQuery reports the raw `INFORMATION_SCHEMA` text (`string(20)`, `numeric(10, 2)`, > `array`), so only unparameterized names ever match. @@ -97,6 +132,11 @@ binds every `BigDecimal` as `NUMERIC`, so generated values are clamped to `NUMER binding is what forces this, not the destination column — a `NUMERIC`-range value is always valid in a `BIGNUMERIC` column. +That clamp is a deliberate limit rather than a gap to close. `BigDecimalGenerator` already caps +itself at 25 significant digits on every database, on the grounds that enormous precision is not +useful test data (CockroachDB reports 131,089), so generating true 76-digit `BIGNUMERIC` values +would contradict that. If you need them, supply a per-column generator. + ### Required driver settings Both are the driver's defaults; overriding either breaks the fill. @@ -132,6 +172,11 @@ auto-commit to stay on, which is why `BigQuerySupport` opts out of engine-manage `CommitStrategy` at its default** — each `executeBatch` is already one atomic job, so an explicit strategy buys nothing and costs the load path. Bloviate logs a warning if you set one anyway. +The load path is also unavailable to any table holding a `JSON`, `GEOGRAPHY`, `INTERVAL` or +`DATETIME` column, for the reason given under [server-constructed +values](#server-constructed-values). Such tables still collapse into multi-row `INSERT` statements +on driver 4.4.0 and later; they simply stay on DML. + > Filling a BigQuery dataset writes real data to real storage and runs real jobs. Both cost money. > **PostgreSQL connection requirement:** the vendor types above are bound as their text diff --git a/pom.xml b/pom.xml index fd22d21..2767707 100644 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,12 @@ green without it. To run the BigQuery integration test, either `./mvnw clean install` the driver repo or drop its GitHub Releases jar into ~/.m2. Dependabot and versions:display cannot resolve this coordinate; bump it by hand. + + 4.3.0 is the functional floor. 4.4.0 collapses a batch whose VALUES tuple wraps a + placeholder (PARSE_JSON(?), CAST(? AS DATETIME)), which BigQuerySupport emits for JSON, + GEOGRAPHY, INTERVAL and DATETIME columns; without it those tables still fill correctly, + just one query job per row. 4.4.0 is merged but unreleased, so this stays at 4.3.0; + bump it once 4.4.0 ships. --> 4.3.0