diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java index 08a2cb4612ff..413b0ac1ffa8 100644 --- a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java @@ -20,7 +20,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; -import com.google.common.io.BaseEncoding; import java.io.IOException; import java.math.BigDecimal; import java.nio.channels.Channels; @@ -37,11 +36,13 @@ import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.TimeMicroVector; import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorLoader; import org.apache.arrow.vector.VectorSchemaRoot; @@ -355,18 +356,20 @@ static FieldValue arrowVectorToFieldValue( } // Handle primitive types - String stringVal; if (type == LegacySQLTypeName.TIMESTAMP) { TimeStampVector tsVector = (TimeStampVector) vector; - stringVal = formatTimestampMicros(tsVector.get(rowIndex)); + return FieldValue.ofTimestampMicros(tsVector.get(rowIndex)); } else if (type == LegacySQLTypeName.DATE) { + String stringVal; if (vector instanceof DateDayVector) { int days = ((DateDayVector) vector).get(rowIndex); stringVal = LocalDate.ofEpochDay(days).toString(); } else { stringVal = String.valueOf(vector.getObject(rowIndex)); } + return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal); } else if (type == LegacySQLTypeName.TIME) { + String stringVal; if (vector instanceof TimeMicroVector) { long microsOfDay = ((TimeMicroVector) vector).get(rowIndex); stringVal = @@ -374,28 +377,33 @@ static FieldValue arrowVectorToFieldValue( } else { stringVal = String.valueOf(vector.getObject(rowIndex)); } + return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal); } else if (vector instanceof VarCharVector) { - stringVal = new String(((VarCharVector) vector).get(rowIndex), StandardCharsets.UTF_8); + String stringVal = new String(((VarCharVector) vector).get(rowIndex), StandardCharsets.UTF_8); + return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal); } else if (vector instanceof IntVector) { - stringVal = Integer.toString(((IntVector) vector).get(rowIndex)); + return FieldValue.ofLong(((IntVector) vector).get(rowIndex)); } else if (vector instanceof BigIntVector) { - stringVal = Long.toString(((BigIntVector) vector).get(rowIndex)); + return FieldValue.ofLong(((BigIntVector) vector).get(rowIndex)); } else if (vector instanceof BitVector) { - stringVal = ((BitVector) vector).get(rowIndex) == 1 ? "true" : "false"; + String stringVal = ((BitVector) vector).get(rowIndex) == 1 ? "true" : "false"; + return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal); } else if (vector instanceof Float8Vector) { - stringVal = Double.toString(((Float8Vector) vector).get(rowIndex)); + return FieldValue.ofDouble(((Float8Vector) vector).get(rowIndex)); + } else if (vector instanceof VarBinaryVector) { + return FieldValue.ofBytes(((VarBinaryVector) vector).get(rowIndex)); + } else if (vector instanceof DecimalVector) { + return FieldValue.ofNumeric(((DecimalVector) vector).getObject(rowIndex)); } else { Object value = vector.getObject(rowIndex); if (value instanceof byte[]) { - stringVal = BaseEncoding.base64().encode((byte[]) value); + return FieldValue.ofBytes((byte[]) value); } else if (value instanceof BigDecimal) { - stringVal = ((BigDecimal) value).toPlainString(); + return FieldValue.ofNumeric((BigDecimal) value); } else { - stringVal = String.valueOf(value); + return FieldValue.of(FieldValue.Attribute.PRIMITIVE, String.valueOf(value)); } } - - return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal); } /** diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java index d11df4b956af..25817e906dcf 100644 --- a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java @@ -50,7 +50,9 @@ public class FieldValue implements Serializable { private static final long serialVersionUID = 469098630191710062L; private final Attribute attribute; - private final Object value; + private Object value; + private final Object rawValue; + private final boolean isTimestampMicros; private final Boolean useInt64Timestamps; /** The field value's attribute, giving information on the field's content type. */ @@ -80,11 +82,61 @@ private FieldValue(Attribute attribute, Object value) { } private FieldValue(Attribute attribute, Object value, Boolean useInt64Timestamps) { + this(attribute, value, null, false, useInt64Timestamps); + } + + private FieldValue( + Attribute attribute, + Object value, + Object rawValue, + boolean isTimestampMicros, + Boolean useInt64Timestamps) { this.attribute = checkNotNull(attribute); this.value = value; + this.rawValue = rawValue; + this.isTimestampMicros = isTimestampMicros; this.useInt64Timestamps = useInt64Timestamps; } + static FieldValue ofLong(long value) { + return new FieldValue(Attribute.PRIMITIVE, null, Long.valueOf(value), false, false); + } + + static FieldValue ofDouble(double value) { + return new FieldValue(Attribute.PRIMITIVE, null, Double.valueOf(value), false, false); + } + + static FieldValue ofTimestampMicros(long micros) { + return new FieldValue(Attribute.PRIMITIVE, null, Long.valueOf(micros), true, false); + } + + static FieldValue ofNumeric(BigDecimal value) { + return new FieldValue(Attribute.PRIMITIVE, null, checkNotNull(value), false, false); + } + + static FieldValue ofBytes(byte[] value) { + return new FieldValue(Attribute.PRIMITIVE, null, checkNotNull(value), false, false); + } + + private String formatRawValue() { + if (rawValue instanceof Long) { + long longVal = (Long) rawValue; + return isTimestampMicros + ? ArrowDeserializer.formatTimestampMicros(longVal) + : Long.toString(longVal); + } + if (rawValue instanceof Double) { + return Double.toString((Double) rawValue); + } + if (rawValue instanceof BigDecimal) { + return ((BigDecimal) rawValue).toPlainString(); + } + if (rawValue instanceof byte[]) { + return BaseEncoding.base64().encode((byte[]) rawValue); + } + return String.valueOf(rawValue); + } + /** * Returns the attribute of this Field Value. * @@ -103,7 +155,7 @@ public Attribute getAttribute() { /** Returns {@code true} if this field's value is {@code null}, {@code false} otherwise. */ public boolean isNull() { - return value == null; + return value == null && rawValue == null; } /** @@ -111,7 +163,12 @@ public boolean isNull() { * method returns {@code null}. */ public Object getValue() { - return value; + Object localValue = value; + if (localValue == null && rawValue != null) { + localValue = formatRawValue(); + value = localValue; + } + return localValue; } public Boolean getUseInt64Timestamps() { @@ -130,8 +187,9 @@ public Boolean getUseInt64Timestamps() { */ @SuppressWarnings("unchecked") public String getStringValue() { - checkNotNull(value); - return (String) value; + Object val = getValue(); + checkNotNull(val); + return (String) val; } /** @@ -157,6 +215,9 @@ public String getStringValueOrDefault(String defaultValue) { * @throws IllegalStateException if the field value is not encoded in base64 */ public byte[] getBytesValue() { + if (rawValue instanceof byte[]) { + return ((byte[]) rawValue).clone(); + } try { return BaseEncoding.base64().decode(getStringValue()); } catch (IllegalArgumentException ex) { @@ -174,6 +235,9 @@ public byte[] getBytesValue() { */ @SuppressWarnings("unchecked") public long getLongValue() { + if (rawValue instanceof Long && !isTimestampMicros) { + return (Long) rawValue; + } return Long.parseLong(getStringValue()); } @@ -187,6 +251,12 @@ public long getLongValue() { */ @SuppressWarnings("unchecked") public double getDoubleValue() { + if (rawValue instanceof Double) { + return (Double) rawValue; + } + if (rawValue instanceof Long && !isTimestampMicros) { + return ((Long) rawValue).doubleValue(); + } return Double.parseDouble(getStringValue()); } @@ -218,6 +288,9 @@ public boolean getBooleanValue() { */ @SuppressWarnings("unchecked") public long getTimestampValue() { + if (rawValue instanceof Long && isTimestampMicros) { + return (Long) rawValue; + } if (useInt64Timestamps) { return new BigInteger(getStringValue()).longValue(); } @@ -242,7 +315,7 @@ public long getTimestampValue() { */ @SuppressWarnings("unchecked") public Instant getTimestampInstant() { - checkNotNull(value); + checkNotNull(rawValue != null ? rawValue : value); return Instant.EPOCH.plus(getTimestampValue(), MICROS); } @@ -257,6 +330,9 @@ public Instant getTimestampInstant() { */ @SuppressWarnings("unchecked") public BigDecimal getNumericValue() { + if (rawValue instanceof BigDecimal) { + return (BigDecimal) rawValue; + } return new BigDecimal(getStringValue()); } @@ -330,14 +406,14 @@ public FieldValueList getRecordValue() { public String toString() { return MoreObjects.toStringHelper(this) .add("attribute", attribute) - .add("value", value) + .add("value", getValue()) .add("useInt64Timestamps", useInt64Timestamps) .toString(); } @Override public final int hashCode() { - return Objects.hash(attribute, value, useInt64Timestamps); + return Objects.hash(attribute, getValue(), useInt64Timestamps); } @Override @@ -350,7 +426,7 @@ public final boolean equals(Object obj) { } FieldValue other = (FieldValue) obj; return attribute == other.attribute - && Objects.equals(value, other.value) + && Objects.equals(getValue(), other.getValue()) && Objects.equals(useInt64Timestamps, other.useInt64Timestamps); } diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ArrowDeserializerTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ArrowDeserializerTest.java index f39c8a0c0dfe..e8c77a2445b2 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ArrowDeserializerTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ArrowDeserializerTest.java @@ -729,11 +729,13 @@ public void testArrowVectorToFieldValue_primitives() { FieldValue int64Val = ArrowDeserializer.arrowVectorToFieldValue( int64Vector, 0, LegacySQLTypeName.INTEGER, Field.Mode.NULLABLE, null, "int64_col"); + assertEquals(9876543210L, int64Val.getLongValue()); assertEquals("9876543210", int64Val.getStringValue()); FieldValue int32Val = ArrowDeserializer.arrowVectorToFieldValue( int32Vector, 0, LegacySQLTypeName.INTEGER, Field.Mode.NULLABLE, null, "int32_col"); + assertEquals(42L, int32Val.getLongValue()); assertEquals("42", int32Val.getStringValue()); FieldValue strVal = @@ -744,11 +746,13 @@ public void testArrowVectorToFieldValue_primitives() { FieldValue boolVal = ArrowDeserializer.arrowVectorToFieldValue( boolVector, 0, LegacySQLTypeName.BOOLEAN, Field.Mode.NULLABLE, null, "bool_col"); + assertTrue(boolVal.getBooleanValue()); assertEquals("true", boolVal.getStringValue()); FieldValue floatVal = ArrowDeserializer.arrowVectorToFieldValue( floatVector, 0, LegacySQLTypeName.FLOAT, Field.Mode.NULLABLE, null, "float_col"); + assertEquals(3.14159, floatVal.getDoubleValue(), 0.0); assertEquals("3.14159", floatVal.getStringValue()); FieldValue dateVal = @@ -764,11 +768,14 @@ public void testArrowVectorToFieldValue_primitives() { FieldValue tsVal = ArrowDeserializer.arrowVectorToFieldValue( tsVector, 0, LegacySQLTypeName.TIMESTAMP, Field.Mode.NULLABLE, null, "ts_col"); + assertEquals(1408452095220000L, tsVal.getTimestampValue()); assertEquals("1408452095.220000", tsVal.getStringValue()); FieldValue bytesVal = ArrowDeserializer.arrowVectorToFieldValue( bytesVector, 0, LegacySQLTypeName.BYTES, Field.Mode.NULLABLE, null, "bytes_col"); + org.junit.jupiter.api.Assertions.assertArrayEquals( + "raw_data".getBytes(StandardCharsets.UTF_8), bytesVal.getBytesValue()); assertEquals( BaseEncoding.base64().encode("raw_data".getBytes(StandardCharsets.UTF_8)), bytesVal.getStringValue()); @@ -776,6 +783,7 @@ public void testArrowVectorToFieldValue_primitives() { FieldValue decVal = ArrowDeserializer.arrowVectorToFieldValue( decVector, 0, LegacySQLTypeName.NUMERIC, Field.Mode.NULLABLE, null, "num_col"); + assertEquals(new BigDecimal("123.456000000"), decVal.getNumericValue()); assertEquals("123.456000000", decVal.getStringValue()); } } diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java index e6b08ec6b8a3..9cbaf63d9b4e 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java @@ -234,4 +234,41 @@ public void testParseCanonicalInterval() { System.out.println(FieldValue.parseCanonicalInterval(entry.getKey())); } } + + @Test + public void testTypedFieldValuesAndLazyStringMaterialization() { + FieldValue longVal = FieldValue.ofLong(9876543210L); + assertFalse(longVal.isNull()); + assertEquals(9876543210L, longVal.getLongValue()); + assertEquals(9876543210.0, longVal.getDoubleValue(), 0.0); + assertEquals("9876543210", longVal.getStringValue()); + assertEquals(FieldValue.of(FieldValue.Attribute.PRIMITIVE, "9876543210"), longVal); + assertEquals( + FieldValue.of(FieldValue.Attribute.PRIMITIVE, "9876543210").hashCode(), longVal.hashCode()); + + FieldValue doubleVal = FieldValue.ofDouble(3.14159); + assertFalse(doubleVal.isNull()); + assertEquals(3.14159, doubleVal.getDoubleValue(), 0.0); + assertEquals("3.14159", doubleVal.getStringValue()); + assertEquals(FieldValue.of(FieldValue.Attribute.PRIMITIVE, "3.14159"), doubleVal); + + FieldValue tsVal = FieldValue.ofTimestampMicros(1408452095220000L); + assertFalse(tsVal.isNull()); + assertEquals(1408452095220000L, tsVal.getTimestampValue()); + assertEquals("1408452095.220000", tsVal.getStringValue()); + assertEquals(FieldValue.of(FieldValue.Attribute.PRIMITIVE, "1408452095.220000"), tsVal); + + BigDecimal decimal = new BigDecimal("123456789.123456789"); + FieldValue numVal = FieldValue.ofNumeric(decimal); + assertFalse(numVal.isNull()); + assertEquals(decimal, numVal.getNumericValue()); + assertEquals("123456789.123456789", numVal.getStringValue()); + assertEquals(FieldValue.of(FieldValue.Attribute.PRIMITIVE, "123456789.123456789"), numVal); + + FieldValue bytesVal = FieldValue.ofBytes(BYTES); + assertFalse(bytesVal.isNull()); + assertArrayEquals(BYTES, bytesVal.getBytesValue()); + assertEquals(BYTES_BASE64, bytesVal.getStringValue()); + assertEquals(FieldValue.of(FieldValue.Attribute.PRIMITIVE, BYTES_BASE64), bytesVal); + } }