Skip to content

perf(bigquery): [T2-3] support typed FieldValue backing to avoid binary-to-String-to-binary round-trips in ArrowDeserializer #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lqiu96 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

perf(bigquery): [T2-3] support typed FieldValue backing to avoid binary-to-String-to-binary round-trips in ArrowDeserializer #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 21 additions & 13 deletions ...uery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.io.BaseEncoding;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.channels.Channels;
Expand All @@ -37,11 +36,13 @@
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.DateDayVector;
import org.apache.arrow.vector.DecimalVector;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.Float8Vector;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.TimeMicroVector;
import org.apache.arrow.vector.TimeStampVector;
import org.apache.arrow.vector.VarBinaryVector;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorLoader;
import org.apache.arrow.vector.VectorSchemaRoot;
Expand Down Expand Up @@ -355,47 +356,54 @@ static FieldValue arrowVectorToFieldValue(
}

// Handle primitive types
String stringVal;
if (type == LegacySQLTypeName.TIMESTAMP) {
TimeStampVector tsVector = (TimeStampVector) vector;
stringVal = formatTimestampMicros(tsVector.get(rowIndex));
return FieldValue.ofTimestampMicros(tsVector.get(rowIndex));
} else if (type == LegacySQLTypeName.DATE) {
String stringVal;
if (vector instanceof DateDayVector) {
int days = ((DateDayVector) vector).get(rowIndex);
stringVal = LocalDate.ofEpochDay(days).toString();
} else {
stringVal = String.valueOf(vector.getObject(rowIndex));
}
return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal);
} else if (type == LegacySQLTypeName.TIME) {
String stringVal;
if (vector instanceof TimeMicroVector) {
long microsOfDay = ((TimeMicroVector) vector).get(rowIndex);
stringVal =
DateTimeFormatter.ISO_LOCAL_TIME.format(LocalTime.ofNanoOfDay(microsOfDay * 1_000L));
} else {
stringVal = String.valueOf(vector.getObject(rowIndex));
}
return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal);
} else if (vector instanceof VarCharVector) {
stringVal = new String(((VarCharVector) vector).get(rowIndex), StandardCharsets.UTF_8);
String stringVal = new String(((VarCharVector) vector).get(rowIndex), StandardCharsets.UTF_8);
return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal);
} else if (vector instanceof IntVector) {
stringVal = Integer.toString(((IntVector) vector).get(rowIndex));
return FieldValue.ofLong(((IntVector) vector).get(rowIndex));
} else if (vector instanceof BigIntVector) {
stringVal = Long.toString(((BigIntVector) vector).get(rowIndex));
return FieldValue.ofLong(((BigIntVector) vector).get(rowIndex));
} else if (vector instanceof BitVector) {
stringVal = ((BitVector) vector).get(rowIndex) == 1 ? "true" : "false";
String stringVal = ((BitVector) vector).get(rowIndex) == 1 ? "true" : "false";
return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal);
} else if (vector instanceof Float8Vector) {
stringVal = Double.toString(((Float8Vector) vector).get(rowIndex));
return FieldValue.ofDouble(((Float8Vector) vector).get(rowIndex));
} else if (vector instanceof VarBinaryVector) {
return FieldValue.ofBytes(((VarBinaryVector) vector).get(rowIndex));
} else if (vector instanceof DecimalVector) {
return FieldValue.ofNumeric(((DecimalVector) vector).getObject(rowIndex));
} else {
Object value = vector.getObject(rowIndex);
if (value instanceof byte[]) {
stringVal = BaseEncoding.base64().encode((byte[]) value);
return FieldValue.ofBytes((byte[]) value);
} else if (value instanceof BigDecimal) {
stringVal = ((BigDecimal) value).toPlainString();
return FieldValue.ofNumeric((BigDecimal) value);
} else {
stringVal = String.valueOf(value);
return FieldValue.of(FieldValue.Attribute.PRIMITIVE, String.valueOf(value));
}
}

return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal);
}

/**
Expand Down
94 changes: 85 additions & 9 deletions java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ public class FieldValue implements Serializable {
private static final long serialVersionUID = 469098630191710062L;

private final Attribute attribute;
private final Object value;
private Object value;
private final Object rawValue;
private final boolean isTimestampMicros;
private final Boolean useInt64Timestamps;

/** The field value's attribute, giving information on the field's content type. */
Expand Down Expand Up @@ -80,11 +82,61 @@ private FieldValue(Attribute attribute, Object value) {
}

private FieldValue(Attribute attribute, Object value, Boolean useInt64Timestamps) {
this(attribute, value, null, false, useInt64Timestamps);
}

private FieldValue(
Attribute attribute,
Object value,
Object rawValue,
boolean isTimestampMicros,
Boolean useInt64Timestamps) {
this.attribute = checkNotNull(attribute);
this.value = value;
this.rawValue = rawValue;
this.isTimestampMicros = isTimestampMicros;
this.useInt64Timestamps = useInt64Timestamps;
}

static FieldValue ofLong(long value) {
return new FieldValue(Attribute.PRIMITIVE, null, Long.valueOf(value), false, false);
}

static FieldValue ofDouble(double value) {
return new FieldValue(Attribute.PRIMITIVE, null, Double.valueOf(value), false, false);
}

static FieldValue ofTimestampMicros(long micros) {
return new FieldValue(Attribute.PRIMITIVE, null, Long.valueOf(micros), true, false);
}

static FieldValue ofNumeric(BigDecimal value) {
return new FieldValue(Attribute.PRIMITIVE, null, checkNotNull(value), false, false);
}

static FieldValue ofBytes(byte[] value) {
return new FieldValue(Attribute.PRIMITIVE, null, checkNotNull(value), false, false);
}

private String formatRawValue() {
if (rawValue instanceof Long) {
long longVal = (Long) rawValue;
return isTimestampMicros
? ArrowDeserializer.formatTimestampMicros(longVal)
: Long.toString(longVal);
}
if (rawValue instanceof Double) {
return Double.toString((Double) rawValue);
}
if (rawValue instanceof BigDecimal) {
return ((BigDecimal) rawValue).toPlainString();
}
if (rawValue instanceof byte[]) {
return BaseEncoding.base64().encode((byte[]) rawValue);
}
return String.valueOf(rawValue);
}

/**
* Returns the attribute of this Field Value.
*
Expand All @@ -103,15 +155,20 @@ public Attribute getAttribute() {

/** Returns {@code true} if this field's value is {@code null}, {@code false} otherwise. */
public boolean isNull() {
return value == null;
return value == null && rawValue == null;
}

/**
* Returns this field's value as an {@link Object}. If {@link #isNull()} is {@code true} this
* method returns {@code null}.
*/
public Object getValue() {
return value;
Object localValue = value;
if (localValue == null && rawValue != null) {
localValue = formatRawValue();
value = localValue;
}
return localValue;
}

public Boolean getUseInt64Timestamps() {
Expand All @@ -130,8 +187,9 @@ public Boolean getUseInt64Timestamps() {
*/
@SuppressWarnings("unchecked")
public String getStringValue() {
checkNotNull(value);
return (String) value;
Object val = getValue();
checkNotNull(val);
return (String) val;
}

/**
Expand All @@ -157,6 +215,9 @@ public String getStringValueOrDefault(String defaultValue) {
* @throws IllegalStateException if the field value is not encoded in base64
*/
public byte[] getBytesValue() {
if (rawValue instanceof byte[]) {
return ((byte[]) rawValue).clone();
}
try {
return BaseEncoding.base64().decode(getStringValue());
} catch (IllegalArgumentException ex) {
Expand All @@ -174,6 +235,9 @@ public byte[] getBytesValue() {
*/
@SuppressWarnings("unchecked")
public long getLongValue() {
if (rawValue instanceof Long && !isTimestampMicros) {
return (Long) rawValue;
}
return Long.parseLong(getStringValue());
}

Expand All @@ -187,6 +251,12 @@ public long getLongValue() {
*/
@SuppressWarnings("unchecked")
public double getDoubleValue() {
if (rawValue instanceof Double) {
return (Double) rawValue;
}
if (rawValue instanceof Long && !isTimestampMicros) {
return ((Long) rawValue).doubleValue();
}
return Double.parseDouble(getStringValue());
}

Expand Down Expand Up @@ -218,6 +288,9 @@ public boolean getBooleanValue() {
*/
@SuppressWarnings("unchecked")
public long getTimestampValue() {
if (rawValue instanceof Long && isTimestampMicros) {
return (Long) rawValue;
}
if (useInt64Timestamps) {
return new BigInteger(getStringValue()).longValue();
}
Expand All @@ -242,7 +315,7 @@ public long getTimestampValue() {
*/
@SuppressWarnings("unchecked")
public Instant getTimestampInstant() {
checkNotNull(value);
checkNotNull(rawValue != null ? rawValue : value);
return Instant.EPOCH.plus(getTimestampValue(), MICROS);
}

Expand All @@ -257,6 +330,9 @@ public Instant getTimestampInstant() {
*/
@SuppressWarnings("unchecked")
public BigDecimal getNumericValue() {
if (rawValue instanceof BigDecimal) {
return (BigDecimal) rawValue;
}
return new BigDecimal(getStringValue());
}

Expand Down Expand Up @@ -330,14 +406,14 @@ public FieldValueList getRecordValue() {
public String toString() {
return MoreObjects.toStringHelper(this)
.add("attribute", attribute)
.add("value", value)
.add("value", getValue())
.add("useInt64Timestamps", useInt64Timestamps)
.toString();
}

@Override
public final int hashCode() {
return Objects.hash(attribute, value, useInt64Timestamps);
return Objects.hash(attribute, getValue(), useInt64Timestamps);
}

@Override
Expand All @@ -350,7 +426,7 @@ public final boolean equals(Object obj) {
}
FieldValue other = (FieldValue) obj;
return attribute == other.attribute
&& Objects.equals(value, other.value)
&& Objects.equals(getValue(), other.getValue())
&& Objects.equals(useInt64Timestamps, other.useInt64Timestamps);
}

Expand Down
8 changes: 8 additions & 0 deletions .../google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ArrowDeserializerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -729,11 +729,13 @@ public void testArrowVectorToFieldValue_primitives() {
FieldValue int64Val =
ArrowDeserializer.arrowVectorToFieldValue(
int64Vector, 0, LegacySQLTypeName.INTEGER, Field.Mode.NULLABLE, null, "int64_col");
assertEquals(9876543210L, int64Val.getLongValue());
assertEquals("9876543210", int64Val.getStringValue());

FieldValue int32Val =
ArrowDeserializer.arrowVectorToFieldValue(
int32Vector, 0, LegacySQLTypeName.INTEGER, Field.Mode.NULLABLE, null, "int32_col");
assertEquals(42L, int32Val.getLongValue());
assertEquals("42", int32Val.getStringValue());

FieldValue strVal =
Expand All @@ -744,11 +746,13 @@ public void testArrowVectorToFieldValue_primitives() {
FieldValue boolVal =
ArrowDeserializer.arrowVectorToFieldValue(
boolVector, 0, LegacySQLTypeName.BOOLEAN, Field.Mode.NULLABLE, null, "bool_col");
assertTrue(boolVal.getBooleanValue());
assertEquals("true", boolVal.getStringValue());

FieldValue floatVal =
ArrowDeserializer.arrowVectorToFieldValue(
floatVector, 0, LegacySQLTypeName.FLOAT, Field.Mode.NULLABLE, null, "float_col");
assertEquals(3.14159, floatVal.getDoubleValue(), 0.0);
assertEquals("3.14159", floatVal.getStringValue());

FieldValue dateVal =
Expand All @@ -764,18 +768,22 @@ public void testArrowVectorToFieldValue_primitives() {
FieldValue tsVal =
ArrowDeserializer.arrowVectorToFieldValue(
tsVector, 0, LegacySQLTypeName.TIMESTAMP, Field.Mode.NULLABLE, null, "ts_col");
assertEquals(1408452095220000L, tsVal.getTimestampValue());
assertEquals("1408452095.220000", tsVal.getStringValue());

FieldValue bytesVal =
ArrowDeserializer.arrowVectorToFieldValue(
bytesVector, 0, LegacySQLTypeName.BYTES, Field.Mode.NULLABLE, null, "bytes_col");
org.junit.jupiter.api.Assertions.assertArrayEquals(
"raw_data".getBytes(StandardCharsets.UTF_8), bytesVal.getBytesValue());
assertEquals(
BaseEncoding.base64().encode("raw_data".getBytes(StandardCharsets.UTF_8)),
bytesVal.getStringValue());

FieldValue decVal =
ArrowDeserializer.arrowVectorToFieldValue(
decVector, 0, LegacySQLTypeName.NUMERIC, Field.Mode.NULLABLE, null, "num_col");
assertEquals(new BigDecimal("123.456000000"), decVal.getNumericValue());
assertEquals("123.456000000", decVal.getStringValue());
}
}
Expand Down
37 changes: 37 additions & 0 deletions ...igquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,4 +234,41 @@ public void testParseCanonicalInterval() {
System.out.println(FieldValue.parseCanonicalInterval(entry.getKey()));
}
}

@Test
public void testTypedFieldValuesAndLazyStringMaterialization() {
FieldValue longVal = FieldValue.ofLong(9876543210L);
assertFalse(longVal.isNull());
assertEquals(9876543210L, longVal.getLongValue());
assertEquals(9876543210.0, longVal.getDoubleValue(), 0.0);
assertEquals("9876543210", longVal.getStringValue());
assertEquals(FieldValue.of(FieldValue.Attribute.PRIMITIVE, "9876543210"), longVal);
assertEquals(
FieldValue.of(FieldValue.Attribute.PRIMITIVE, "9876543210").hashCode(), longVal.hashCode());

FieldValue doubleVal = FieldValue.ofDouble(3.14159);
assertFalse(doubleVal.isNull());
assertEquals(3.14159, doubleVal.getDoubleValue(), 0.0);
assertEquals("3.14159", doubleVal.getStringValue());
assertEquals(FieldValue.of(FieldValue.Attribute.PRIMITIVE, "3.14159"), doubleVal);

FieldValue tsVal = FieldValue.ofTimestampMicros(1408452095220000L);
assertFalse(tsVal.isNull());
assertEquals(1408452095220000L, tsVal.getTimestampValue());
assertEquals("1408452095.220000", tsVal.getStringValue());
assertEquals(FieldValue.of(FieldValue.Attribute.PRIMITIVE, "1408452095.220000"), tsVal);

BigDecimal decimal = new BigDecimal("123456789.123456789");
FieldValue numVal = FieldValue.ofNumeric(decimal);
assertFalse(numVal.isNull());
assertEquals(decimal, numVal.getNumericValue());
assertEquals("123456789.123456789", numVal.getStringValue());
assertEquals(FieldValue.of(FieldValue.Attribute.PRIMITIVE, "123456789.123456789"), numVal);

FieldValue bytesVal = FieldValue.ofBytes(BYTES);
assertFalse(bytesVal.isNull());
assertArrayEquals(BYTES, bytesVal.getBytesValue());
assertEquals(BYTES_BASE64, bytesVal.getStringValue());
assertEquals(FieldValue.of(FieldValue.Attribute.PRIMITIVE, BYTES_BASE64), bytesVal);
}
}
Loading