diff --git a/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/bridge/TokenStreamMap.kt b/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/bridge/TokenStreamMap.kt new file mode 100644 index 00000000000..3dda194a602 --- /dev/null +++ b/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/bridge/TokenStreamMap.kt @@ -0,0 +1,469 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package com.facebook.react.bridge + +import com.facebook.react.common.annotations.UnstableReactNativeAPI + +/** + * Owns a version-one binary token stream document and its shared string table. Container and string + * offsets are indexed before any maps are exposed, so reading one map never advances another map's + * decoder. The constructor copies the input; [fromOwnedBytes] accepts ownership without copying. + * + * This reader supports ordinary values, not the embedded-document tag. A mounting batch should + * encode its props as objects in one document, sharing the string table across those objects. + * Malformed documents throw [IllegalArgumentException] during construction. + */ +@UnstableReactNativeAPI +public class TokenStreamContext private constructor(bytes: ByteArray, copy: Boolean) { + /** Copies the document so later changes to the caller's buffer cannot affect retained maps. */ + public constructor(bytes: ByteArray) : this(bytes, true) + + private val bytes = if (copy) bytes.copyOf() else bytes + private val containers = HashMap() + private val stringIds = HashMap() + private val strings = ArrayList() + private val numbers = HashMap() + + init { + require( + this.bytes.size >= 4 && + this.bytes[0] == 0x42.toByte() && + this.bytes[1] == 0x54.toByte() && + this.bytes[2] == 0x53.toByte() && + this.bytes[3] == 1.toByte(), + ) { + "Not a BTS v1 document" + } + indexDocument() + } + + /** Returns an object at a previously indexed token offset, or the root object by default. */ + public fun getMap(offset: Int = 4): TokenStreamMap { + require(containerAt(offset).tag == START_OBJECT) { "No object at offset $offset" } + return TokenStreamMap(this, offset) + } + + /** Returns an array at a previously indexed token offset, or the root array by default. */ + public fun getArray(offset: Int = 4): ReadableArray { + require(containerAt(offset).tag == START_ARRAY) { "No array at offset $offset" } + return TokenStreamArray(this, containerAt(offset).values) + } + + internal inner class Container(val tag: Int, val names: IntArray, val values: IntArray) { + val fields: Map by lazy { + LinkedHashMap().apply { + for (i in names.indices) put(stringAt(names[i]), this@Container.values[i]) + } + } + } + + private class StringSlice(val offset: Int, val length: Int) { + @Volatile var decoded: String? = null + } + + private class Frame(val offset: Int, val tag: Int) { + val names = ArrayList() + val values = ArrayList() + var name = -1 + } + + internal fun containerAt(offset: Int): Container = + containers[offset] ?: throw IllegalArgumentException("No container at offset $offset") + + internal fun stringAt(id: Int): String { + val slice = strings[id] + slice.decoded?.let { + return it + } + return synchronized(slice) { + slice.decoded + ?: String(bytes, slice.offset, slice.length, Charsets.UTF_8).also { slice.decoded = it } + } + } + + internal fun typeAt(offset: Int): ReadableType = + when (bytes[offset].toInt()) { + START_OBJECT -> ReadableType.Map + START_ARRAY -> ReadableType.Array + NULL -> ReadableType.Null + FALSE, + TRUE -> ReadableType.Boolean + INTEGER, + NUMBER -> ReadableType.Number + STRING, + STRING_REF -> ReadableType.String + else -> error("Not a value at offset $offset") + } + + internal fun booleanAt(offset: Int): Boolean { + requireType(offset, ReadableType.Boolean) + return bytes[offset].toInt() == TRUE + } + + internal fun doubleAt(offset: Int): Double { + requireType(offset, ReadableType.Number) + return if (bytes[offset].toInt() == INTEGER) Cursor(offset + 1).integer().toDouble() + else checkNotNull(numbers[offset]) + } + + internal fun longAt(offset: Int): Long { + requireType(offset, ReadableType.Number) + return if (bytes[offset].toInt() == INTEGER) Cursor(offset + 1).integer() + else doubleAt(offset).toLong() + } + + internal fun nullableStringAt(offset: Int): String? { + if (typeAt(offset) == ReadableType.Null) return null + requireType(offset, ReadableType.String) + return stringAt(checkNotNull(stringIds[offset])) + } + + internal fun nullableMapAt(offset: Int): ReadableMap? { + if (typeAt(offset) == ReadableType.Null) return null + requireType(offset, ReadableType.Map) + return getMap(offset) + } + + internal fun nullableArrayAt(offset: Int): ReadableArray? { + if (typeAt(offset) == ReadableType.Null) return null + requireType(offset, ReadableType.Array) + return getArray(offset) + } + + internal fun valueAt(offset: Int): Any? = + when (typeAt(offset)) { + ReadableType.Null -> null + ReadableType.Boolean -> booleanAt(offset) + ReadableType.Number -> doubleAt(offset) + ReadableType.String -> nullableStringAt(offset) + ReadableType.Map -> nullableMapAt(offset) + ReadableType.Array -> nullableArrayAt(offset) + } + + private class MaterializationFrame( + val container: Container, + val map: HashMap?, + val array: ArrayList?, + ) { + var index = 0 + } + + private fun materializationFrame(offset: Int): MaterializationFrame { + val container = containerAt(offset) + return if (container.tag == START_OBJECT) { + MaterializationFrame(container, HashMap(), null) + } else { + MaterializationFrame(container, null, ArrayList(container.values.size)) + } + } + + internal fun materializedValueAt(offset: Int): Any? { + if (typeAt(offset) != ReadableType.Map && typeAt(offset) != ReadableType.Array) { + return valueAt(offset) + } + val root = materializationFrame(offset) + val stack = ArrayDeque() + stack.addLast(root) + while (stack.isNotEmpty()) { + val frame = stack.last() + if (frame.index == frame.container.values.size) { + stack.removeLast() + continue + } + val index = frame.index++ + val childOffset = frame.container.values[index] + val type = typeAt(childOffset) + val child = + if (type == ReadableType.Map || type == ReadableType.Array) { + materializationFrame(childOffset) + } else { + null + } + val value = if (child != null) child.map ?: child.array else valueAt(childOffset) + if (frame.map != null) { + frame.map[stringAt(frame.container.names[index])] = value + } else { + checkNotNull(frame.array).add(value) + } + if (child != null) stack.addLast(child) + } + return root.map ?: root.array + } + + private fun requireType(offset: Int, expected: ReadableType) { + val actual = typeAt(offset) + if (actual != expected) { + throw UnexpectedNativeTypeException("Expected $expected, got $actual") + } + } + + private inner class Cursor(var position: Int) { + fun byte(): Int { + require(position < bytes.size) { "Truncated BTS document" } + return bytes[position++].toInt() and 0xff + } + + fun varint(): Long { + var value = 0L + for (shift in 0..63 step 7) { + val byte = byte() + require(shift != 63 || byte <= 1) { "Overflowing BTS varint" } + value = value or ((byte and 0x7f).toLong() shl shift) + if (byte and 0x80 == 0) return value + } + throw IllegalArgumentException("Malformed BTS varint") + } + + fun integer(): Long { + val zigzag = varint() + return (zigzag ushr 1) xor -(zigzag and 1) + } + + fun length(): Int { + val length = varint() + require(length >= 0 && length <= bytes.size - position) { "Invalid BTS string length" } + return length.toInt() + } + + fun stringId(tag: Int): Int { + if (tag == STRING_REF || tag == NAME_REF) { + val id = varint() + require(id >= 0 && id < strings.size) { "Invalid BTS string reference" } + return id.toInt() + } + val length = length() + validateUtf8(position, length) + strings.add(StringSlice(position, length)) + position += length + return strings.lastIndex + } + } + + private fun indexDocument() { + val cursor = Cursor(4) + val stack = ArrayDeque() + var hasRoot = false + + fun recordValue(offset: Int) { + val parent = stack.lastOrNull() + if (parent == null) { + require(!hasRoot) { "Multiple BTS root values" } + hasRoot = true + } else { + if (parent.tag == START_OBJECT) { + require(parent.name != -1) { "BTS object value has no name" } + parent.names.add(parent.name) + parent.name = -1 + } + parent.values.add(offset) + } + } + + while (cursor.position < bytes.size) { + val offset = cursor.position + when (val tag = cursor.byte()) { + START_OBJECT, + START_ARRAY -> { + recordValue(offset) + stack.addLast(Frame(offset, tag)) + } + END_OBJECT, + END_ARRAY -> { + val frame = stack.removeLastOrNull() + require(frame != null && frame.tag == tag - 1 && frame.name == -1) { + "Unbalanced BTS container" + } + containers[frame.offset] = + Container(frame.tag, frame.names.toIntArray(), frame.values.toIntArray()) + } + NAME, + NAME_REF -> { + val frame = stack.lastOrNull() + require(frame != null && frame.tag == START_OBJECT && frame.name == -1) { + "Unexpected BTS property name" + } + frame.name = cursor.stringId(tag) + } + STRING, + STRING_REF -> { + recordValue(offset) + stringIds[offset] = cursor.stringId(tag) + } + INTEGER -> { + recordValue(offset) + cursor.varint() + } + NUMBER -> { + recordValue(offset) + val length = cursor.length() + val text = String(bytes, cursor.position, length, Charsets.US_ASCII) + numbers[offset] = + text.toDoubleOrNull() + ?: throw IllegalArgumentException("Invalid number in token stream") + cursor.position += length + } + NULL, + FALSE, + TRUE -> recordValue(offset) + else -> throw IllegalArgumentException("Unsupported BTS tag $tag") + } + } + require(hasRoot && stack.isEmpty()) { "Incomplete BTS document" } + } + + private fun validateUtf8(offset: Int, length: Int) { + var position = offset + val end = offset + length + while (position < end) { + val lead = bytes[position++].toInt() and 0xff + if (lead < 0x80) continue + val continuationCount = + when (lead) { + in 0xc2..0xdf -> 1 + in 0xe0..0xef -> 2 + in 0xf0..0xf4 -> 3 + else -> throw IllegalArgumentException("Invalid BTS UTF-8") + } + require(continuationCount <= end - position) { "Truncated BTS UTF-8" } + val second = bytes[position].toInt() and 0xff + require( + when (lead) { + 0xe0 -> second in 0xa0..0xbf + 0xed -> second in 0x80..0x9f + 0xf0 -> second in 0x90..0xbf + 0xf4 -> second in 0x80..0x8f + else -> second in 0x80..0xbf + }, + ) { + "Invalid BTS UTF-8" + } + repeat(continuationCount) { + require(bytes[position++].toInt() and 0xc0 == 0x80) { "Invalid BTS UTF-8" } + } + } + } + + public companion object { + /** + * Takes ownership of a document without copying its bytes. The caller must not modify the + * buffer after this call. Maps and arrays keep the buffer alive through their shared context. + */ + @JvmStatic + public fun fromOwnedBytes(bytes: ByteArray): TokenStreamContext = + TokenStreamContext(bytes, false) + + private const val START_OBJECT = 1 + private const val END_OBJECT = 2 + private const val START_ARRAY = 3 + private const val END_ARRAY = 4 + private const val NULL = 5 + private const val FALSE = 6 + private const val TRUE = 7 + private const val INTEGER = 8 + private const val NUMBER = 9 + private const val STRING = 10 + private const val STRING_REF = 11 + private const val NAME = 12 + private const val NAME_REF = 13 + } +} + +/** + * A read-only object view into a [TokenStreamContext]. Maps share decoded strings but not read + * cursors or iteration state. Missing keys throw [NoSuchKeyException]; explicit nulls remain + * distinguishable from absent keys. Native-map-only operations such as [WritableNativeMap.merge] + * require materializing this map into a compatible native map first. + */ +@UnstableReactNativeAPI +public class TokenStreamMap +internal constructor( + private val context: TokenStreamContext, + offset: Int, +) : ReadableMap { + private val container = context.containerAt(offset) + private val fields: Map + get() = container.fields + + private fun offset(name: String): Int = fields[name] ?: throw NoSuchKeyException(name) + + override fun hasKey(name: String): Boolean = fields.containsKey(name) + + override fun isNull(name: String): Boolean = getType(name) == ReadableType.Null + + override fun getType(name: String): ReadableType = context.typeAt(offset(name)) + + override fun getBoolean(name: String): Boolean = context.booleanAt(offset(name)) + + override fun getDouble(name: String): Double = context.doubleAt(offset(name)) + + override fun getInt(name: String): Int = getDouble(name).toInt() + + override fun getLong(name: String): Long = context.longAt(offset(name)) + + override fun getString(name: String): String? = context.nullableStringAt(offset(name)) + + override fun getMap(name: String): ReadableMap? = context.nullableMapAt(offset(name)) + + override fun getArray(name: String): ReadableArray? = context.nullableArrayAt(offset(name)) + + override fun getDynamic(name: String): Dynamic = DynamicFromMap.create(this, name) + + override fun keySetIterator(): ReadableMapKeySetIterator { + val iterator = fields.keys.iterator() + return object : ReadableMapKeySetIterator { + override fun hasNextKey(): Boolean = iterator.hasNext() + + override fun nextKey(): String = iterator.next() + } + } + + override val entryIterator: Iterator> + get() = + fields.entries + .asSequence() + .map { java.util.AbstractMap.SimpleImmutableEntry(it.key, context.valueAt(it.value)) } + .iterator() + + override fun toHashMap(): HashMap = + HashMap().apply { + for ((name, offset) in fields) put(name, context.materializedValueAt(offset)) + } +} + +@OptIn(UnstableReactNativeAPI::class) +private class TokenStreamArray( + private val context: TokenStreamContext, + private val offsets: IntArray, +) : ReadableArray { + override fun size(): Int = offsets.size + + override fun isNull(index: Int): Boolean = getType(index) == ReadableType.Null + + override fun getType(index: Int): ReadableType = context.typeAt(offsets[index]) + + override fun getBoolean(index: Int): Boolean = context.booleanAt(offsets[index]) + + override fun getDouble(index: Int): Double = context.doubleAt(offsets[index]) + + override fun getInt(index: Int): Int = getDouble(index).toInt() + + override fun getLong(index: Int): Long = context.longAt(offsets[index]) + + override fun getString(index: Int): String? = context.nullableStringAt(offsets[index]) + + override fun getMap(index: Int): ReadableMap? = context.nullableMapAt(offsets[index]) + + override fun getArray(index: Int): ReadableArray? = context.nullableArrayAt(offsets[index]) + + override fun getDynamic(index: Int): Dynamic = DynamicFromArray.create(this, index) + + override fun toArrayList(): ArrayList = + ArrayList(offsets.size).apply { + for (offset in offsets) add(context.materializedValueAt(offset)) + } +} diff --git a/packages/react-native/ReactAndroid/src/test/java/com/facebook/react/bridge/TokenStreamMapTest.kt b/packages/react-native/ReactAndroid/src/test/java/com/facebook/react/bridge/TokenStreamMapTest.kt new file mode 100644 index 00000000000..108a6ff67bc --- /dev/null +++ b/packages/react-native/ReactAndroid/src/test/java/com/facebook/react/bridge/TokenStreamMapTest.kt @@ -0,0 +1,489 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package com.facebook.react.bridge + +import com.facebook.react.common.annotations.UnstableReactNativeAPI +import java.io.ByteArrayOutputStream +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.Test + +@OptIn(UnstableReactNativeAPI::class) +class TokenStreamMapTest { + private class Writer { + private val out = ByteArrayOutputStream().apply { write(byteArrayOf(0x42, 0x54, 0x53, 1)) } + private val strings = HashMap() + + fun tag(tag: Int) = apply { out.write(tag) } + + fun position(): Int = out.size() + + fun bytes(): ByteArray = out.toByteArray() + + fun varint(value: Long) = apply { + var remaining = value + while (remaining ushr 7 != 0L) { + out.write(((remaining and 0x7f) or 0x80).toInt()) + remaining = remaining ushr 7 + } + out.write(remaining.toInt()) + } + + private fun intern(tag: Int, value: String) = apply { + val id = strings[value] + if (id != null) { + tag(tag + 1).varint(id.toLong()) + } else { + strings[value] = strings.size + val bytes = value.toByteArray(Charsets.UTF_8) + tag(tag).varint(bytes.size.toLong()) + out.write(bytes) + } + } + + fun name(name: String) = intern(12, name) + + fun string(value: String) = intern(10, value) + + fun integer(value: Long) = tag(8).varint((value shl 1) xor (value shr 63)) + + fun number(text: String) = apply { + val bytes = text.toByteArray(Charsets.US_ASCII) + tag(9).varint(bytes.size.toLong()) + out.write(bytes) + } + } + + @Test + fun mapsReadInReverseOrderShareStrings() { + val writer = Writer().tag(3) + val firstOffset = writer.position() + writer.tag(1).name("title").string("shared").name("n").integer(1).tag(2) + val secondOffset = writer.position() + writer.tag(1).name("title").string("shared").name("n").integer(2).tag(2).tag(4) + val context = TokenStreamContext(writer.bytes()) + val second = context.getMap(secondOffset) + val first = context.getMap(firstOffset) + + assertThat(second.getString("title")).isEqualTo("shared") + assertThat(second.getInt("n")).isEqualTo(2) + assertThat(first.getInt("n")).isEqualTo(1) + assertThat(first.getString("title")).isSameAs(second.getString("title")) + assertThat(context.getArray().getMap(1)?.getString("title")).isSameAs(first.getString("title")) + assertThat(first.getString("title")).isSameAs(first.getString("title")) + } + + @Test + fun referencesInsideSkippedMapsAndArraysAreIndexed() { + val writer = + Writer() + .tag(3) + .tag(1) + .name("nested") + .tag(3) + .tag(1) + .name("shared") + .string("value") + .tag(2) + .tag(4) + .tag(2) + val laterOffset = writer.position() + writer.tag(1).name("shared").string("value").tag(2).tag(4) + val context = TokenStreamContext(writer.bytes()) + val later = context.getMap(laterOffset) + + assertThat(later.getString("shared")).isEqualTo("value") + val earlier = checkNotNull(context.getArray().getMap(0)) + val nested = checkNotNull(earlier.getArray("nested")?.getMap(0)) + assertThat(nested.getString("shared")).isSameAs(later.getString("shared")) + } + + @Test + fun namesAndValuesShareOneStringTable() { + val writer = Writer().tag(1).name("same").string("same").tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + val key = map.keySetIterator().nextKey() + + assertThat(map.getString("same")).isSameAs(key) + } + + @Test + fun iteratorsAndNestedReadsHaveIndependentState() { + val writer = + Writer() + .tag(1) + .name("a") + .tag(1) + .name("x") + .integer(3) + .tag(2) + .name("b") + .tag(3) + .integer(4) + .integer(5) + .tag(4) + .tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + val first = map.keySetIterator() + val second = map.keySetIterator() + + assertThat(first.nextKey()).isEqualTo("a") + assertThat(map.getArray("b")?.getInt(1)).isEqualTo(5) + assertThat(map.getMap("a")?.getInt("x")).isEqualTo(3) + assertThat(second.nextKey()).isEqualTo("a") + assertThat(first.nextKey()).isEqualTo("b") + assertThat(second.nextKey()).isEqualTo("b") + assertThat(first.hasNextKey()).isFalse() + assertThat(map.entryIterator.asSequence().map { it.key }.toList()).containsExactly("a", "b") + } + + @Test + fun retainedMapOwnsItsBytes() { + val writer = Writer().tag(3).tag(1).name("value").string("retained").tag(2).tag(4) + val bytes = writer.bytes() + val map = checkNotNull(TokenStreamContext(bytes).getArray().getMap(0)) + bytes.fill(0) + + assertThat(map.getString("value")).isEqualTo("retained") + assertThat(map.toHashMap()).containsEntry("value", "retained") + } + + @Test + fun ownedBufferAvoidsCopyingWhileDefaultConstructorStillCopies() { + val writer = Writer().tag(1).name("value") + val valueOffset = writer.position() + 2 + writer.string("retained").tag(2) + val bytes = writer.bytes() + val owned = TokenStreamContext.fromOwnedBytes(bytes).getMap() + val copied = TokenStreamContext(bytes).getMap() + + // Mutating transferred input is a copy-detection probe, not a permitted caller operation. + bytes[valueOffset] = 'c'.code.toByte() + assertThat(owned.getString("value")).isEqualTo("cetained") + assertThat(copied.getString("value")).isEqualTo("retained") + } + + @Test + fun reopenedMapsReuseTheSharedFieldIndex() { + val writer = Writer().tag(1).name("value").integer(42).tag(2) + val context = TokenStreamContext(writer.bytes()) + val first = context.getMap() + assertThat(first.getInt("value")).isEqualTo(42) + val fields = context.containerAt(4).fields + val second = context.getMap() + assertThat(second.getInt("value")).isEqualTo(42) + assertThat(context.containerAt(4).fields).isSameAs(fields) + } + + @Test + fun integerConversionMatchesExistingReadableMaps() { + val values = + listOf( + Long.MIN_VALUE, + Int.MIN_VALUE.toLong() - 1, + Int.MIN_VALUE.toLong(), + -1L, + 0L, + 1L, + Int.MAX_VALUE.toLong(), + Int.MAX_VALUE.toLong() + 1, + 9007199254740993L, + Long.MAX_VALUE, + ) + val writer = Writer().tag(1) + values.forEachIndexed { index, value -> writer.name(index.toString()).integer(value) } + writer.name("array").tag(3) + values.forEach { writer.integer(it) } + writer.tag(4).tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + val array = checkNotNull(map.getArray("array")) + values.forEachIndexed { index, value -> + val expected = JavaOnlyMap.of("value", value).getInt("value") + assertThat(map.getInt(index.toString())).isEqualTo(expected) + assertThat(array.getInt(index)).isEqualTo(expected) + assertThat(map.getLong(index.toString())).isEqualTo(value) + } + } + + @Test + fun materializedCollectionsDoNotAliasEachOtherOrTheContext() { + val writer = + Writer() + .tag(1) + .name("nested") + .tag(1) + .name("n") + .integer(7) + .tag(2) + .name("array") + .tag(3) + .integer(9) + .tag(4) + .tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + val first = map.toHashMap() + val second = map.toHashMap() + assertThat(first).isNotSameAs(second) + assertThat(first["nested"]).isNotSameAs(second["nested"]) + assertThat(first["array"]).isNotSameAs(second["array"]) + first.clear() + val array = checkNotNull(map.getArray("array")) + val materialized = array.toArrayList() + materialized.clear() + assertThat(second).isEqualTo(mapOf("nested" to mapOf("n" to 7.0), "array" to listOf(9.0))) + assertThat(map.getMap("nested")?.getInt("n")).isEqualTo(7) + assertThat(array.getInt(0)).isEqualTo(9) + } + + @Test + fun preservesNullMissingAndTypeErrors() { + val writer = Writer().tag(1).name("reset").tag(5).name("text").string("value").tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + + assertThat(map.hasKey("reset")).isTrue() + assertThat(map.isNull("reset")).isTrue() + assertThat(map.getType("reset")).isEqualTo(ReadableType.Null) + assertThat(map.getString("reset")).isNull() + assertThat(map.getMap("reset")).isNull() + assertThat(map.getArray("reset")).isNull() + assertThat(map.hasKey("missing")).isFalse() + assertThatThrownBy { map.isNull("missing") }.isInstanceOf(NoSuchKeyException::class.java) + assertThatThrownBy { map.getString("missing") }.isInstanceOf(NoSuchKeyException::class.java) + assertThatThrownBy { map.getDouble("text") } + .isInstanceOf(UnexpectedNativeTypeException::class.java) + assertThatThrownBy { map.getBoolean("reset") } + .isInstanceOf(UnexpectedNativeTypeException::class.java) + } + + @Test + fun readsAllScalarTypesAndNumericBoundaries() { + val writer = + Writer() + .tag(1) + .name("true") + .tag(7) + .name("false") + .tag(6) + .name("int") + .integer(-129) + .name("max") + .integer(Long.MAX_VALUE) + .name("min") + .integer(Long.MIN_VALUE) + .name("fraction") + .number("1.5") + .name("negativeZero") + .number("-0") + .name("nan") + .number("NaN") + .name("infinity") + .number("Infinity") + .name("text") + .string("日本語 🦊\u0000�") + .tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + + assertThat(map.getBoolean("true")).isTrue() + assertThat(map.getBoolean("false")).isFalse() + assertThat(map.getInt("int")).isEqualTo(-129) + assertThat(map.getLong("max")).isEqualTo(Long.MAX_VALUE) + assertThat(map.getLong("min")).isEqualTo(Long.MIN_VALUE) + assertThat(map.getDouble("fraction")).isEqualTo(1.5) + assertThat(map.getInt("fraction")).isEqualTo(1) + assertThat(map.getDouble("negativeZero").toRawBits()).isEqualTo((-0.0).toRawBits()) + assertThat(map.getDouble("nan")).isNaN() + assertThat(map.getDouble("infinity")).isEqualTo(Double.POSITIVE_INFINITY) + assertThat(map.getString("text")).isEqualTo("日本語 🦊\u0000�") + assertThat(map.getDynamic("fraction").asDouble()).isEqualTo(1.5) + } + + @Test + fun materializesNestedMapsAndArrays() { + val writer = + Writer() + .tag(1) + .name("nested") + .tag(1) + .name("n") + .integer(7) + .tag(2) + .name("array") + .tag(3) + .tag(5) + .tag(7) + .number("0.5") + .string("x") + .tag(1) + .name("n") + .integer(8) + .tag(2) + .tag(3) + .integer(9) + .tag(4) + .tag(4) + .tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + val array = checkNotNull(map.getArray("array")) + + assertThat(map.getType("nested")).isEqualTo(ReadableType.Map) + assertThat(map.getType("array")).isEqualTo(ReadableType.Array) + assertThat(array.size()).isEqualTo(6) + assertThat(array.isNull(0)).isTrue() + assertThat(array.getBoolean(1)).isTrue() + assertThat(array.getDouble(2)).isEqualTo(0.5) + assertThat(array.getString(3)).isEqualTo("x") + assertThat(array.getMap(4)?.getInt("n")).isEqualTo(8) + assertThat(array.getArray(5)?.getInt(0)).isEqualTo(9) + assertThat(array.getDynamic(3).asString()).isEqualTo("x") + assertThat(map.toHashMap()) + .isEqualTo( + mapOf( + "nested" to mapOf("n" to 7.0), + "array" to listOf(null, true, 0.5, "x", mapOf("n" to 8.0), listOf(9.0)), + ), + ) + assertThat(array.toArrayList()) + .isEqualTo(listOf(null, true, 0.5, "x", mapOf("n" to 8.0), listOf(9.0))) + assertThatThrownBy { array.getString(6) }.isInstanceOf(IndexOutOfBoundsException::class.java) + } + + @Test + fun duplicateNamesKeepTheLastValue() { + val writer = Writer().tag(1).name("a").integer(1).name("a").integer(2).tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + + assertThat(map.getInt("a")).isEqualTo(2) + assertThat(map.toHashMap()).hasSize(1) + } + + @Test + fun concurrentReadersShareTheDecodedStringInstance() { + val writer = Writer().tag(3) + val offsets = ArrayList() + repeat(32) { + offsets.add(writer.position()) + writer.tag(1).name("value").string("shared".repeat(128)).tag(2) + } + writer.tag(4) + val context = TokenStreamContext(writer.bytes()) + val results = arrayOfNulls(offsets.size) + val failures = ConcurrentLinkedQueue() + val completed = CountDownLatch(offsets.size) + offsets.reversed().forEachIndexed { index, offset -> + Thread { + try { + results[index] = context.getMap(offset).getString("value") + } catch (error: Throwable) { + failures.add(error) + } finally { + completed.countDown() + } + } + .apply { isDaemon = true } + .start() + } + assertThat(completed.await(10, TimeUnit.SECONDS)).isTrue() + assertThat(failures).isEmpty() + for (value in results) assertThat(value).isSameAs(results.first()) + } + + @Test + fun rejectsInvalidHeadersAndOffsets() { + assertThatThrownBy { TokenStreamContext(byteArrayOf()) } + .isInstanceOf(IllegalArgumentException::class.java) + assertThatThrownBy { TokenStreamContext(byteArrayOf(0x42, 0x54, 0x53, 2, 1, 2)) } + .isInstanceOf(IllegalArgumentException::class.java) + val context = TokenStreamContext(Writer().tag(1).name("a").integer(1).tag(2).bytes()) + assertThatThrownBy { context.getMap(5) }.isInstanceOf(IllegalArgumentException::class.java) + assertThatThrownBy { context.getArray() }.isInstanceOf(IllegalArgumentException::class.java) + } + + @Test + fun rejectsMalformedTokensAndReferences() { + val malformed = + listOf( + Writer().bytes(), + Writer().tag(1).bytes(), + Writer().tag(2).bytes(), + Writer().tag(1).tag(4).bytes(), + Writer().tag(1).name("a").tag(2).bytes(), + Writer().tag(1).integer(1).tag(2).bytes(), + Writer().name("a").bytes(), + Writer().tag(3).name("a").tag(4).bytes(), + Writer().tag(1).name("a").name("b").integer(1).tag(2).bytes(), + Writer().tag(5).tag(5).bytes(), + Writer().tag(11).varint(0).bytes(), + Writer().tag(10).varint(-1L).bytes(), + Writer().tag(10).varint(100).bytes(), + Writer().number("not a number").bytes(), + Writer().tag(14).bytes(), + Writer().tag(255).bytes(), + ) + for ((index, bytes) in malformed.withIndex()) { + assertThatThrownBy { TokenStreamContext(bytes) } + .describedAs("case %s", index) + .isInstanceOf(IllegalArgumentException::class.java) + } + } + + @Test + fun rejectsInvalidUtf8AndOverflowingVarints() { + val invalid = + listOf( + byteArrayOf(0xc0.toByte(), 0x80.toByte()), + byteArrayOf(0xed.toByte(), 0xa0.toByte(), 0x80.toByte()), + byteArrayOf(0xf4.toByte(), 0x90.toByte(), 0x80.toByte(), 0x80.toByte()), + byteArrayOf(0xe2.toByte(), 0x82.toByte()), + byteArrayOf(0xe2.toByte(), 0x28, 0xa1.toByte()), + ) + for (bytes in invalid) { + val prefix = Writer().tag(10).varint(bytes.size.toLong()).bytes() + assertThatThrownBy { TokenStreamContext(prefix + bytes) } + .isInstanceOf(IllegalArgumentException::class.java) + } + val overflow = Writer().tag(8).bytes() + ByteArray(9) { 0x80.toByte() } + byteArrayOf(2) + assertThatThrownBy { TokenStreamContext(overflow) } + .isInstanceOf(IllegalArgumentException::class.java) + } + + @Test + fun indexesDeepDocumentsWithoutRecursiveParsing() { + val writer = Writer() + repeat(2048) { writer.tag(3) } + writer.tag(1).name("value").integer(42).tag(2) + repeat(2048) { writer.tag(4) } + val context = TokenStreamContext(writer.bytes()) + var array = context.getArray() + repeat(2047) { array = checkNotNull(array.getArray(0)) } + + assertThat(array.getMap(0)?.getInt("value")).isEqualTo(42) + var materialized: Any? = context.getArray().toArrayList() + repeat(2048) { materialized = (checkNotNull(materialized) as List<*>)[0] } + assertThat(materialized).isEqualTo(mapOf("value" to 42.0)) + } + + @Test + fun materializesDeepMapsAndArraysWithoutRecursiveCalls() { + val writer = Writer() + repeat(2048) { index -> if (index % 2 == 0) writer.tag(1).name("child") else writer.tag(3) } + writer.tag(1).name("value").integer(42).tag(2) + for (index in 2047 downTo 0) writer.tag(if (index % 2 == 0) 2 else 4) + val context = TokenStreamContext(writer.bytes()) + var materialized: Any? = context.getMap().toHashMap() + repeat(2048) { index -> + materialized = + if (index % 2 == 0) (checkNotNull(materialized) as Map<*, *>)["child"] + else (checkNotNull(materialized) as List<*>)[0] + } + + assertThat(materialized).isEqualTo(mapOf("value" to 42.0)) + } +}