From 4906e03d06648b5ab24a2e7dc0dd9329b7131445 Mon Sep 17 00:00:00 2001 From: Pieter De Baets Date: Thu, 29 Oct 2026 08:12:13 -0700 Subject: [PATCH] Add token-stream readable maps with a shared context Summary: Add experimental TokenStreamMap and TokenStreamContext adapters for typed access to binary token-stream data. One shared context owns the document, indexes containers and string literals, and supports independent out-of-order map and array reads. The default constructor snapshots caller-owned bytes; fromOwnedBytes accepts explicit ownership transfer without copying. Reopened maps reuse one field index per container, each string is decoded once even with concurrent readers, and numeric text is parsed once during validation. Typed access does not materialise map values. Explicit map/list conversion creates independent mutable collections using an iterative walk. Changelog: [Android][Added] - Add experimental token-stream-backed readable maps with shared string contexts Differential Revision: D128313426 --- .../facebook/react/bridge/TokenStreamMap.kt | 469 +++++++++++++++++ .../react/bridge/TokenStreamMapTest.kt | 489 ++++++++++++++++++ 2 files changed, 958 insertions(+) create mode 100644 packages/react-native/ReactAndroid/src/main/java/com/facebook/react/bridge/TokenStreamMap.kt create mode 100644 packages/react-native/ReactAndroid/src/test/java/com/facebook/react/bridge/TokenStreamMapTest.kt diff --git a/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/bridge/TokenStreamMap.kt b/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/bridge/TokenStreamMap.kt new file mode 100644 index 00000000000..3dda194a602 --- /dev/null +++ b/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/bridge/TokenStreamMap.kt @@ -0,0 +1,469 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package com.facebook.react.bridge + +import com.facebook.react.common.annotations.UnstableReactNativeAPI + +/** + * Owns a version-one binary token stream document and its shared string table. Container and string + * offsets are indexed before any maps are exposed, so reading one map never advances another map's + * decoder. The constructor copies the input; [fromOwnedBytes] accepts ownership without copying. + * + * This reader supports ordinary values, not the embedded-document tag. A mounting batch should + * encode its props as objects in one document, sharing the string table across those objects. + * Malformed documents throw [IllegalArgumentException] during construction. + */ +@UnstableReactNativeAPI +public class TokenStreamContext private constructor(bytes: ByteArray, copy: Boolean) { + /** Copies the document so later changes to the caller's buffer cannot affect retained maps. */ + public constructor(bytes: ByteArray) : this(bytes, true) + + private val bytes = if (copy) bytes.copyOf() else bytes + private val containers = HashMap() + private val stringIds = HashMap() + private val strings = ArrayList() + private val numbers = HashMap() + + init { + require( + this.bytes.size >= 4 && + this.bytes[0] == 0x42.toByte() && + this.bytes[1] == 0x54.toByte() && + this.bytes[2] == 0x53.toByte() && + this.bytes[3] == 1.toByte(), + ) { + "Not a BTS v1 document" + } + indexDocument() + } + + /** Returns an object at a previously indexed token offset, or the root object by default. */ + public fun getMap(offset: Int = 4): TokenStreamMap { + require(containerAt(offset).tag == START_OBJECT) { "No object at offset $offset" } + return TokenStreamMap(this, offset) + } + + /** Returns an array at a previously indexed token offset, or the root array by default. */ + public fun getArray(offset: Int = 4): ReadableArray { + require(containerAt(offset).tag == START_ARRAY) { "No array at offset $offset" } + return TokenStreamArray(this, containerAt(offset).values) + } + + internal inner class Container(val tag: Int, val names: IntArray, val values: IntArray) { + val fields: Map by lazy { + LinkedHashMap().apply { + for (i in names.indices) put(stringAt(names[i]), this@Container.values[i]) + } + } + } + + private class StringSlice(val offset: Int, val length: Int) { + @Volatile var decoded: String? = null + } + + private class Frame(val offset: Int, val tag: Int) { + val names = ArrayList() + val values = ArrayList() + var name = -1 + } + + internal fun containerAt(offset: Int): Container = + containers[offset] ?: throw IllegalArgumentException("No container at offset $offset") + + internal fun stringAt(id: Int): String { + val slice = strings[id] + slice.decoded?.let { + return it + } + return synchronized(slice) { + slice.decoded + ?: String(bytes, slice.offset, slice.length, Charsets.UTF_8).also { slice.decoded = it } + } + } + + internal fun typeAt(offset: Int): ReadableType = + when (bytes[offset].toInt()) { + START_OBJECT -> ReadableType.Map + START_ARRAY -> ReadableType.Array + NULL -> ReadableType.Null + FALSE, + TRUE -> ReadableType.Boolean + INTEGER, + NUMBER -> ReadableType.Number + STRING, + STRING_REF -> ReadableType.String + else -> error("Not a value at offset $offset") + } + + internal fun booleanAt(offset: Int): Boolean { + requireType(offset, ReadableType.Boolean) + return bytes[offset].toInt() == TRUE + } + + internal fun doubleAt(offset: Int): Double { + requireType(offset, ReadableType.Number) + return if (bytes[offset].toInt() == INTEGER) Cursor(offset + 1).integer().toDouble() + else checkNotNull(numbers[offset]) + } + + internal fun longAt(offset: Int): Long { + requireType(offset, ReadableType.Number) + return if (bytes[offset].toInt() == INTEGER) Cursor(offset + 1).integer() + else doubleAt(offset).toLong() + } + + internal fun nullableStringAt(offset: Int): String? { + if (typeAt(offset) == ReadableType.Null) return null + requireType(offset, ReadableType.String) + return stringAt(checkNotNull(stringIds[offset])) + } + + internal fun nullableMapAt(offset: Int): ReadableMap? { + if (typeAt(offset) == ReadableType.Null) return null + requireType(offset, ReadableType.Map) + return getMap(offset) + } + + internal fun nullableArrayAt(offset: Int): ReadableArray? { + if (typeAt(offset) == ReadableType.Null) return null + requireType(offset, ReadableType.Array) + return getArray(offset) + } + + internal fun valueAt(offset: Int): Any? = + when (typeAt(offset)) { + ReadableType.Null -> null + ReadableType.Boolean -> booleanAt(offset) + ReadableType.Number -> doubleAt(offset) + ReadableType.String -> nullableStringAt(offset) + ReadableType.Map -> nullableMapAt(offset) + ReadableType.Array -> nullableArrayAt(offset) + } + + private class MaterializationFrame( + val container: Container, + val map: HashMap?, + val array: ArrayList?, + ) { + var index = 0 + } + + private fun materializationFrame(offset: Int): MaterializationFrame { + val container = containerAt(offset) + return if (container.tag == START_OBJECT) { + MaterializationFrame(container, HashMap(), null) + } else { + MaterializationFrame(container, null, ArrayList(container.values.size)) + } + } + + internal fun materializedValueAt(offset: Int): Any? { + if (typeAt(offset) != ReadableType.Map && typeAt(offset) != ReadableType.Array) { + return valueAt(offset) + } + val root = materializationFrame(offset) + val stack = ArrayDeque() + stack.addLast(root) + while (stack.isNotEmpty()) { + val frame = stack.last() + if (frame.index == frame.container.values.size) { + stack.removeLast() + continue + } + val index = frame.index++ + val childOffset = frame.container.values[index] + val type = typeAt(childOffset) + val child = + if (type == ReadableType.Map || type == ReadableType.Array) { + materializationFrame(childOffset) + } else { + null + } + val value = if (child != null) child.map ?: child.array else valueAt(childOffset) + if (frame.map != null) { + frame.map[stringAt(frame.container.names[index])] = value + } else { + checkNotNull(frame.array).add(value) + } + if (child != null) stack.addLast(child) + } + return root.map ?: root.array + } + + private fun requireType(offset: Int, expected: ReadableType) { + val actual = typeAt(offset) + if (actual != expected) { + throw UnexpectedNativeTypeException("Expected $expected, got $actual") + } + } + + private inner class Cursor(var position: Int) { + fun byte(): Int { + require(position < bytes.size) { "Truncated BTS document" } + return bytes[position++].toInt() and 0xff + } + + fun varint(): Long { + var value = 0L + for (shift in 0..63 step 7) { + val byte = byte() + require(shift != 63 || byte <= 1) { "Overflowing BTS varint" } + value = value or ((byte and 0x7f).toLong() shl shift) + if (byte and 0x80 == 0) return value + } + throw IllegalArgumentException("Malformed BTS varint") + } + + fun integer(): Long { + val zigzag = varint() + return (zigzag ushr 1) xor -(zigzag and 1) + } + + fun length(): Int { + val length = varint() + require(length >= 0 && length <= bytes.size - position) { "Invalid BTS string length" } + return length.toInt() + } + + fun stringId(tag: Int): Int { + if (tag == STRING_REF || tag == NAME_REF) { + val id = varint() + require(id >= 0 && id < strings.size) { "Invalid BTS string reference" } + return id.toInt() + } + val length = length() + validateUtf8(position, length) + strings.add(StringSlice(position, length)) + position += length + return strings.lastIndex + } + } + + private fun indexDocument() { + val cursor = Cursor(4) + val stack = ArrayDeque() + var hasRoot = false + + fun recordValue(offset: Int) { + val parent = stack.lastOrNull() + if (parent == null) { + require(!hasRoot) { "Multiple BTS root values" } + hasRoot = true + } else { + if (parent.tag == START_OBJECT) { + require(parent.name != -1) { "BTS object value has no name" } + parent.names.add(parent.name) + parent.name = -1 + } + parent.values.add(offset) + } + } + + while (cursor.position < bytes.size) { + val offset = cursor.position + when (val tag = cursor.byte()) { + START_OBJECT, + START_ARRAY -> { + recordValue(offset) + stack.addLast(Frame(offset, tag)) + } + END_OBJECT, + END_ARRAY -> { + val frame = stack.removeLastOrNull() + require(frame != null && frame.tag == tag - 1 && frame.name == -1) { + "Unbalanced BTS container" + } + containers[frame.offset] = + Container(frame.tag, frame.names.toIntArray(), frame.values.toIntArray()) + } + NAME, + NAME_REF -> { + val frame = stack.lastOrNull() + require(frame != null && frame.tag == START_OBJECT && frame.name == -1) { + "Unexpected BTS property name" + } + frame.name = cursor.stringId(tag) + } + STRING, + STRING_REF -> { + recordValue(offset) + stringIds[offset] = cursor.stringId(tag) + } + INTEGER -> { + recordValue(offset) + cursor.varint() + } + NUMBER -> { + recordValue(offset) + val length = cursor.length() + val text = String(bytes, cursor.position, length, Charsets.US_ASCII) + numbers[offset] = + text.toDoubleOrNull() + ?: throw IllegalArgumentException("Invalid number in token stream") + cursor.position += length + } + NULL, + FALSE, + TRUE -> recordValue(offset) + else -> throw IllegalArgumentException("Unsupported BTS tag $tag") + } + } + require(hasRoot && stack.isEmpty()) { "Incomplete BTS document" } + } + + private fun validateUtf8(offset: Int, length: Int) { + var position = offset + val end = offset + length + while (position < end) { + val lead = bytes[position++].toInt() and 0xff + if (lead < 0x80) continue + val continuationCount = + when (lead) { + in 0xc2..0xdf -> 1 + in 0xe0..0xef -> 2 + in 0xf0..0xf4 -> 3 + else -> throw IllegalArgumentException("Invalid BTS UTF-8") + } + require(continuationCount <= end - position) { "Truncated BTS UTF-8" } + val second = bytes[position].toInt() and 0xff + require( + when (lead) { + 0xe0 -> second in 0xa0..0xbf + 0xed -> second in 0x80..0x9f + 0xf0 -> second in 0x90..0xbf + 0xf4 -> second in 0x80..0x8f + else -> second in 0x80..0xbf + }, + ) { + "Invalid BTS UTF-8" + } + repeat(continuationCount) { + require(bytes[position++].toInt() and 0xc0 == 0x80) { "Invalid BTS UTF-8" } + } + } + } + + public companion object { + /** + * Takes ownership of a document without copying its bytes. The caller must not modify the + * buffer after this call. Maps and arrays keep the buffer alive through their shared context. + */ + @JvmStatic + public fun fromOwnedBytes(bytes: ByteArray): TokenStreamContext = + TokenStreamContext(bytes, false) + + private const val START_OBJECT = 1 + private const val END_OBJECT = 2 + private const val START_ARRAY = 3 + private const val END_ARRAY = 4 + private const val NULL = 5 + private const val FALSE = 6 + private const val TRUE = 7 + private const val INTEGER = 8 + private const val NUMBER = 9 + private const val STRING = 10 + private const val STRING_REF = 11 + private const val NAME = 12 + private const val NAME_REF = 13 + } +} + +/** + * A read-only object view into a [TokenStreamContext]. Maps share decoded strings but not read + * cursors or iteration state. Missing keys throw [NoSuchKeyException]; explicit nulls remain + * distinguishable from absent keys. Native-map-only operations such as [WritableNativeMap.merge] + * require materializing this map into a compatible native map first. + */ +@UnstableReactNativeAPI +public class TokenStreamMap +internal constructor( + private val context: TokenStreamContext, + offset: Int, +) : ReadableMap { + private val container = context.containerAt(offset) + private val fields: Map + get() = container.fields + + private fun offset(name: String): Int = fields[name] ?: throw NoSuchKeyException(name) + + override fun hasKey(name: String): Boolean = fields.containsKey(name) + + override fun isNull(name: String): Boolean = getType(name) == ReadableType.Null + + override fun getType(name: String): ReadableType = context.typeAt(offset(name)) + + override fun getBoolean(name: String): Boolean = context.booleanAt(offset(name)) + + override fun getDouble(name: String): Double = context.doubleAt(offset(name)) + + override fun getInt(name: String): Int = getDouble(name).toInt() + + override fun getLong(name: String): Long = context.longAt(offset(name)) + + override fun getString(name: String): String? = context.nullableStringAt(offset(name)) + + override fun getMap(name: String): ReadableMap? = context.nullableMapAt(offset(name)) + + override fun getArray(name: String): ReadableArray? = context.nullableArrayAt(offset(name)) + + override fun getDynamic(name: String): Dynamic = DynamicFromMap.create(this, name) + + override fun keySetIterator(): ReadableMapKeySetIterator { + val iterator = fields.keys.iterator() + return object : ReadableMapKeySetIterator { + override fun hasNextKey(): Boolean = iterator.hasNext() + + override fun nextKey(): String = iterator.next() + } + } + + override val entryIterator: Iterator> + get() = + fields.entries + .asSequence() + .map { java.util.AbstractMap.SimpleImmutableEntry(it.key, context.valueAt(it.value)) } + .iterator() + + override fun toHashMap(): HashMap = + HashMap().apply { + for ((name, offset) in fields) put(name, context.materializedValueAt(offset)) + } +} + +@OptIn(UnstableReactNativeAPI::class) +private class TokenStreamArray( + private val context: TokenStreamContext, + private val offsets: IntArray, +) : ReadableArray { + override fun size(): Int = offsets.size + + override fun isNull(index: Int): Boolean = getType(index) == ReadableType.Null + + override fun getType(index: Int): ReadableType = context.typeAt(offsets[index]) + + override fun getBoolean(index: Int): Boolean = context.booleanAt(offsets[index]) + + override fun getDouble(index: Int): Double = context.doubleAt(offsets[index]) + + override fun getInt(index: Int): Int = getDouble(index).toInt() + + override fun getLong(index: Int): Long = context.longAt(offsets[index]) + + override fun getString(index: Int): String? = context.nullableStringAt(offsets[index]) + + override fun getMap(index: Int): ReadableMap? = context.nullableMapAt(offsets[index]) + + override fun getArray(index: Int): ReadableArray? = context.nullableArrayAt(offsets[index]) + + override fun getDynamic(index: Int): Dynamic = DynamicFromArray.create(this, index) + + override fun toArrayList(): ArrayList = + ArrayList(offsets.size).apply { + for (offset in offsets) add(context.materializedValueAt(offset)) + } +} diff --git a/packages/react-native/ReactAndroid/src/test/java/com/facebook/react/bridge/TokenStreamMapTest.kt b/packages/react-native/ReactAndroid/src/test/java/com/facebook/react/bridge/TokenStreamMapTest.kt new file mode 100644 index 00000000000..108a6ff67bc --- /dev/null +++ b/packages/react-native/ReactAndroid/src/test/java/com/facebook/react/bridge/TokenStreamMapTest.kt @@ -0,0 +1,489 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package com.facebook.react.bridge + +import com.facebook.react.common.annotations.UnstableReactNativeAPI +import java.io.ByteArrayOutputStream +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.Test + +@OptIn(UnstableReactNativeAPI::class) +class TokenStreamMapTest { + private class Writer { + private val out = ByteArrayOutputStream().apply { write(byteArrayOf(0x42, 0x54, 0x53, 1)) } + private val strings = HashMap() + + fun tag(tag: Int) = apply { out.write(tag) } + + fun position(): Int = out.size() + + fun bytes(): ByteArray = out.toByteArray() + + fun varint(value: Long) = apply { + var remaining = value + while (remaining ushr 7 != 0L) { + out.write(((remaining and 0x7f) or 0x80).toInt()) + remaining = remaining ushr 7 + } + out.write(remaining.toInt()) + } + + private fun intern(tag: Int, value: String) = apply { + val id = strings[value] + if (id != null) { + tag(tag + 1).varint(id.toLong()) + } else { + strings[value] = strings.size + val bytes = value.toByteArray(Charsets.UTF_8) + tag(tag).varint(bytes.size.toLong()) + out.write(bytes) + } + } + + fun name(name: String) = intern(12, name) + + fun string(value: String) = intern(10, value) + + fun integer(value: Long) = tag(8).varint((value shl 1) xor (value shr 63)) + + fun number(text: String) = apply { + val bytes = text.toByteArray(Charsets.US_ASCII) + tag(9).varint(bytes.size.toLong()) + out.write(bytes) + } + } + + @Test + fun mapsReadInReverseOrderShareStrings() { + val writer = Writer().tag(3) + val firstOffset = writer.position() + writer.tag(1).name("title").string("shared").name("n").integer(1).tag(2) + val secondOffset = writer.position() + writer.tag(1).name("title").string("shared").name("n").integer(2).tag(2).tag(4) + val context = TokenStreamContext(writer.bytes()) + val second = context.getMap(secondOffset) + val first = context.getMap(firstOffset) + + assertThat(second.getString("title")).isEqualTo("shared") + assertThat(second.getInt("n")).isEqualTo(2) + assertThat(first.getInt("n")).isEqualTo(1) + assertThat(first.getString("title")).isSameAs(second.getString("title")) + assertThat(context.getArray().getMap(1)?.getString("title")).isSameAs(first.getString("title")) + assertThat(first.getString("title")).isSameAs(first.getString("title")) + } + + @Test + fun referencesInsideSkippedMapsAndArraysAreIndexed() { + val writer = + Writer() + .tag(3) + .tag(1) + .name("nested") + .tag(3) + .tag(1) + .name("shared") + .string("value") + .tag(2) + .tag(4) + .tag(2) + val laterOffset = writer.position() + writer.tag(1).name("shared").string("value").tag(2).tag(4) + val context = TokenStreamContext(writer.bytes()) + val later = context.getMap(laterOffset) + + assertThat(later.getString("shared")).isEqualTo("value") + val earlier = checkNotNull(context.getArray().getMap(0)) + val nested = checkNotNull(earlier.getArray("nested")?.getMap(0)) + assertThat(nested.getString("shared")).isSameAs(later.getString("shared")) + } + + @Test + fun namesAndValuesShareOneStringTable() { + val writer = Writer().tag(1).name("same").string("same").tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + val key = map.keySetIterator().nextKey() + + assertThat(map.getString("same")).isSameAs(key) + } + + @Test + fun iteratorsAndNestedReadsHaveIndependentState() { + val writer = + Writer() + .tag(1) + .name("a") + .tag(1) + .name("x") + .integer(3) + .tag(2) + .name("b") + .tag(3) + .integer(4) + .integer(5) + .tag(4) + .tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + val first = map.keySetIterator() + val second = map.keySetIterator() + + assertThat(first.nextKey()).isEqualTo("a") + assertThat(map.getArray("b")?.getInt(1)).isEqualTo(5) + assertThat(map.getMap("a")?.getInt("x")).isEqualTo(3) + assertThat(second.nextKey()).isEqualTo("a") + assertThat(first.nextKey()).isEqualTo("b") + assertThat(second.nextKey()).isEqualTo("b") + assertThat(first.hasNextKey()).isFalse() + assertThat(map.entryIterator.asSequence().map { it.key }.toList()).containsExactly("a", "b") + } + + @Test + fun retainedMapOwnsItsBytes() { + val writer = Writer().tag(3).tag(1).name("value").string("retained").tag(2).tag(4) + val bytes = writer.bytes() + val map = checkNotNull(TokenStreamContext(bytes).getArray().getMap(0)) + bytes.fill(0) + + assertThat(map.getString("value")).isEqualTo("retained") + assertThat(map.toHashMap()).containsEntry("value", "retained") + } + + @Test + fun ownedBufferAvoidsCopyingWhileDefaultConstructorStillCopies() { + val writer = Writer().tag(1).name("value") + val valueOffset = writer.position() + 2 + writer.string("retained").tag(2) + val bytes = writer.bytes() + val owned = TokenStreamContext.fromOwnedBytes(bytes).getMap() + val copied = TokenStreamContext(bytes).getMap() + + // Mutating transferred input is a copy-detection probe, not a permitted caller operation. + bytes[valueOffset] = 'c'.code.toByte() + assertThat(owned.getString("value")).isEqualTo("cetained") + assertThat(copied.getString("value")).isEqualTo("retained") + } + + @Test + fun reopenedMapsReuseTheSharedFieldIndex() { + val writer = Writer().tag(1).name("value").integer(42).tag(2) + val context = TokenStreamContext(writer.bytes()) + val first = context.getMap() + assertThat(first.getInt("value")).isEqualTo(42) + val fields = context.containerAt(4).fields + val second = context.getMap() + assertThat(second.getInt("value")).isEqualTo(42) + assertThat(context.containerAt(4).fields).isSameAs(fields) + } + + @Test + fun integerConversionMatchesExistingReadableMaps() { + val values = + listOf( + Long.MIN_VALUE, + Int.MIN_VALUE.toLong() - 1, + Int.MIN_VALUE.toLong(), + -1L, + 0L, + 1L, + Int.MAX_VALUE.toLong(), + Int.MAX_VALUE.toLong() + 1, + 9007199254740993L, + Long.MAX_VALUE, + ) + val writer = Writer().tag(1) + values.forEachIndexed { index, value -> writer.name(index.toString()).integer(value) } + writer.name("array").tag(3) + values.forEach { writer.integer(it) } + writer.tag(4).tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + val array = checkNotNull(map.getArray("array")) + values.forEachIndexed { index, value -> + val expected = JavaOnlyMap.of("value", value).getInt("value") + assertThat(map.getInt(index.toString())).isEqualTo(expected) + assertThat(array.getInt(index)).isEqualTo(expected) + assertThat(map.getLong(index.toString())).isEqualTo(value) + } + } + + @Test + fun materializedCollectionsDoNotAliasEachOtherOrTheContext() { + val writer = + Writer() + .tag(1) + .name("nested") + .tag(1) + .name("n") + .integer(7) + .tag(2) + .name("array") + .tag(3) + .integer(9) + .tag(4) + .tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + val first = map.toHashMap() + val second = map.toHashMap() + assertThat(first).isNotSameAs(second) + assertThat(first["nested"]).isNotSameAs(second["nested"]) + assertThat(first["array"]).isNotSameAs(second["array"]) + first.clear() + val array = checkNotNull(map.getArray("array")) + val materialized = array.toArrayList() + materialized.clear() + assertThat(second).isEqualTo(mapOf("nested" to mapOf("n" to 7.0), "array" to listOf(9.0))) + assertThat(map.getMap("nested")?.getInt("n")).isEqualTo(7) + assertThat(array.getInt(0)).isEqualTo(9) + } + + @Test + fun preservesNullMissingAndTypeErrors() { + val writer = Writer().tag(1).name("reset").tag(5).name("text").string("value").tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + + assertThat(map.hasKey("reset")).isTrue() + assertThat(map.isNull("reset")).isTrue() + assertThat(map.getType("reset")).isEqualTo(ReadableType.Null) + assertThat(map.getString("reset")).isNull() + assertThat(map.getMap("reset")).isNull() + assertThat(map.getArray("reset")).isNull() + assertThat(map.hasKey("missing")).isFalse() + assertThatThrownBy { map.isNull("missing") }.isInstanceOf(NoSuchKeyException::class.java) + assertThatThrownBy { map.getString("missing") }.isInstanceOf(NoSuchKeyException::class.java) + assertThatThrownBy { map.getDouble("text") } + .isInstanceOf(UnexpectedNativeTypeException::class.java) + assertThatThrownBy { map.getBoolean("reset") } + .isInstanceOf(UnexpectedNativeTypeException::class.java) + } + + @Test + fun readsAllScalarTypesAndNumericBoundaries() { + val writer = + Writer() + .tag(1) + .name("true") + .tag(7) + .name("false") + .tag(6) + .name("int") + .integer(-129) + .name("max") + .integer(Long.MAX_VALUE) + .name("min") + .integer(Long.MIN_VALUE) + .name("fraction") + .number("1.5") + .name("negativeZero") + .number("-0") + .name("nan") + .number("NaN") + .name("infinity") + .number("Infinity") + .name("text") + .string("日本語 🦊\u0000�") + .tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + + assertThat(map.getBoolean("true")).isTrue() + assertThat(map.getBoolean("false")).isFalse() + assertThat(map.getInt("int")).isEqualTo(-129) + assertThat(map.getLong("max")).isEqualTo(Long.MAX_VALUE) + assertThat(map.getLong("min")).isEqualTo(Long.MIN_VALUE) + assertThat(map.getDouble("fraction")).isEqualTo(1.5) + assertThat(map.getInt("fraction")).isEqualTo(1) + assertThat(map.getDouble("negativeZero").toRawBits()).isEqualTo((-0.0).toRawBits()) + assertThat(map.getDouble("nan")).isNaN() + assertThat(map.getDouble("infinity")).isEqualTo(Double.POSITIVE_INFINITY) + assertThat(map.getString("text")).isEqualTo("日本語 🦊\u0000�") + assertThat(map.getDynamic("fraction").asDouble()).isEqualTo(1.5) + } + + @Test + fun materializesNestedMapsAndArrays() { + val writer = + Writer() + .tag(1) + .name("nested") + .tag(1) + .name("n") + .integer(7) + .tag(2) + .name("array") + .tag(3) + .tag(5) + .tag(7) + .number("0.5") + .string("x") + .tag(1) + .name("n") + .integer(8) + .tag(2) + .tag(3) + .integer(9) + .tag(4) + .tag(4) + .tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + val array = checkNotNull(map.getArray("array")) + + assertThat(map.getType("nested")).isEqualTo(ReadableType.Map) + assertThat(map.getType("array")).isEqualTo(ReadableType.Array) + assertThat(array.size()).isEqualTo(6) + assertThat(array.isNull(0)).isTrue() + assertThat(array.getBoolean(1)).isTrue() + assertThat(array.getDouble(2)).isEqualTo(0.5) + assertThat(array.getString(3)).isEqualTo("x") + assertThat(array.getMap(4)?.getInt("n")).isEqualTo(8) + assertThat(array.getArray(5)?.getInt(0)).isEqualTo(9) + assertThat(array.getDynamic(3).asString()).isEqualTo("x") + assertThat(map.toHashMap()) + .isEqualTo( + mapOf( + "nested" to mapOf("n" to 7.0), + "array" to listOf(null, true, 0.5, "x", mapOf("n" to 8.0), listOf(9.0)), + ), + ) + assertThat(array.toArrayList()) + .isEqualTo(listOf(null, true, 0.5, "x", mapOf("n" to 8.0), listOf(9.0))) + assertThatThrownBy { array.getString(6) }.isInstanceOf(IndexOutOfBoundsException::class.java) + } + + @Test + fun duplicateNamesKeepTheLastValue() { + val writer = Writer().tag(1).name("a").integer(1).name("a").integer(2).tag(2) + val map = TokenStreamContext(writer.bytes()).getMap() + + assertThat(map.getInt("a")).isEqualTo(2) + assertThat(map.toHashMap()).hasSize(1) + } + + @Test + fun concurrentReadersShareTheDecodedStringInstance() { + val writer = Writer().tag(3) + val offsets = ArrayList() + repeat(32) { + offsets.add(writer.position()) + writer.tag(1).name("value").string("shared".repeat(128)).tag(2) + } + writer.tag(4) + val context = TokenStreamContext(writer.bytes()) + val results = arrayOfNulls(offsets.size) + val failures = ConcurrentLinkedQueue() + val completed = CountDownLatch(offsets.size) + offsets.reversed().forEachIndexed { index, offset -> + Thread { + try { + results[index] = context.getMap(offset).getString("value") + } catch (error: Throwable) { + failures.add(error) + } finally { + completed.countDown() + } + } + .apply { isDaemon = true } + .start() + } + assertThat(completed.await(10, TimeUnit.SECONDS)).isTrue() + assertThat(failures).isEmpty() + for (value in results) assertThat(value).isSameAs(results.first()) + } + + @Test + fun rejectsInvalidHeadersAndOffsets() { + assertThatThrownBy { TokenStreamContext(byteArrayOf()) } + .isInstanceOf(IllegalArgumentException::class.java) + assertThatThrownBy { TokenStreamContext(byteArrayOf(0x42, 0x54, 0x53, 2, 1, 2)) } + .isInstanceOf(IllegalArgumentException::class.java) + val context = TokenStreamContext(Writer().tag(1).name("a").integer(1).tag(2).bytes()) + assertThatThrownBy { context.getMap(5) }.isInstanceOf(IllegalArgumentException::class.java) + assertThatThrownBy { context.getArray() }.isInstanceOf(IllegalArgumentException::class.java) + } + + @Test + fun rejectsMalformedTokensAndReferences() { + val malformed = + listOf( + Writer().bytes(), + Writer().tag(1).bytes(), + Writer().tag(2).bytes(), + Writer().tag(1).tag(4).bytes(), + Writer().tag(1).name("a").tag(2).bytes(), + Writer().tag(1).integer(1).tag(2).bytes(), + Writer().name("a").bytes(), + Writer().tag(3).name("a").tag(4).bytes(), + Writer().tag(1).name("a").name("b").integer(1).tag(2).bytes(), + Writer().tag(5).tag(5).bytes(), + Writer().tag(11).varint(0).bytes(), + Writer().tag(10).varint(-1L).bytes(), + Writer().tag(10).varint(100).bytes(), + Writer().number("not a number").bytes(), + Writer().tag(14).bytes(), + Writer().tag(255).bytes(), + ) + for ((index, bytes) in malformed.withIndex()) { + assertThatThrownBy { TokenStreamContext(bytes) } + .describedAs("case %s", index) + .isInstanceOf(IllegalArgumentException::class.java) + } + } + + @Test + fun rejectsInvalidUtf8AndOverflowingVarints() { + val invalid = + listOf( + byteArrayOf(0xc0.toByte(), 0x80.toByte()), + byteArrayOf(0xed.toByte(), 0xa0.toByte(), 0x80.toByte()), + byteArrayOf(0xf4.toByte(), 0x90.toByte(), 0x80.toByte(), 0x80.toByte()), + byteArrayOf(0xe2.toByte(), 0x82.toByte()), + byteArrayOf(0xe2.toByte(), 0x28, 0xa1.toByte()), + ) + for (bytes in invalid) { + val prefix = Writer().tag(10).varint(bytes.size.toLong()).bytes() + assertThatThrownBy { TokenStreamContext(prefix + bytes) } + .isInstanceOf(IllegalArgumentException::class.java) + } + val overflow = Writer().tag(8).bytes() + ByteArray(9) { 0x80.toByte() } + byteArrayOf(2) + assertThatThrownBy { TokenStreamContext(overflow) } + .isInstanceOf(IllegalArgumentException::class.java) + } + + @Test + fun indexesDeepDocumentsWithoutRecursiveParsing() { + val writer = Writer() + repeat(2048) { writer.tag(3) } + writer.tag(1).name("value").integer(42).tag(2) + repeat(2048) { writer.tag(4) } + val context = TokenStreamContext(writer.bytes()) + var array = context.getArray() + repeat(2047) { array = checkNotNull(array.getArray(0)) } + + assertThat(array.getMap(0)?.getInt("value")).isEqualTo(42) + var materialized: Any? = context.getArray().toArrayList() + repeat(2048) { materialized = (checkNotNull(materialized) as List<*>)[0] } + assertThat(materialized).isEqualTo(mapOf("value" to 42.0)) + } + + @Test + fun materializesDeepMapsAndArraysWithoutRecursiveCalls() { + val writer = Writer() + repeat(2048) { index -> if (index % 2 == 0) writer.tag(1).name("child") else writer.tag(3) } + writer.tag(1).name("value").integer(42).tag(2) + for (index in 2047 downTo 0) writer.tag(if (index % 2 == 0) 2 else 4) + val context = TokenStreamContext(writer.bytes()) + var materialized: Any? = context.getMap().toHashMap() + repeat(2048) { index -> + materialized = + if (index % 2 == 0) (checkNotNull(materialized) as Map<*, *>)["child"] + else (checkNotNull(materialized) as List<*>)[0] + } + + assertThat(materialized).isEqualTo(mapOf("value" to 42.0)) + } +}