From adc601e11948449a5f3bb68c9c683c7e25a60e09 Mon Sep 17 00:00:00 2001 From: eden <198768181+lcweden@users.noreply.github.com> Date: Sun, 12 Jul 2026 14:15:31 +0000 Subject: [PATCH 1/3] feat: enhance string consumption with resumable support in decoder --- src/modules/decoder.ts | 47 +++++++++++++++++++++++++++++++++++------- src/utils/wire.ts | 38 +++++++++++++++++++++++++++------- 2 files changed, 71 insertions(+), 14 deletions(-) diff --git a/src/modules/decoder.ts b/src/modules/decoder.ts index d69009e..d496856 100644 --- a/src/modules/decoder.ts +++ b/src/modules/decoder.ts @@ -15,7 +15,7 @@ import { consumeNumber, consumeSimpleNumber, consumeSimpleString, - consumeString, + consumeStringResumable, consumeTrue, consumeWhitespace, } from "#src/utils/wire"; @@ -28,6 +28,7 @@ import { * @internal */ class Decoder { + #offset: number; #cursor: Cursor; #state: State; #options: DecoderOptions; @@ -39,6 +40,7 @@ class Decoder { * @param options Decoder configuration options. */ constructor(bytes: Uint8Array, options: DecoderOptions) { + this.#offset = 0; this.#cursor = new Cursor(bytes); this.#state = new State(options); this.#options = options; @@ -197,6 +199,7 @@ class Decoder { * and structural state. */ reset(): void { + this.#offset = 0; this.#cursor = new Cursor(new Uint8Array()); this.#state = new State(this.#options); } @@ -407,8 +410,25 @@ class Decoder { } if (kind === KIND.STRING) { - return consumeSimpleString(bytes, position) || - consumeString(bytes, position, !this.#options.allowInvalidUTF8); + if (this.#offset === 0) { + const size = consumeSimpleString(bytes, position); + if (size > 0) return size; + } + + const result = consumeStringResumable( + bytes, + position, + this.#offset, + !this.#options.allowInvalidUTF8, + ); + + if (!result.completed) { + this.#offset = result.consumed; + return 0; + } + + this.#offset = 0; + return result.consumed; } if (kind === KIND.NUMBER) { @@ -529,14 +549,27 @@ class Decoder { } #consumeString(start: number): number { - let size = consumeSimpleString(this.#cursor.bytes, start); + let size = 0; - if (size === 0) { - size = consumeString(this.#cursor.bytes, start, !this.#options.allowInvalidUTF8); + if (this.#offset === 0) { + size = consumeSimpleString(this.#cursor.bytes, start); + } - if (size === 0) { + if (size === 0) { + const result = consumeStringResumable( + this.#cursor.bytes, + start, + this.#offset, + !this.#options.allowInvalidUTF8, + ); + + if (!result.completed) { + this.#offset = result.consumed; return 0; } + + size = result.consumed; + this.#offset = 0; } if (this.#state.needObjectName()) { diff --git a/src/utils/wire.ts b/src/utils/wire.ts index 1269522..7e3d517 100644 --- a/src/utils/wire.ts +++ b/src/utils/wire.ts @@ -112,13 +112,33 @@ function consumeFalse(bytes: Uint8Array, position: number): number { * @returns The number of bytes consumed if a string literal is found, otherwise 0. */ function consumeString(bytes: Uint8Array, position: number, validateUTF8 = true): number { + const result = consumeStringResumable(bytes, position, 0, validateUTF8); + + return result.completed ? result.consumed : 0; +} + +/** + * Consumes a string literal from the given position, supporting resumption from a previous offset. + * @param bytes The Uint8Array bytes to consume from. + * @param position The position in the bytes to start consuming. + * @param offset The relative number of bytes already scanned in previous chunks. + * @param validateUTF8 Whether to validate the string as UTF-8. + * @returns An object containing the number of bytes consumed (or the scan offset if incomplete) and whether the string is complete. + */ +function consumeStringResumable( + bytes: Uint8Array, + position: number, + offset: number, + validateUTF8 = true, +): { consumed: number; completed: boolean } { if (position >= bytes.length || bytes[position] !== ASCII.QUOTE) { - return 0; + return { consumed: 0, completed: false }; } let inEscape = false; + let index = offset > 0 ? position + offset : position + 1; - for (let index = position + 1; index < bytes.length; index++) { + for (; index < bytes.length; index++) { const byte = bytes[index]; if (inEscape) { @@ -127,12 +147,15 @@ function consumeString(bytes: Uint8Array, position: number, validateUTF8 = true) } if (byte === ASCII.BACKSLASH) { + if (index === bytes.length - 1) { + return { consumed: index - position, completed: false }; + } inEscape = true; continue; } if (byte < ASCII.SPACE) { - return 0; + return { consumed: 0, completed: false }; } if (byte === ASCII.QUOTE) { @@ -141,16 +164,16 @@ function consumeString(bytes: Uint8Array, position: number, validateUTF8 = true) if (validateUTF8) { try { decodeText(chunk, true); - } catch (_error) { - return 0; + } catch { + return { consumed: 0, completed: false }; } } - return index - position + 1; + return { consumed: index - position + 1, completed: true }; } } - return 0; + return { consumed: index - position, completed: false }; } /** @@ -332,6 +355,7 @@ export { consumeSimpleNumber, consumeSimpleString, consumeString, + consumeStringResumable, consumeTrue, consumeWhitespace, }; From ad6c332cdf69ce141e486c85e859490384083cf1 Mon Sep 17 00:00:00 2001 From: eden <198768181+lcweden@users.noreply.github.com> Date: Sun, 12 Jul 2026 14:50:52 +0000 Subject: [PATCH 2/3] feat: optimize byte handling in Cursor for improved memory management --- src/modules/cursor.ts | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/modules/cursor.ts b/src/modules/cursor.ts index 0ead8f5..8a1045e 100644 --- a/src/modules/cursor.ts +++ b/src/modules/cursor.ts @@ -5,12 +5,13 @@ */ class Cursor { #baseOffset: number; + #bytes: Uint8Array; #ended: boolean; + #owned: boolean; #previousStart: number; #previousEnd: number; #peekPosition: number; #peekError: Error | null; - #bytes: Uint8Array; /** * Creates a new Cursor instance starting with an initial chunk of bytes. @@ -19,12 +20,13 @@ class Cursor { */ constructor(bytes: Uint8Array) { this.#baseOffset = 0; + this.#bytes = bytes; this.#ended = false; + this.#owned = false; this.#previousStart = 0; this.#previousEnd = 0; this.#peekPosition = 0; this.#peekError = null; - this.#bytes = bytes; } /** @@ -87,25 +89,48 @@ class Cursor { /** * Appends a newly received chunk of bytes to the cursor. - * - **Fast-Path**: If all previous bytes were consumed, it simply reassigns the internal pointer. - * - **Slow-Path**: If there are unread bytes, it allocates a new buffer and merges them. * * @param bytes The new Uint8Array chunk arriving from the stream. */ appendBytes(bytes: Uint8Array): void { const unread = this.unreadBytes(); + const start = this.#previousEnd; if (unread.length > 0) { - const merged = new Uint8Array(unread.length + bytes.length); + const capacity = start + unread.length + bytes.length; + const length = unread.length + bytes.length; + + if (this.#owned && capacity <= this.#bytes.buffer.byteLength) { + const buffer = this.#bytes.buffer; + + this.#bytes = new Uint8Array(buffer, 0, capacity); + this.#bytes.set(bytes, start + unread.length); - merged.set(unread, 0); - merged.set(bytes, unread.length); + return; + } - this.#bytes = merged; + if (this.#owned && length <= this.#bytes.buffer.byteLength) { + const buffer = this.#bytes.buffer; + const view = new Uint8Array(buffer); + + view.copyWithin(0, start, start + unread.length); + + this.#bytes = new Uint8Array(buffer, 0, length); + this.#bytes.set(bytes, unread.length); + } else { + const capacity = this.#owned ? this.#bytes.buffer.byteLength : 0; + const array = new Uint8Array(Math.max(length, capacity * 2)); + + array.set(unread, 0); + array.set(bytes, unread.length); + + this.#bytes = new Uint8Array(array.buffer, 0, length); + this.#owned = true; + } } else { this.#bytes = bytes; + this.#owned = false; } - if (this.#peekPosition > 0) { this.#peekPosition -= this.#previousEnd; } From e2a8d0c321dde7f69a88db5af95d9e41f71bc6d2 Mon Sep 17 00:00:00 2001 From: eden <198768181+lcweden@users.noreply.github.com> Date: Sun, 12 Jul 2026 15:09:15 +0000 Subject: [PATCH 3/3] test: add resumable string consumption tests for decoder and wire utilities --- tests/integration/decoder.test.ts | 71 +++++++++++++++++++++++++++++++ tests/wire.test.ts | 60 ++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/tests/integration/decoder.test.ts b/tests/integration/decoder.test.ts index f8342e5..4ee2820 100644 --- a/tests/integration/decoder.test.ts +++ b/tests/integration/decoder.test.ts @@ -244,6 +244,77 @@ Deno.test("[integration] JSONTextDecoder", async (test) => { assertEquals(decoder.readValue()?.text(), "1"); assertEquals(decoder.readValue()?.text(), "2"); }); + + await test.step("should read an object string value split across many chunks", () => { + const value = "a".repeat(64 * 1024); + const decoder = new JSONTextDecoder(); + + decoder.push(encodeText(`{"key":"${value.slice(0, 1024)}`)); + + assertEquals(decoder.readToken()?.kind, KIND.OBJECT_BEGIN); + assertEquals(decoder.readToken()?.asString(), "key"); + assertEquals(decoder.readToken(), undefined); + + for (let i = 1024; i < value.length; i += 1024) { + decoder.push(encodeText(value.slice(i, i + 1024))); + assertEquals(decoder.readToken(), undefined); + } + + decoder.push(encodeText('"}')); + decoder.end(); + + const token = decoder.readToken(); + + assertEquals(token?.kind, KIND.STRING); + assertEquals(token?.asString(), value); + assertEquals(decoder.readToken()?.kind, KIND.OBJECT_END); + assertEquals(decoder.readToken(), undefined); + }); + + await test.step("should preserve an escaped quote split across chunks", () => { + const decoder = new JSONTextDecoder(); + const first = '{"key":"say ' + "\\"; + const second = '"hi\\""}'; + + decoder.push(encodeText(first)); + + assertEquals(decoder.readToken()?.kind, KIND.OBJECT_BEGIN); + assertEquals(decoder.readToken()?.asString(), "key"); + assertEquals(decoder.readToken(), undefined); + + decoder.push(encodeText(second)); + decoder.end(); + + const token = decoder.readToken(); + + assertEquals(token?.kind, KIND.STRING); + assertEquals(token?.asString(), 'say "hi"'); + assertEquals(decoder.readToken()?.kind, KIND.OBJECT_END); + assertEquals(decoder.readToken(), undefined); + }); + + await test.step("should read a complete object value after a long string spans many chunks", () => { + const value = "b".repeat(64 * 1024); + const decoder = new JSONTextDecoder(); + + decoder.push(encodeText(`{"key":"${value.slice(0, 1024)}`)); + + assertEquals(decoder.readValue(), undefined); + + for (let i = 1024; i < value.length; i += 1024) { + decoder.push(encodeText(value.slice(i, i + 1024))); + assertEquals(decoder.readValue(), undefined); + } + + decoder.push(encodeText('"}')); + decoder.end(); + + const parsed = decoder.readValue(); + + assertEquals(parsed?.kind, KIND.OBJECT_BEGIN); + assertEquals(parsed?.text(), `{"key":"${value}"}`); + assertEquals(decoder.readValue(), undefined); + }); }); await test.step("[scenario] options", async (test) => { diff --git a/tests/wire.test.ts b/tests/wire.test.ts index a529c74..45f6b74 100644 --- a/tests/wire.test.ts +++ b/tests/wire.test.ts @@ -7,6 +7,7 @@ import { consumeSimpleNumber, consumeSimpleString, consumeString, + consumeStringResumable, consumeTrue, consumeWhitespace, } from "#src/utils/wire"; @@ -257,6 +258,65 @@ Deno.test("[utils] wire", async (test) => { }); }); + await test.step("[function] consumeStringResumable", async (test) => { + await test.step("resume scanning a string across chunks", () => { + const full = e('"hello world"'); + const partial = full.subarray(0, full.length - 1); + + const first = consumeStringResumable(partial, 0, 0); + + assertEquals(first, { + consumed: partial.length, + completed: false, + }); + + const second = consumeStringResumable(full, 0, first.consumed); + + assertEquals(second, { + consumed: full.length, + completed: true, + }); + }); + + await test.step("resume correctly when a chunk ends with a backslash", () => { + const full = e('"\\""'); + const partial = full.subarray(0, 2); + + const first = consumeStringResumable(partial, 0, 0); + + assertEquals(first, { + consumed: 1, + completed: false, + }); + + const second = consumeStringResumable(full, 0, first.consumed); + + assertEquals(second, { + consumed: full.length, + completed: true, + }); + }); + + await test.step("defer final UTF-8 validation until the closing quote is present", () => { + const full = new Uint8Array([0x22, 0x80, 0x22]); + const partial = full.subarray(0, 2); + + const first = consumeStringResumable(partial, 0, 0); + + assertEquals(first, { + consumed: partial.length, + completed: false, + }); + + const second = consumeStringResumable(full, 0, first.consumed); + + assertEquals(second, { + consumed: 0, + completed: false, + }); + }); + }); + await test.step("[function] consumeNumber", async (test) => { await test.step("consume a valid number literal", () => { const cases = [