From 87d7751f5ac1b82491cf915946476c17c17c9a65 Mon Sep 17 00:00:00 2001 From: eden <198768181+lcweden@users.noreply.github.com> Date: Sat, 16 May 2026 17:54:47 +0000 Subject: [PATCH 1/7] fix: checkEOF throws on truncated JSON by checking depth --- src/modules/decoder.ts | 8 ++++++++ src/modules/encoder.ts | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/modules/decoder.ts b/src/modules/decoder.ts index 613e31f..6637145 100644 --- a/src/modules/decoder.ts +++ b/src/modules/decoder.ts @@ -31,6 +31,10 @@ class Decoder { } checkEOF(): void { + if (this.#state.depth() > 1) { + throw new SyntaxError(`Unexpected end of input`); + } + const position = consumeWhitespace(this.#cursor.bytes, this.#cursor.previousEnd); if (!this.#cursor.needMore(position)) { @@ -38,6 +42,10 @@ class Decoder { } } + depth(): number { + return this.#state.depth(); + } + end(): void { this.#cursor.end(); } diff --git a/src/modules/encoder.ts b/src/modules/encoder.ts index 991a0c8..c269a1b 100644 --- a/src/modules/encoder.ts +++ b/src/modules/encoder.ts @@ -23,6 +23,10 @@ class Encoder { return this.#tape.bytes(); } + depth(): number { + return this.#state.depth(); + } + outputOffset(): number { return this.#tape.outputOffset(); } From 6c796764a4570cccc532d83d53276df09bc61bcd Mon Sep 17 00:00:00 2001 From: eden <198768181+lcweden@users.noreply.github.com> Date: Sat, 16 May 2026 19:13:00 +0000 Subject: [PATCH 2/7] feat: add stream encoder and decoder classes --- deno.json | 21 ++++++--- package.json | 11 +++-- src/index.ts | 4 ++ src/libs/stream-decoder.ts | 60 ++++++++++++++++++++++++ src/libs/stream-encoder.ts | 42 +++++++++++++++++ tests/e2e/round-trip.bench.ts | 49 +++++++++++++++++++ tests/e2e/round-trip.test.ts | 32 ++++++++++++- tests/integration/stream-decoder.test.ts | 57 ++++++++++++++++++++++ tests/integration/stream-encoder.test.ts | 51 ++++++++++++++++++++ 9 files changed, 315 insertions(+), 12 deletions(-) create mode 100644 src/libs/stream-decoder.ts create mode 100644 src/libs/stream-encoder.ts create mode 100644 tests/e2e/round-trip.bench.ts create mode 100644 tests/integration/stream-decoder.test.ts create mode 100644 tests/integration/stream-encoder.test.ts diff --git a/deno.json b/deno.json index 40cf98f..072f6cd 100644 --- a/deno.json +++ b/deno.json @@ -1,9 +1,10 @@ { "tasks": { - "test": "deno test --coverage --allow-all", + "bench": "deno bench --allow-all", "check": "deno check && deno fmt --check && deno lint", "format": "deno fmt", - "lint": "deno lint" + "lint": "deno lint", + "test": "deno test --coverage --allow-all" }, "imports": { "#src/*": "./src/*.ts", @@ -14,9 +15,17 @@ }, "lint": { "rules": { - "tags": ["recommended"], - "exclude": ["no-sloppy-imports"] + "tags": [ + "recommended" + ], + "exclude": [ + "no-sloppy-imports" + ] } }, - "exclude": ["node_modules/", "public/", "dist/"] -} + "exclude": [ + "node_modules/", + "public/", + "dist/" + ] +} \ No newline at end of file diff --git a/package.json b/package.json index 6a4f01d..df5be8f 100644 --- a/package.json +++ b/package.json @@ -7,16 +7,17 @@ "#src/*": "./src/*.ts" }, "scripts": { - "dev": "vite", - "test": "deno task test", "build": "tsc && vite build", - "preview": "vite preview", + "bench": "deno task bench", + "dev": "vite", "format": "deno task format", - "lint": "deno task lint" + "lint": "deno task lint", + "preview": "vite preview", + "test": "deno task test" }, "devDependencies": { "@types/node": "^25.6.0", "typescript": "~6.0.2", "vite": "^8.0.9" } -} +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index b186869..241ef62 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,9 @@ export type { JSONTextDecoderOptions } from "#src/libs/decoder"; export type { JSONTextEncoderOptions } from "#src/libs/encoder"; +export type { JSONTextDecoderStreamOptions } from "#src/libs/stream-decoder"; +export type { JSONTextEncoderStreamOptions } from "#src/libs/stream-encoder"; export { default as JSONTextDecoder } from "#src/libs/decoder"; export { default as JSONTextEncoder } from "#src/libs/encoder"; +export { default as JSONTextDecoderStream } from "#src/libs/stream-decoder"; +export { default as JSONTextEncoderStream } from "#src/libs/stream-encoder"; diff --git a/src/libs/stream-decoder.ts b/src/libs/stream-decoder.ts new file mode 100644 index 0000000..bd55246 --- /dev/null +++ b/src/libs/stream-decoder.ts @@ -0,0 +1,60 @@ +import { DEFAULT_DECODER_OPTIONS } from "#src/common/constants"; +import Decoder from "#src/modules/decoder"; +import type Token from "#src/modules/token"; +import type { DecoderOptions } from "#src/types/options"; + +type JSONTextDecoderStreamOptions = DecoderOptions; + +class JSONTextDecoderStream extends TransformStream { + constructor(options?: JSONTextDecoderStreamOptions) { + const decoder = new Decoder(new Uint8Array(), { ...DEFAULT_DECODER_OPTIONS, ...options }); + + super({ + transform(chunk, controller) { + try { + decoder.push(chunk); + + let token; + + while ((token = decoder.readToken()) !== undefined) { + controller.enqueue(token); + } + } catch (error) { + controller.error(error); + } + }, + flush(controller) { + try { + decoder.end(); + + let token; + + while ((token = decoder.readToken()) !== undefined) { + controller.enqueue(token); + } + + decoder.checkEOF(); + } catch (error) { + controller.error(error); + } + }, + }); + } + + async *[Symbol.asyncIterator](): AsyncIterableIterator { + const reader = this.readable.getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + yield value; + } + } finally { + reader.releaseLock(); + } + } +} + +export default JSONTextDecoderStream; +export type { JSONTextDecoderStreamOptions }; diff --git a/src/libs/stream-encoder.ts b/src/libs/stream-encoder.ts new file mode 100644 index 0000000..719eddc --- /dev/null +++ b/src/libs/stream-encoder.ts @@ -0,0 +1,42 @@ +import { DEFAULT_ENCODER_OPTIONS } from "#src/common/constants"; +import Encoder from "#src/modules/encoder"; +import type Token from "#src/modules/token"; +import type { EncoderOptions } from "#src/types/options"; + +type JSONTextEncoderStreamOptions = EncoderOptions; + +class JSONTextEncoderStream extends TransformStream { + constructor(options?: JSONTextEncoderStreamOptions) { + const encoder = new Encoder({ ...DEFAULT_ENCODER_OPTIONS, ...options }); + + super({ + transform(token, controller) { + try { + encoder.writeToken(token); + + const bytes = encoder.takeBytes(); + + if (bytes.length > 0) { + controller.enqueue(bytes); + } + } catch (error) { + controller.error(error); + } + }, + flush(controller) { + try { + const bytes = encoder.takeBytes(); + + if (bytes.length > 0) { + controller.enqueue(bytes); + } + } catch (error) { + controller.error(error); + } + }, + }); + } +} + +export default JSONTextEncoderStream; +export type { JSONTextEncoderStreamOptions }; diff --git a/tests/e2e/round-trip.bench.ts b/tests/e2e/round-trip.bench.ts new file mode 100644 index 0000000..0819de4 --- /dev/null +++ b/tests/e2e/round-trip.bench.ts @@ -0,0 +1,49 @@ +import { + JSONTextDecoder, + JSONTextDecoderStream, + JSONTextEncoder, + JSONTextEncoderStream, +} from "#src/index"; + +const HAR_URL = new URL("../../public/example.com.har", import.meta.url); +const bytes = await Deno.readFile(HAR_URL); + +Deno.bench({ + name: "pull", + group: "round-trip", + baseline: true, + fn() { + const decoder = new JSONTextDecoder(bytes); + const encoder = new JSONTextEncoder({ multiline: false, spaceAfterColon: false }); + + decoder.end(); + + let token; + while ((token = decoder.readToken()) !== undefined) { + encoder.writeToken(token); + } + + decoder.checkEOF(); + }, +}); + +Deno.bench({ + name: "stream", + group: "round-trip", + async fn() { + const inputStream = new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); + + const outputStream = inputStream + .pipeThrough(new JSONTextDecoderStream()) + .pipeThrough(new JSONTextEncoderStream({ multiline: false, spaceAfterColon: false })); + + for await (const _ of outputStream) { + // drain + } + }, +}); diff --git a/tests/e2e/round-trip.test.ts b/tests/e2e/round-trip.test.ts index 3098231..b12deed 100644 --- a/tests/e2e/round-trip.test.ts +++ b/tests/e2e/round-trip.test.ts @@ -1,4 +1,9 @@ -import { JSONTextDecoder, JSONTextEncoder } from "#src/index"; +import { + JSONTextDecoder, + JSONTextDecoderStream, + JSONTextEncoder, + JSONTextEncoderStream, +} from "#src/index"; import { decodeText } from "#src/utils/text"; import { assertEquals } from "#std/assert"; @@ -24,4 +29,29 @@ Deno.test("[e2e] round-trip", async (test) => { JSON.parse(decodeText(bytes)), ); }); + + await test.step("should round-trip example.com.har through stream pipeline", async () => { + const chunks: Uint8Array[] = []; + const file = await Deno.open(HAR_URL, { read: true }); + const stream = file.readable + .pipeThrough(new JSONTextDecoderStream()) + .pipeThrough(new JSONTextEncoderStream({ multiline: false, spaceAfterColon: false })); + + for await (const chunk of stream) { + chunks.push(chunk); + } + + const output = new Uint8Array(chunks.reduce((acc, c) => acc + c.length, 0)); + let offset = 0; + + for (const chunk of chunks) { + output.set(chunk, offset); + offset += chunk.length; + } + + assertEquals( + JSON.parse(decodeText(output)), + JSON.parse(decodeText(await Deno.readFile(HAR_URL))), + ); + }); }); diff --git a/tests/integration/stream-decoder.test.ts b/tests/integration/stream-decoder.test.ts new file mode 100644 index 0000000..968f7e5 --- /dev/null +++ b/tests/integration/stream-decoder.test.ts @@ -0,0 +1,57 @@ +import { JSONTextDecoderStream } from "#src/index"; +import type Token from "#src/modules/token"; +import { encodeText } from "#src/utils/text"; +import { assertEquals, assertRejects } from "#std/assert"; + +Deno.test("[integration] JSONTextDecoderStream", async (test) => { + await test.step("should emit a number token deferred to flush", async () => { + const stream = new JSONTextDecoderStream(); + const tokens: Token[] = []; + + const writing = (async () => { + const writer = stream.writable.getWriter(); + for (const chunk of [encodeText("42")]) await writer.write(chunk); + await writer.close(); + })().catch(() => {}); + + for await (const token of stream) { + tokens.push(token); + } + + await writing; + + assertEquals(tokens.length, 1); + }); + + await test.step("should error the stream on invalid JSON", async () => { + const stream = new JSONTextDecoderStream(); + + const writing = (async () => { + const writer = stream.writable.getWriter(); + for (const chunk of [encodeText("{invalid}")]) await writer.write(chunk); + await writer.close(); + })().catch(() => {}); + + await assertRejects(async () => { + for await (const _ of stream) { /* drain */ } + }); + + await writing; + }); + + await test.step("should error the stream on truncated JSON", async () => { + const stream = new JSONTextDecoderStream(); + + const writing = (async () => { + const writer = stream.writable.getWriter(); + for (const chunk of [encodeText('{"a":1')]) await writer.write(chunk); + await writer.close(); + })().catch(() => {}); + + await assertRejects(async () => { + for await (const _ of stream) { /* drain */ } + }); + + await writing; + }); +}); diff --git a/tests/integration/stream-encoder.test.ts b/tests/integration/stream-encoder.test.ts new file mode 100644 index 0000000..4595670 --- /dev/null +++ b/tests/integration/stream-encoder.test.ts @@ -0,0 +1,51 @@ +import { JSONTextEncoderStream } from "#src/index"; +import Token from "#src/modules/token"; +import { decodeText } from "#src/utils/text"; +import { assertEquals, assertRejects } from "#std/assert"; + +Deno.test("[integration] JSONTextEncoderStream", async (test) => { + await test.step("should encode tokens to bytes", async () => { + const stream = new JSONTextEncoderStream({ multiline: false }); + const chunks: Uint8Array[] = []; + + const writing = (async () => { + const writer = stream.writable.getWriter(); + await writer.write(Token.fromText("[")); + await writer.write(Token.fromNumber(1)); + await writer.write(Token.fromText("]")); + await writer.close(); + })(); + + for await (const chunk of stream.readable) { + chunks.push(chunk); + } + + await writing; + + const output = chunks.reduce((acc, c) => { + const merged = new Uint8Array(acc.length + c.length); + merged.set(acc); + merged.set(c, acc.length); + return merged; + }, new Uint8Array()); + + assertEquals(decodeText(output), "[1]"); + }); + + await test.step("should error the stream on invalid token sequence", async () => { + const stream = new JSONTextEncoderStream(); + + const writing = (async () => { + const writer = stream.writable.getWriter(); + await writer.write(Token.fromText("{")); + await writer.write(Token.fromNumber(42)); + await writer.close(); + })().catch(() => {}); + + await assertRejects(async () => { + for await (const _ of stream.readable) { /* drain */ } + }); + + await writing; + }); +}); From 751d91f416a8b853b8f8f2dd4bd124955ffdf823 Mon Sep 17 00:00:00 2001 From: eden <198768181+lcweden@users.noreply.github.com> Date: Sun, 17 May 2026 12:34:12 +0000 Subject: [PATCH 3/7] feat: implement JSONTextSelectorStream and Path module --- deno.json | 2 +- package.json | 2 +- src/common/constants.ts | 20 ++ src/index.ts | 2 + src/libs/stream-selector.ts | 97 +++++++++ src/modules/automaton.ts | 2 +- src/modules/path.ts | 378 ++++++++++++++++++++++++++++++++++++ src/modules/pointer.ts | 28 ++- src/modules/state.ts | 12 +- src/types/path.ts | 11 ++ tests/path.test.ts | 114 +++++++++++ 11 files changed, 655 insertions(+), 13 deletions(-) create mode 100644 src/libs/stream-selector.ts create mode 100644 src/modules/path.ts create mode 100644 src/types/path.ts create mode 100644 tests/path.test.ts diff --git a/deno.json b/deno.json index 072f6cd..5e12755 100644 --- a/deno.json +++ b/deno.json @@ -28,4 +28,4 @@ "public/", "dist/" ] -} \ No newline at end of file +} diff --git a/package.json b/package.json index df5be8f..0c28cd3 100644 --- a/package.json +++ b/package.json @@ -20,4 +20,4 @@ "typescript": "~6.0.2", "vite": "^8.0.9" } -} \ No newline at end of file +} diff --git a/src/common/constants.ts b/src/common/constants.ts index cf1c59f..d1da5e3 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -5,7 +5,10 @@ const ASCII = { SPACE: 0x20, QUOTE: 0x22, NUMBER_SIGN: 0x23, + DOLLAR_SIGN: 0x24, AMPERSAND: 0x26, + SINGLE_QUOTE: 0x27, + ASTERISK: 0x2A, PLUS: 0x2B, COMMA: 0x2C, MINUS: 0x2D, @@ -23,7 +26,9 @@ const ASCII = { COLON: 0x3A, OPEN_ANGLED_BRACKET: 0x3C, CLOSE_ANGLED_BRACKET: 0x3E, + UPPER_CASE_A: 0x41, UPPER_CASE_E: 0x45, + UPPER_CASE_Z: 0x5A, OPENING_BRACKET: 0x5B, BACKSLASH: 0x5C, CLOSING_BRACKET: 0x5D, @@ -38,6 +43,7 @@ const ASCII = { LOWER_CASE_S: 0x73, LOWER_CASE_T: 0x74, LOWER_CASE_U: 0x75, + LOWER_CASE_Z: 0x7A, OPENING_BRACE: 0x7B, CLOSING_BRACE: 0x7D, DELETE: 0x7F, @@ -81,11 +87,25 @@ const DEFAULT_ENCODER_OPTIONS = { indentPrefix: "", } as const; +const SELECTOR = { + NAME: "NAME", + WILDCARD: "WILDCARD", + INDEX: "INDEX", + ARRAY_SLICE: "ARRAY_SLICE", +} as const; + +const SEGMENT = { + CHILD: "CHILD", + DESCENDANT: "DESCENDANT", +} as const; + export { ASCII, DEFAULT_DECODER_OPTIONS, DEFAULT_ENCODER_OPTIONS, KIND, MAX_NESTING_DEPTH, + SEGMENT, + SELECTOR, UNICODE, }; diff --git a/src/index.ts b/src/index.ts index 241ef62..9df2476 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,8 +2,10 @@ export type { JSONTextDecoderOptions } from "#src/libs/decoder"; export type { JSONTextEncoderOptions } from "#src/libs/encoder"; export type { JSONTextDecoderStreamOptions } from "#src/libs/stream-decoder"; export type { JSONTextEncoderStreamOptions } from "#src/libs/stream-encoder"; +export type { JSONTextSelectorStreamOptions } from "#src/libs/stream-selector"; export { default as JSONTextDecoder } from "#src/libs/decoder"; export { default as JSONTextEncoder } from "#src/libs/encoder"; export { default as JSONTextDecoderStream } from "#src/libs/stream-decoder"; export { default as JSONTextEncoderStream } from "#src/libs/stream-encoder"; +export { default as JSONTextSelectorStream } from "#src/libs/stream-selector"; diff --git a/src/libs/stream-selector.ts b/src/libs/stream-selector.ts new file mode 100644 index 0000000..c44cb28 --- /dev/null +++ b/src/libs/stream-selector.ts @@ -0,0 +1,97 @@ +import { DEFAULT_DECODER_OPTIONS, KIND } from "#src/common/constants"; +import Decoder from "#src/modules/decoder"; +import Path from "#src/modules/path"; +import type Value from "#src/modules/value"; +import type { DecoderOptions } from "#src/types/options"; +import { encodeText } from "#src/utils/text"; + +type JSONTextSelectorStreamOptions = DecoderOptions; + +class JSONTextSelectorStream extends TransformStream { + constructor(query: string, options?: JSONTextSelectorStreamOptions) { + const path = new Path(encodeText(query)); + const decoder = new Decoder(new Uint8Array(), { ...DEFAULT_DECODER_OPTIONS, ...options }); + + super({ + transform(chunk, controller) { + try { + decoder.push(chunk); + + while (true) { + const kind = decoder.peekKind(); + + if (kind === undefined) { + break; + } + + if (kind === KIND.OBJECT_END || kind === KIND.ARRAY_END) { + decoder.readToken(); + + continue; + } + + const pointer = decoder.stackPointer(1); + + if (path.match(pointer.tokens)) { + const value = decoder.readValue(); + + if (value === undefined) { + break; + } + + controller.enqueue(value); + } else { + if (decoder.readToken() === undefined) { + break; + } + } + } + } catch (error) { + controller.error(error); + } + }, + flush(controller) { + try { + decoder.end(); + + while (true) { + const kind = decoder.peekKind(); + + if (kind === undefined) { + break; + } + + if (kind === KIND.OBJECT_END || kind === KIND.ARRAY_END) { + decoder.readToken(); + + continue; + } + + const pointer = decoder.stackPointer(1); + + if (path.match(pointer.tokens)) { + const value = decoder.readValue(); + + if (value === undefined) { + break; + } + + controller.enqueue(value); + } else { + if (decoder.readToken() === undefined) { + break; + } + } + } + + decoder.checkEOF(); + } catch (error) { + controller.error(error); + } + }, + }); + } +} + +export default JSONTextSelectorStream; +export type { JSONTextSelectorStreamOptions }; diff --git a/src/modules/automaton.ts b/src/modules/automaton.ts index 702628d..cb2f291 100644 --- a/src/modules/automaton.ts +++ b/src/modules/automaton.ts @@ -39,7 +39,7 @@ class Automaton { return this.#stack.length + 1; } - index(index: number): Entry { + getEntry(index: number): Entry { if (index === this.#stack.length) { return this.#last; } diff --git a/src/modules/path.ts b/src/modules/path.ts new file mode 100644 index 0000000..2394895 --- /dev/null +++ b/src/modules/path.ts @@ -0,0 +1,378 @@ +import { ASCII, SEGMENT, SELECTOR } from "#src/common/constants"; +import type { Segment, Selector } from "#src/types/path"; +import { decodeText } from "#src/utils/text"; +import { consumeNumber, consumeWhitespace } from "#src/utils/wire"; + +class Path { + #bytes: Uint8Array; + #segments: Array; + + constructor(bytes: Uint8Array) { + this.#bytes = bytes; + this.#segments = this.#parse(bytes); + } + + match(tokens: string[]): boolean { + if (this.#segments.length === 0) { + return tokens.length === 0; + } + + return this.#match(tokens, 0, 0); + } + + toString(): string { + return decodeText(this.#bytes); + } + + #match(tokens: string[], segmentIndex: number, tokenIndex: number): boolean { + if (segmentIndex === this.#segments.length) { + return tokenIndex === tokens.length; + } + + if (tokenIndex === tokens.length) { + return false; + } + + const segment = this.#segments[segmentIndex]; + + switch (segment.type) { + case SEGMENT.CHILD: { + if (this.#isFulfilled(tokens[tokenIndex], segment.selectors)) { + return this.#match(tokens, segmentIndex + 1, tokenIndex + 1); + } + + return false; + } + case SEGMENT.DESCENDANT: { + for (let index = tokenIndex; index < tokens.length; index++) { + if (this.#isFulfilled(tokens[index], segment.selectors)) { + if (this.#match(tokens, segmentIndex + 1, index + 1)) { + return true; + } + } + } + + return false; + } + } + } + + #isFulfilled(token: string, selectors: Selector[]): boolean { + for (const selector of selectors) { + if (selector.type === SELECTOR.WILDCARD) { + return true; + } + + if (selector.type === SELECTOR.NAME) { + if (selector.value === token) { + return true; + } + } + + if (selector.type === SELECTOR.INDEX) { + const index = Number.parseInt(token, 10); + + if (!Number.isNaN(index) && index === selector.value) { + return true; + } + } + + if (selector.type === SELECTOR.ARRAY_SLICE) { + const index = Number.parseInt(token, 10); + + if (Number.isNaN(index)) { + continue; + } + + const start = selector.start !== undefined ? selector.start : 0; + const step = selector.step !== undefined ? selector.step : 1; + const end = selector.end; + + if (index < start) { + continue; + } + + if (end !== undefined && index >= end) { + continue; + } + + if ((index - start) % step === 0) { + return true; + } + } + } + + return false; + } + + #parse(bytes: Uint8Array): Array { + let index = 0; + + index = consumeWhitespace(bytes, index); + + if (index >= bytes.length || bytes[index] !== ASCII.DOLLAR_SIGN) { + throw new SyntaxError("path must start with '$'"); + } + + index++; + + const segments: Segment[] = []; + let isDescendant = false; + + while (index < bytes.length) { + index = consumeWhitespace(bytes, index); + + if (index >= bytes.length) { + break; + } + + const byte = bytes[index]; + + if (byte === ASCII.DOT) { + index++; + + if (index < bytes.length && bytes[index] === ASCII.DOT) { + isDescendant = true; + index++; + } + + if (index >= bytes.length) { + throw new SyntaxError("unexpected end of path"); + } + + const byte = bytes[index]; + + if (byte === ASCII.ASTERISK) { + const type = isDescendant ? SEGMENT.DESCENDANT : SEGMENT.CHILD; + const selectors = [{ type: SELECTOR.WILDCARD }]; + const segment = { type, selectors }; + + segments.push(segment); + isDescendant = false; + index++; + } else if (byte === ASCII.OPENING_BRACKET) { + if (!isDescendant) { + throw new SyntaxError("unexpected '[' after '.'"); + } + + continue; + } else { + const start = index; + const isValidChar = (byte: number, index: number) => { + if ( + (byte >= ASCII.UPPER_CASE_A && byte <= ASCII.UPPER_CASE_Z) || + (byte >= ASCII.LOWER_CASE_A && byte <= ASCII.LOWER_CASE_Z) || + byte === 0x5f || + byte >= 0x80 + ) { + return true; + } + + if (index !== start && byte >= ASCII.DIGIT_0 && byte <= ASCII.DIGIT_9) { + return true; + } + + return false; + }; + + while (index < bytes.length && isValidChar(bytes[index], index)) { + index++; + } + + if (index === start) { + throw new SyntaxError("expected a name after '.'"); + } + + const type = isDescendant ? SEGMENT.DESCENDANT : SEGMENT.CHILD; + const value = decodeText(bytes.subarray(start, index)); + const selectors = [{ type: SELECTOR.NAME, value }]; + const segment = { type, selectors }; + + segments.push(segment); + isDescendant = false; + } + } else if (byte === ASCII.OPENING_BRACKET) { + const selectors = []; + + index++; + + while (index < bytes.length) { + index = consumeWhitespace(bytes, index); + + if (index >= bytes.length) { + break; + } + + const byte = bytes[index]; + + if (byte === ASCII.CLOSING_BRACKET) { + if (selectors.length === 0) { + throw new SyntaxError("empty bracket selection is not allowed"); + } + + break; + } + + if (byte === ASCII.ASTERISK) { + const type = SELECTOR.WILDCARD; + const selector = { type }; + + selectors.push(selector); + index++; + } else if (byte === ASCII.SINGLE_QUOTE || byte === ASCII.QUOTE) { + const quote = byte; + + index++; + + const start = index; + let inEscape = false; + + while (index < bytes.length) { + if (inEscape) { + inEscape = false; + index++; + + continue; + } + + if (bytes[index] === ASCII.BACKSLASH) { + inEscape = true; + index++; + + continue; + } + + if (bytes[index] === quote) { + break; + } + + index++; + } + + if (index >= bytes.length) { + throw new SyntaxError("unterminated string literal"); + } + + const type = SELECTOR.NAME; + const value = decodeText(bytes.subarray(start, index)); + const selector = { type, value }; + + selectors.push(selector); + index++; + } else if ( + byte === ASCII.MINUS || + (byte >= ASCII.DIGIT_0 && byte <= ASCII.DIGIT_9) || + byte === ASCII.COLON + ) { + const size = consumeNumber(bytes, index); + let start; + + if (size > 0) { + start = Number.parseInt(decodeText(bytes.subarray(index, index + size)), 10); + index += size; + } + + index = consumeWhitespace(bytes, index); + + if (index < bytes.length && bytes[index] !== ASCII.COLON) { + if (start === undefined) { + throw new SyntaxError("unexpected token in bracket"); + } + + if (start < 0) { + throw new SyntaxError("negative index is not supported"); + } + + const type = SELECTOR.INDEX; + const value = start; + const selector = { type, value }; + + selectors.push(selector); + } else if (index < bytes.length && bytes[index] === ASCII.COLON) { + index++; + index = consumeWhitespace(bytes, index); + + const size = consumeNumber(bytes, index); + let end; + let step; + + if (size > 0) { + end = Number.parseInt(decodeText(bytes.subarray(index, index + size)), 10); + index += size; + } + + index = consumeWhitespace(bytes, index); + + if (index < bytes.length && bytes[index] === ASCII.COLON) { + index++; + index = consumeWhitespace(bytes, index); + + const size = consumeNumber(bytes, index); + + if (size > 0) { + step = Number.parseInt(decodeText(bytes.subarray(index, index + size)), 10); + index += size; + } + } + + if (start !== undefined && start < 0) { + throw new SyntaxError("negative slice start is not supported"); + } + + if (end !== undefined && end < 0) { + throw new SyntaxError("negative slice end is not supported"); + } + + if (step !== undefined && step < 0) { + throw new SyntaxError("negative slice step is not supported"); + } + + const type = SELECTOR.ARRAY_SLICE; + const selector = { type, start, end, step }; + + selectors.push(selector); + } else { + throw new SyntaxError("unexpected token in bracket"); + } + + index = consumeWhitespace(bytes, index); + + if (index < bytes.length && bytes[index] === ASCII.COMMA) { + index++; + } else if (index < bytes.length && bytes[index] !== ASCII.CLOSING_BRACKET) { + throw new SyntaxError("expected ',' or ']' in bracket"); + } + } else { + throw new SyntaxError(`unexpected token '${String.fromCharCode(byte)}' in bracket`); + } + + index = consumeWhitespace(bytes, index); + + if (index < bytes.length && bytes[index] === ASCII.COMMA) { + index++; + } else if (index < bytes.length && bytes[index] !== ASCII.CLOSING_BRACKET) { + throw new SyntaxError("expected ',' or ']' in bracket"); + } + } + + if (index >= bytes.length || bytes[index] !== ASCII.CLOSING_BRACKET) { + throw new SyntaxError("expected ']' in bracket"); + } + + index++; + + const type = isDescendant ? SEGMENT.DESCENDANT : SEGMENT.CHILD; + const segment = { type, selectors }; + + segments.push(segment); + isDescendant = false; + } else { + throw new SyntaxError(`unexpected token '${String.fromCharCode(byte)}'`); + } + } + + return segments; + } +} + +export default Path; diff --git a/src/modules/pointer.ts b/src/modules/pointer.ts index 87e2570..646c3aa 100644 --- a/src/modules/pointer.ts +++ b/src/modules/pointer.ts @@ -1,8 +1,24 @@ class Pointer { - #value: string; + #tokens: string[]; - constructor(value: string) { - this.#value = value; + constructor(tokens: string[]) { + this.#tokens = tokens; + } + + get tokens(): string[] { + return this.#tokens; + } + + static parse(value: string): Pointer { + if (value === "") { + return new Pointer([]); + } + + if (value[0] !== "/") { + throw new TypeError("JSON Pointer must be empty or start with '/'"); + } + + return new Pointer(value.slice(1).split("/").map(Pointer.unescapeToken)); } static unescapeToken(token: string): string { @@ -18,7 +34,11 @@ class Pointer { } toString(): string { - return this.#value; + if (this.#tokens.length === 0) { + return ""; + } + + return "/" + this.#tokens.map(Pointer.escapeToken).join("/"); } } diff --git a/src/modules/state.ts b/src/modules/state.ts index cdb58bb..3c5374e 100644 --- a/src/modules/state.ts +++ b/src/modules/state.ts @@ -78,11 +78,11 @@ class State { } stackPointer(where: -1 | 0 | 1): Pointer { - let result = ""; + const tokens: string[] = []; let depth = 0; for (let index = 1; index < this.#automaton.depth(); index++) { - const entry = this.#automaton.index(index); + const entry = this.#automaton.getEntry(index); let delta = -1; if (index === this.#automaton.depth() - 1) { @@ -91,7 +91,7 @@ class State { const isExpectingName = where > 0 && entry.needObjectName(); if (isEmpty || isNotInObject || isExpectingName) { - return new Pointer(result); + return new Pointer(tokens); } if (where > 0 && entry.isArray()) { @@ -100,14 +100,14 @@ class State { } if (entry.isObject()) { - result += "/" + Pointer.escapeToken(this.#names.getObjectName(depth)); + tokens.push(this.#names.getObjectName(depth)); depth++; } else { - result += "/" + (entry.count() + delta); + tokens.push(String(entry.count() + delta)); } } - return new Pointer(result); + return new Pointer(tokens); } } diff --git a/src/types/path.ts b/src/types/path.ts new file mode 100644 index 0000000..abbb3cc --- /dev/null +++ b/src/types/path.ts @@ -0,0 +1,11 @@ +import type { SEGMENT, SELECTOR } from "#src/common/constants"; + +export type Selector = + | { type: typeof SELECTOR.NAME; value: string } + | { type: typeof SELECTOR.WILDCARD } + | { type: typeof SELECTOR.INDEX; value: number } + | { type: typeof SELECTOR.ARRAY_SLICE; start?: number; end?: number; step?: number }; + +export type Segment = + | { type: typeof SEGMENT.CHILD; selectors: Selector[] } + | { type: typeof SEGMENT.DESCENDANT; selectors: Selector[] }; diff --git a/tests/path.test.ts b/tests/path.test.ts new file mode 100644 index 0000000..ed6a014 --- /dev/null +++ b/tests/path.test.ts @@ -0,0 +1,114 @@ +import Path from "#src/modules/path"; +import Pointer from "#src/modules/pointer"; +import { encodeText } from "#src/utils/text"; +import { assert, assertFalse, assertThrows } from "#std/assert"; + +Deno.test("[module] path", async (test) => { + await test.step("[function] constructor", async (test) => { + await test.step("should create instances of Path", () => { + const cases = [ + { expr: new Path(encodeText("$")) }, + { expr: new Path(encodeText("$.child")) }, + { expr: new Path(encodeText("$..descendant")) }, + { expr: new Path(encodeText("$.name")) }, + { expr: new Path(encodeText("$.*")) }, + { expr: new Path(encodeText("$[0]")) }, + { expr: new Path(encodeText("$[0:1]")) }, + { expr: new Path(encodeText("$['child']")) }, + { expr: new Path(encodeText('$["child"]')) }, + { expr: new Path(encodeText("$[*]")) }, + { expr: new Path(encodeText("$..*")) }, + { expr: new Path(encodeText("$..[*]")) }, + { expr: new Path(encodeText("$[10]")) }, + { expr: new Path(encodeText("$[1:]")) }, + { expr: new Path(encodeText("$[:5]")) }, + { expr: new Path(encodeText("$[:]")) }, + { expr: new Path(encodeText("$.store.branches[*].categories.electronics[0:2]")) }, + { expr: new Path(encodeText("$.store.branches[1].categories.*[0]")) }, + { expr: new Path(encodeText("$..branches[*].categories.electronics[1:3].tags[*]")) }, + ]; + + for (const { expr } of cases) { + assert(expr); + } + }); + + await test.step("should throw errors for unaccepted paths", () => { + const cases = [ + { fn: () => new Path(encodeText("$[-1]")) }, + { fn: () => new Path(encodeText("$[-10]")) }, + { fn: () => new Path(encodeText("$[-2:]")) }, + { fn: () => new Path(encodeText("$[:-1]")) }, + { fn: () => new Path(encodeText("$[-2:-1]")) }, + { fn: () => new Path(encodeText("$[?(@.price > 10)]")) }, + { fn: () => new Path(encodeText("$.['name','age']")) }, + { fn: () => new Path(encodeText("$[0,1]")) }, + { fn: () => new Path(encodeText("")) }, + { fn: () => new Path(encodeText("child")) }, + { fn: () => new Path(encodeText("$[")) }, + { fn: () => new Path(encodeText("$.[")) }, + { fn: () => new Path(encodeText("$.")) }, + { fn: () => new Path(encodeText("$...child")) }, + ]; + + for (const { fn } of cases) { + assertThrows(fn); + } + }); + }); + + await test.step("[function] match", async (test) => { + await test.step("should return true for matching paths", () => { + const cases = [ + new Path(encodeText("$")).match(new Pointer([]).tokens), + new Path(encodeText("$.foo")).match(new Pointer(["foo"]).tokens), + new Path(encodeText("$.*")).match(new Pointer(["foo"]).tokens), + new Path(encodeText("$.*")).match(new Pointer(["0"]).tokens), + new Path(encodeText("$[2]")).match(new Pointer(["2"]).tokens), + new Path(encodeText("$[1:3]")).match(new Pointer(["1"]).tokens), + new Path(encodeText("$[1:3]")).match(new Pointer(["2"]).tokens), + new Path(encodeText("$[:]")).match(new Pointer(["0"]).tokens), + new Path(encodeText("$[:]")).match(new Pointer(["99"]).tokens), + new Path(encodeText("$['child']")).match(new Pointer(["child"]).tokens), + new Path(encodeText("$.foo.bar")).match(new Pointer(["foo", "bar"]).tokens), + new Path(encodeText("$..foo")).match(new Pointer(["foo"]).tokens), + new Path(encodeText("$..foo")).match(new Pointer(["a", "foo"]).tokens), + new Path(encodeText("$..foo")).match(new Pointer(["a", "b", "foo"]).tokens), + new Path(encodeText("$..*")).match(new Pointer(["x"]).tokens), + new Path(encodeText("$..*")).match(new Pointer(["x", "y"]).tokens), + new Path(encodeText("$.a.b[*]")).match(new Pointer(["a", "b", "0"]).tokens), + new Path(encodeText("$.a.b[*]")).match(new Pointer(["a", "b", "5"]).tokens), + ]; + + for (const expr of cases) { + assert(expr); + } + }); + + await test.step("should return false for non-matching paths", () => { + const cases = [ + new Path(encodeText("$")).match(new Pointer(["foo"]).tokens), + new Path(encodeText("$.foo")).match(new Pointer(["bar"]).tokens), + new Path(encodeText("$.foo")).match(new Pointer([]).tokens), + new Path(encodeText("$.foo")).match(new Pointer(["foo", "bar"]).tokens), + new Path(encodeText("$.*")).match(new Pointer([]).tokens), + new Path(encodeText("$.*")).match(new Pointer(["foo", "bar"]).tokens), + new Path(encodeText("$[2]")).match(new Pointer(["1"]).tokens), + new Path(encodeText("$[2]")).match(new Pointer(["foo"]).tokens), + new Path(encodeText("$[1:3]")).match(new Pointer(["0"]).tokens), + new Path(encodeText("$[1:3]")).match(new Pointer(["3"]).tokens), + new Path(encodeText("$[:]")).match(new Pointer(["foo"]).tokens), + new Path(encodeText("$.foo.bar")).match(new Pointer(["foo"]).tokens), + new Path(encodeText("$.foo.bar")).match(new Pointer(["foo", "baz"]).tokens), + new Path(encodeText("$..foo")).match(new Pointer(["bar"]).tokens), + new Path(encodeText("$..foo")).match(new Pointer(["foo", "bar"]).tokens), + new Path(encodeText("$.a.b[*]")).match(new Pointer(["a", "b"]).tokens), + new Path(encodeText("$.a.b[*]")).match(new Pointer(["a", "b", "0", "name"]).tokens), + ]; + + for (const expr of cases) { + assertFalse(expr); + } + }); + }); +}); From 83ac96e3c2b7d2df07d69624e0baba9d2a34d0ba Mon Sep 17 00:00:00 2001 From: eden <198768181+lcweden@users.noreply.github.com> Date: Mon, 18 May 2026 11:25:12 +0000 Subject: [PATCH 4/7] refactor: restructure classes and implement JSONTextLineStream --- src/{libs => api}/decoder.ts | 16 ++- src/{libs => api}/encoder.ts | 12 ++- src/common/errors.ts | 8 +- src/index.ts | 18 +++- src/libs/stream-decoder.ts | 81 +++++++-------- src/libs/stream-encoder.ts | 52 ++++++---- src/libs/stream-line.ts | 52 ++++++++++ src/libs/stream-selector.ts | 122 ++++++++++++----------- src/modules/decoder.ts | 31 +++--- src/modules/encoder.ts | 7 +- src/modules/token.ts | 14 +++ src/modules/value.ts | 30 +++++- tests/e2e/round-trip.test.ts | 39 ++++++++ tests/e2e/streaming.test.ts | 2 +- tests/integration/decoder.test.ts | 18 ++-- tests/integration/stream-decoder.test.ts | 6 +- tests/value.test.ts | 66 ++++++------ 17 files changed, 369 insertions(+), 205 deletions(-) rename src/{libs => api}/decoder.ts (81%) rename src/{libs => api}/encoder.ts (79%) create mode 100644 src/libs/stream-line.ts diff --git a/src/libs/decoder.ts b/src/api/decoder.ts similarity index 81% rename from src/libs/decoder.ts rename to src/api/decoder.ts index ec79c3b..ae1a6e1 100644 --- a/src/libs/decoder.ts +++ b/src/api/decoder.ts @@ -18,6 +18,10 @@ class JSONTextDecoder { this.#decoder.checkEOF(); } + depth(): number { + return this.#decoder.depth(); + } + end(): void { this.#decoder.end(); } @@ -34,6 +38,10 @@ class JSONTextDecoder { return this.#decoder.peekKind(); } + reset(): void { + this.#decoder.reset(); + } + readToken(): Token | undefined { return this.#decoder.readToken(); } @@ -42,12 +50,12 @@ class JSONTextDecoder { return this.#decoder.readValue(); } - skipValue(): void { - this.#decoder.skipValue(); + skipValue(): boolean { + return this.#decoder.skipValue(); } - stackPointer(where: 0 | 1 | -1 = 1) { - return this.#decoder.stackPointer(where); + stackPointer(where: 0 | 1 | -1 = 1): string { + return this.#decoder.stackPointer(where).toString(); } unreadBytes(): Uint8Array { diff --git a/src/libs/encoder.ts b/src/api/encoder.ts similarity index 79% rename from src/libs/encoder.ts rename to src/api/encoder.ts index aa04dce..ac091e4 100644 --- a/src/libs/encoder.ts +++ b/src/api/encoder.ts @@ -17,12 +17,20 @@ class JSONTextEncoder { return this.#encoder.bytes(); } + depth(): number { + return this.#encoder.depth(); + } + outputOffset(): number { return this.#encoder.outputOffset(); } - stackPointer(where: 0 | 1 | -1 = 1) { - return this.#encoder.stackPointer(where); + reset(): void { + this.#encoder.reset(); + } + + stackPointer(where: 0 | 1 | -1 = 1): string { + return this.#encoder.stackPointer(where).toString(); } writeToken(token: Token): void { diff --git a/src/common/errors.ts b/src/common/errors.ts index 7023bda..84cad5b 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -1,11 +1,9 @@ -import type Pointer from "#src/modules/pointer"; - class SyntacticError extends SyntaxError { - pointer: Pointer; + pointer: string; offset: number; - constructor(message: string, pointer: Pointer, offset: number) { - const within = pointer.toString() ? ` within ${pointer.toString()}` : ""; + constructor(message: string, pointer: string, offset: number) { + const within = pointer ? ` within ${pointer}` : ""; const at = ` at offset ${offset}`; super(`${message}${within}${at}`); diff --git a/src/index.ts b/src/index.ts index 9df2476..842ff66 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,21 @@ -export type { JSONTextDecoderOptions } from "#src/libs/decoder"; -export type { JSONTextEncoderOptions } from "#src/libs/encoder"; +export type { JSONTextDecoderOptions } from "#src/api/decoder"; +export type { JSONTextEncoderOptions } from "#src/api/encoder"; export type { JSONTextDecoderStreamOptions } from "#src/libs/stream-decoder"; export type { JSONTextEncoderStreamOptions } from "#src/libs/stream-encoder"; +export type { JSONTextLineStreamOptions } from "#src/libs/stream-line"; export type { JSONTextSelectorStreamOptions } from "#src/libs/stream-selector"; -export { default as JSONTextDecoder } from "#src/libs/decoder"; -export { default as JSONTextEncoder } from "#src/libs/encoder"; +export type { Kind } from "#src/types/kind"; + +export { default as JSONTextDecoder } from "#src/api/decoder"; +export { default as JSONTextEncoder } from "#src/api/encoder"; +export { default as Token } from "#src/modules/token"; +export { default as Value } from "#src/modules/value"; + export { default as JSONTextDecoderStream } from "#src/libs/stream-decoder"; export { default as JSONTextEncoderStream } from "#src/libs/stream-encoder"; +export { default as JSONTextLineStream } from "#src/libs/stream-line"; export { default as JSONTextSelectorStream } from "#src/libs/stream-selector"; + +export { KIND } from "#src/common/constants"; +export { SyntacticError } from "#src/common/errors"; diff --git a/src/libs/stream-decoder.ts b/src/libs/stream-decoder.ts index bd55246..e60dd41 100644 --- a/src/libs/stream-decoder.ts +++ b/src/libs/stream-decoder.ts @@ -3,56 +3,51 @@ import Decoder from "#src/modules/decoder"; import type Token from "#src/modules/token"; import type { DecoderOptions } from "#src/types/options"; -type JSONTextDecoderStreamOptions = DecoderOptions; +type JSONTextDecoderStreamOptions = DecoderOptions & { + writableStrategy?: QueuingStrategy; + readableStrategy?: QueuingStrategy; +}; class JSONTextDecoderStream extends TransformStream { - constructor(options?: JSONTextDecoderStreamOptions) { - const decoder = new Decoder(new Uint8Array(), { ...DEFAULT_DECODER_OPTIONS, ...options }); - - super({ - transform(chunk, controller) { - try { - decoder.push(chunk); - - let token; - - while ((token = decoder.readToken()) !== undefined) { - controller.enqueue(token); + constructor(options: JSONTextDecoderStreamOptions = {}) { + const { writableStrategy, readableStrategy, ...rest } = options; + const decoderOptions = { ...DEFAULT_DECODER_OPTIONS, ...rest }; + const decoder = new Decoder(new Uint8Array(), decoderOptions); + + super( + { + transform(chunk, controller) { + try { + decoder.push(chunk); + + let token; + + while ((token = decoder.readToken()) !== undefined) { + controller.enqueue(token); + } + } catch (error) { + controller.error(error); } - } catch (error) { - controller.error(error); - } - }, - flush(controller) { - try { - decoder.end(); + }, + flush(controller) { + try { + decoder.end(); - let token; + let token; - while ((token = decoder.readToken()) !== undefined) { - controller.enqueue(token); - } + while ((token = decoder.readToken()) !== undefined) { + controller.enqueue(token); + } - decoder.checkEOF(); - } catch (error) { - controller.error(error); - } + decoder.checkEOF(); + } catch (error) { + controller.error(error); + } + }, }, - }); - } - - async *[Symbol.asyncIterator](): AsyncIterableIterator { - const reader = this.readable.getReader(); - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - yield value; - } - } finally { - reader.releaseLock(); - } + writableStrategy, + readableStrategy, + ); } } diff --git a/src/libs/stream-encoder.ts b/src/libs/stream-encoder.ts index 719eddc..3e53581 100644 --- a/src/libs/stream-encoder.ts +++ b/src/libs/stream-encoder.ts @@ -3,38 +3,46 @@ import Encoder from "#src/modules/encoder"; import type Token from "#src/modules/token"; import type { EncoderOptions } from "#src/types/options"; -type JSONTextEncoderStreamOptions = EncoderOptions; +type JSONTextEncoderStreamOptions = EncoderOptions & { + writableStrategy?: QueuingStrategy; + readableStrategy?: QueuingStrategy; +}; class JSONTextEncoderStream extends TransformStream { constructor(options?: JSONTextEncoderStreamOptions) { - const encoder = new Encoder({ ...DEFAULT_ENCODER_OPTIONS, ...options }); + const { writableStrategy, readableStrategy, ...rest } = options ?? {}; + const encoder = new Encoder({ ...DEFAULT_ENCODER_OPTIONS, ...rest }); - super({ - transform(token, controller) { - try { - encoder.writeToken(token); + super( + { + transform(token, controller) { + try { + encoder.writeToken(token); - const bytes = encoder.takeBytes(); + const bytes = encoder.takeBytes(); - if (bytes.length > 0) { - controller.enqueue(bytes); + if (bytes.length > 0) { + controller.enqueue(bytes); + } + } catch (error) { + controller.error(error); } - } catch (error) { - controller.error(error); - } - }, - flush(controller) { - try { - const bytes = encoder.takeBytes(); + }, + flush(controller) { + try { + const bytes = encoder.takeBytes(); - if (bytes.length > 0) { - controller.enqueue(bytes); + if (bytes.length > 0) { + controller.enqueue(bytes); + } + } catch (error) { + controller.error(error); } - } catch (error) { - controller.error(error); - } + }, }, - }); + writableStrategy, + readableStrategy, + ); } } diff --git a/src/libs/stream-line.ts b/src/libs/stream-line.ts new file mode 100644 index 0000000..b47357f --- /dev/null +++ b/src/libs/stream-line.ts @@ -0,0 +1,52 @@ +import { DEFAULT_DECODER_OPTIONS } from "#src/common/constants"; +import Decoder from "#src/modules/decoder"; +import type Value from "#src/modules/value"; +import type { DecoderOptions } from "#src/types/options"; + +type JSONTextLineStreamOptions = DecoderOptions & { + writableStrategy?: QueuingStrategy; + readableStrategy?: QueuingStrategy; +}; + +class JSONTextLineStream extends TransformStream { + constructor(options?: JSONTextLineStreamOptions) { + const { writableStrategy, readableStrategy, ...rest } = options ?? {}; + const decoder = new Decoder(new Uint8Array(), { ...DEFAULT_DECODER_OPTIONS, ...rest }); + + super( + { + transform(chunk, controller) { + try { + decoder.push(chunk); + + let value; + + while ((value = decoder.readValue()) !== undefined) { + controller.enqueue(value); + } + } catch (error) { + controller.error(error); + } + }, + flush(controller) { + try { + decoder.end(); + + let value; + + while ((value = decoder.readValue()) !== undefined) { + controller.enqueue(value); + } + } catch (error) { + controller.error(error); + } + }, + }, + writableStrategy, + readableStrategy, + ); + } +} + +export default JSONTextLineStream; +export type { JSONTextLineStreamOptions }; diff --git a/src/libs/stream-selector.ts b/src/libs/stream-selector.ts index c44cb28..dd4ad61 100644 --- a/src/libs/stream-selector.ts +++ b/src/libs/stream-selector.ts @@ -5,91 +5,99 @@ import type Value from "#src/modules/value"; import type { DecoderOptions } from "#src/types/options"; import { encodeText } from "#src/utils/text"; -type JSONTextSelectorStreamOptions = DecoderOptions; +type JSONTextSelectorStreamOptions = DecoderOptions & { + writableStrategy?: QueuingStrategy; + readableStrategy?: QueuingStrategy; +}; class JSONTextSelectorStream extends TransformStream { constructor(query: string, options?: JSONTextSelectorStreamOptions) { + const { writableStrategy, readableStrategy, ...rest } = options ?? {}; + const decoder = new Decoder(new Uint8Array(), { ...DEFAULT_DECODER_OPTIONS, ...rest }); const path = new Path(encodeText(query)); - const decoder = new Decoder(new Uint8Array(), { ...DEFAULT_DECODER_OPTIONS, ...options }); - super({ - transform(chunk, controller) { - try { - decoder.push(chunk); + super( + { + transform(chunk, controller) { + try { + decoder.push(chunk); - while (true) { - const kind = decoder.peekKind(); + while (true) { + const kind = decoder.peekKind(); - if (kind === undefined) { - break; - } + if (kind === undefined) { + break; + } - if (kind === KIND.OBJECT_END || kind === KIND.ARRAY_END) { - decoder.readToken(); + if (kind === KIND.OBJECT_END || kind === KIND.ARRAY_END) { + decoder.readToken(); - continue; - } + continue; + } - const pointer = decoder.stackPointer(1); + const pointer = decoder.stackPointer(1); - if (path.match(pointer.tokens)) { - const value = decoder.readValue(); + if (path.match(pointer.tokens)) { + const value = decoder.readValue(); - if (value === undefined) { - break; - } + if (value === undefined) { + break; + } - controller.enqueue(value); - } else { - if (decoder.readToken() === undefined) { - break; + controller.enqueue(value); + } else { + if (decoder.readToken() === undefined) { + break; + } } } + } catch (error) { + controller.error(error); } - } catch (error) { - controller.error(error); - } - }, - flush(controller) { - try { - decoder.end(); + }, + flush(controller) { + try { + decoder.end(); - while (true) { - const kind = decoder.peekKind(); + while (true) { + const kind = decoder.peekKind(); - if (kind === undefined) { - break; - } + if (kind === undefined) { + break; + } - if (kind === KIND.OBJECT_END || kind === KIND.ARRAY_END) { - decoder.readToken(); + if (kind === KIND.OBJECT_END || kind === KIND.ARRAY_END) { + decoder.readToken(); - continue; - } + continue; + } - const pointer = decoder.stackPointer(1); + const pointer = decoder.stackPointer(1); - if (path.match(pointer.tokens)) { - const value = decoder.readValue(); + if (path.match(pointer.tokens)) { + const value = decoder.readValue(); - if (value === undefined) { - break; - } + if (value === undefined) { + break; + } - controller.enqueue(value); - } else { - if (decoder.readToken() === undefined) { - break; + controller.enqueue(value); + } else { + if (decoder.readToken() === undefined) { + break; + } } } - } - decoder.checkEOF(); - } catch (error) { - controller.error(error); - } + decoder.checkEOF(); + } catch (error) { + controller.error(error); + } + }, }, - }); + writableStrategy, + readableStrategy, + ); } } diff --git a/src/modules/decoder.ts b/src/modules/decoder.ts index 6637145..a497426 100644 --- a/src/modules/decoder.ts +++ b/src/modules/decoder.ts @@ -1,6 +1,7 @@ import { ASCII, KIND } from "#src/common/constants"; import { SyntacticError } from "#src/common/errors"; import Cursor from "#src/modules/cursor"; +import type Pointer from "#src/modules/pointer"; import State from "#src/modules/state"; import Token from "#src/modules/token"; import Value from "#src/modules/value"; @@ -95,7 +96,7 @@ class Decoder { if (!kind) { const message = `invalid character`; - const pointer = this.#state.stackPointer(0); + const pointer = this.#state.stackPointer(0).toString(); const offset = this.#cursor.offsetAt(position); const error = new SyntacticError(message, pointer, offset); @@ -107,7 +108,7 @@ class Decoder { const expected = this.#state.needDelimiter(kind); if (expected !== delimiter) { - const pointer = this.#state.stackPointer(0); + const pointer = this.#state.stackPointer(0).toString(); const offset = this.#cursor.offsetAt(position); const error = new SyntacticError("invalid delimiter", pointer, offset); @@ -172,7 +173,7 @@ class Decoder { } if (error instanceof SyntaxError) { - const pointer = this.#state.stackPointer(0); + const pointer = this.#state.stackPointer(0).toString(); const offset = this.#cursor.offsetAt(start); throw new SyntacticError(error.message, pointer, offset); @@ -242,11 +243,11 @@ class Decoder { return new Value(bytes.slice()); } - skipValue(): void { - this.readValue(); + skipValue(): boolean { + return this.readValue() !== undefined; } - stackPointer(where: 0 | 1 | -1 = 1) { + stackPointer(where: 0 | 1 | -1 = 1): Pointer { return this.#state.stackPointer(where); } @@ -341,7 +342,7 @@ class Decoder { return 0; } - const pointer = this.#state.stackPointer(0); + const pointer = this.#state.stackPointer(0).toString(); const offset = this.#cursor.offsetAt(start); const error = new SyntacticError("invalid literal null", pointer, offset); @@ -361,7 +362,7 @@ class Decoder { return 0; } - const pointer = this.#state.stackPointer(0); + const pointer = this.#state.stackPointer(0).toString(); const offset = this.#cursor.offsetAt(start); const error = new SyntacticError("invalid literal true", pointer, offset); @@ -381,7 +382,7 @@ class Decoder { return 0; } - const pointer = this.#state.stackPointer(0); + const pointer = this.#state.stackPointer(0).toString(); const offset = this.#cursor.offsetAt(start); const error = new SyntacticError("invalid literal false", pointer, offset); @@ -404,11 +405,11 @@ class Decoder { } } - const bytes = this.#cursor.bytes.subarray(start, start + size); - const decoded = decodeText(bytes, { fatal: !this.#options.allowInvalidUTF8 }); - const string = JSON.parse(decoded); - if (this.#state.needObjectName()) { + const bytes = this.#cursor.bytes.subarray(start, start + size); + const decoded = decodeText(bytes, { fatal: !this.#options.allowInvalidUTF8 }); + const string = JSON.parse(decoded); + this.#state.setLast(string); } @@ -432,10 +433,6 @@ class Decoder { return 0; } - const bytes = this.#cursor.bytes.subarray(start, start + size); - const decoded = decodeText(bytes, { fatal: true }); - - JSON.parse(decoded); this.#state.appendNumber(); return size; diff --git a/src/modules/encoder.ts b/src/modules/encoder.ts index c269a1b..c58ee98 100644 --- a/src/modules/encoder.ts +++ b/src/modules/encoder.ts @@ -1,5 +1,6 @@ import { ASCII, KIND, UNICODE } from "#src/common/constants"; import { SyntacticError } from "#src/common/errors"; +import type Pointer from "#src/modules/pointer"; import State from "#src/modules/state"; import Tape from "#src/modules/tape"; import Token from "#src/modules/token"; @@ -36,7 +37,7 @@ class Encoder { this.#state = new State(this.#options); } - stackPointer(where: 0 | 1 | -1) { + stackPointer(where: 0 | 1 | -1): Pointer { return this.#state.stackPointer(where); } @@ -125,7 +126,7 @@ class Encoder { } if (error instanceof SyntaxError) { - const pointer = this.#state.stackPointer(1); + const pointer = this.#state.stackPointer(1).toString(); const offset = this.#tape.outputOffset(); throw new SyntacticError(error.message, pointer, offset); @@ -198,7 +199,7 @@ class Encoder { } if (error instanceof SyntaxError) { - const pointer = this.#state.stackPointer(1); + const pointer = this.#state.stackPointer(1).toString(); const offset = this.#tape.outputOffset(); throw new SyntacticError(error.message, pointer, offset); diff --git a/src/modules/token.ts b/src/modules/token.ts index aa74d56..a20c67d 100644 --- a/src/modules/token.ts +++ b/src/modules/token.ts @@ -63,6 +63,20 @@ class Token { return token; } + static NULL = Token.fromText("null"); + + static TRUE = Token.fromBoolean(true); + + static FALSE = Token.fromBoolean(false); + + static OBJECT_BEGIN = Token.fromText("{"); + + static OBJECT_END = Token.fromText("}"); + + static ARRAY_BEGIN = Token.fromText("["); + + static ARRAY_END = Token.fromText("]"); + clone(): Token { return new Token(this.bytes.slice()); } diff --git a/src/modules/value.ts b/src/modules/value.ts index a404c9a..397147c 100644 --- a/src/modules/value.ts +++ b/src/modules/value.ts @@ -1,5 +1,6 @@ -import { ASCII, KIND } from "#src/common/constants"; +import { ASCII, DEFAULT_DECODER_OPTIONS, KIND } from "#src/common/constants"; import Decoder from "#src/modules/decoder"; +import type Token from "#src/modules/token"; import type { Kind } from "#src/types/kind"; import { normalize } from "#src/utils/kind"; import { decodeText, encodeText } from "#src/utils/text"; @@ -38,6 +39,13 @@ class Value { return this.#bytes; } + static from(input: unknown): Value { + const json = JSON.stringify(input); + const encoded = encodeText(json); + + return new Value(encoded); + } + canonicalize(): Value { const decoder = new Decoder(this.#bytes, { allowDuplicateNames: true }); decoder.end(); @@ -69,10 +77,28 @@ class Value { } } - toText(): string { + json(): unknown { + return JSON.parse(this.text()); + } + + text(): string { return decodeText(this.bytes, { fatal: true }); } + *tokens(): Generator { + const decoder = new Decoder(this.bytes, DEFAULT_DECODER_OPTIONS); + + while (true) { + const token = decoder.readToken(); + + if (token === undefined) { + break; + } + + yield token; + } + } + #processValue(decoder: Decoder): Uint8Array { const kind = decoder.peekKind(); diff --git a/tests/e2e/round-trip.test.ts b/tests/e2e/round-trip.test.ts index b12deed..6a2da33 100644 --- a/tests/e2e/round-trip.test.ts +++ b/tests/e2e/round-trip.test.ts @@ -3,6 +3,8 @@ import { JSONTextDecoderStream, JSONTextEncoder, JSONTextEncoderStream, + JSONTextSelectorStream, + Token, } from "#src/index"; import { decodeText } from "#src/utils/text"; import { assertEquals } from "#std/assert"; @@ -54,4 +56,41 @@ Deno.test("[e2e] round-trip", async (test) => { JSON.parse(decodeText(await Deno.readFile(HAR_URL))), ); }); + + await test.step("should round-trip example.com.har through selector pipeline", async () => { + const chunks: Uint8Array[] = []; + const file = await Deno.open(HAR_URL, { read: true }); + const stream = file.readable + .pipeThrough(new JSONTextSelectorStream("$..headers")) + .pipeThrough( + new TransformStream({ + start(controller) { + controller.enqueue(Token.ARRAY_BEGIN); + }, + transform(value, controller) { + for (const token of value.tokens()) { + controller.enqueue(token); + } + }, + flush(controller) { + controller.enqueue(Token.ARRAY_END); + }, + }), + ) + .pipeThrough(new JSONTextEncoderStream({ multiline: true, spaceAfterColon: false })); + + for await (const chunk of stream) { + chunks.push(chunk); + } + + const output = new Uint8Array(chunks.reduce((acc, c) => acc + c.length, 0)); + let offset = 0; + + for (const chunk of chunks) { + output.set(chunk, offset); + offset += chunk.length; + } + + assertEquals(JSON.parse(decodeText(output)).length, 2); + }); }); diff --git a/tests/e2e/streaming.test.ts b/tests/e2e/streaming.test.ts index 3ed9756..53f912a 100644 --- a/tests/e2e/streaming.test.ts +++ b/tests/e2e/streaming.test.ts @@ -44,7 +44,7 @@ Deno.test("[e2e] streaming readValue", async (test) => { break; } - values.push(value.toText()); + values.push(value.text()); } } diff --git a/tests/integration/decoder.test.ts b/tests/integration/decoder.test.ts index cec1cdd..f8342e5 100644 --- a/tests/integration/decoder.test.ts +++ b/tests/integration/decoder.test.ts @@ -110,7 +110,7 @@ Deno.test("[integration] JSONTextDecoder", async (test) => { const value = decoder.readValue(); assertEquals(value?.kind, KIND.NUMBER); - assertEquals(value?.toText(), "42"); + assertEquals(value?.text(), "42"); }); await test.step("should read a nested array as one value", () => { @@ -123,7 +123,7 @@ Deno.test("[integration] JSONTextDecoder", async (test) => { const value = decoder.readValue(); assertEquals(value?.kind, KIND.ARRAY_BEGIN); - assertEquals(value?.toText(), "[1,2]"); + assertEquals(value?.text(), "[1,2]"); }); await test.step("should read a nested object as one value", () => { @@ -136,7 +136,7 @@ Deno.test("[integration] JSONTextDecoder", async (test) => { const value = decoder.readValue(); assertEquals(value?.kind, KIND.OBJECT_BEGIN); - assertEquals(value?.toText(), '{"a":1}'); + assertEquals(value?.text(), '{"a":1}'); }); await test.step("should read elements inside an outer array one at a time", () => { @@ -147,8 +147,8 @@ Deno.test("[integration] JSONTextDecoder", async (test) => { decoder.end(); decoder.readToken(); - assertEquals(decoder.readValue()?.toText(), "1"); - assertEquals(decoder.readValue()?.toText(), '"two"'); + assertEquals(decoder.readValue()?.text(), "1"); + assertEquals(decoder.readValue()?.text(), '"two"'); assertEquals(decoder.readValue()?.kind, KIND.ARRAY_BEGIN); decoder.readToken(); @@ -195,7 +195,7 @@ Deno.test("[integration] JSONTextDecoder", async (test) => { decoder.readToken(); decoder.skipValue(); - assertEquals(decoder.readValue()?.toText(), "2"); + assertEquals(decoder.readValue()?.text(), "2"); }); await test.step("should skip a nested structure and allow reading the next value", () => { @@ -207,7 +207,7 @@ Deno.test("[integration] JSONTextDecoder", async (test) => { decoder.readToken(); decoder.skipValue(); - assertEquals(decoder.readValue()?.toText(), "3"); + assertEquals(decoder.readValue()?.text(), "3"); }); }); @@ -241,8 +241,8 @@ Deno.test("[integration] JSONTextDecoder", async (test) => { decoder.push(encoded.subarray(2)); decoder.end(); - assertEquals(decoder.readValue()?.toText(), "1"); - assertEquals(decoder.readValue()?.toText(), "2"); + assertEquals(decoder.readValue()?.text(), "1"); + assertEquals(decoder.readValue()?.text(), "2"); }); }); diff --git a/tests/integration/stream-decoder.test.ts b/tests/integration/stream-decoder.test.ts index 968f7e5..e5df894 100644 --- a/tests/integration/stream-decoder.test.ts +++ b/tests/integration/stream-decoder.test.ts @@ -14,7 +14,7 @@ Deno.test("[integration] JSONTextDecoderStream", async (test) => { await writer.close(); })().catch(() => {}); - for await (const token of stream) { + for await (const token of stream.readable) { tokens.push(token); } @@ -33,7 +33,7 @@ Deno.test("[integration] JSONTextDecoderStream", async (test) => { })().catch(() => {}); await assertRejects(async () => { - for await (const _ of stream) { /* drain */ } + for await (const _ of stream.readable) { /* drain */ } }); await writing; @@ -49,7 +49,7 @@ Deno.test("[integration] JSONTextDecoderStream", async (test) => { })().catch(() => {}); await assertRejects(async () => { - for await (const _ of stream) { /* drain */ } + for await (const _ of stream.readable) { /* drain */ } }); await writing; diff --git a/tests/value.test.ts b/tests/value.test.ts index c3605e7..4023538 100644 --- a/tests/value.test.ts +++ b/tests/value.test.ts @@ -7,12 +7,12 @@ Deno.test("[module] value", async (test) => { await test.step("[function] constructor", async (test) => { await test.step("should return the correct string representation", () => { const cases = [ - { actual: new Value(e('"hello"')).toText(), expected: '"hello"' }, - { actual: new Value(e("true")).toText(), expected: "true" }, - { actual: new Value(e("false")).toText(), expected: "false" }, - { actual: new Value(e("null")).toText(), expected: "null" }, - { actual: new Value(e("{}")).toText(), expected: "{}" }, - { actual: new Value(e("[]")).toText(), expected: "[]" }, + { actual: new Value(e('"hello"')).text(), expected: '"hello"' }, + { actual: new Value(e("true")).text(), expected: "true" }, + { actual: new Value(e("false")).text(), expected: "false" }, + { actual: new Value(e("null")).text(), expected: "null" }, + { actual: new Value(e("{}")).text(), expected: "{}" }, + { actual: new Value(e("[]")).text(), expected: "[]" }, ]; for (const { actual, expected } of cases) { @@ -65,23 +65,23 @@ Deno.test("[module] value", async (test) => { await test.step("should sort object keys lexicographically", () => { const cases = [ { - actual: new Value(e('{"b":1,"a":2}')).canonicalize().toText(), + actual: new Value(e('{"b":1,"a":2}')).canonicalize().text(), expected: '{"a":2,"b":1}', }, { - actual: new Value(e('{"a":1,"b":2}')).canonicalize().toText(), + actual: new Value(e('{"a":1,"b":2}')).canonicalize().text(), expected: '{"a":1,"b":2}', }, { - actual: new Value(e('{"z":3,"y":2,"x":1}')).canonicalize().toText(), + actual: new Value(e('{"z":3,"y":2,"x":1}')).canonicalize().text(), expected: '{"x":1,"y":2,"z":3}', }, { - actual: new Value(e('{"b":{"z":1,"a":2},"a":0}')).canonicalize().toText(), + actual: new Value(e('{"b":{"z":1,"a":2},"a":0}')).canonicalize().text(), expected: '{"a":0,"b":{"a":2,"z":1}}', }, { - actual: new Value(e("[3,1,2,3,2,1]")).canonicalize().toText(), + actual: new Value(e("[3,1,2,3,2,1]")).canonicalize().text(), expected: "[3,1,2,3,2,1]", }, ]; @@ -93,14 +93,14 @@ Deno.test("[module] value", async (test) => { await test.step("should normalize number representation", () => { const cases = [ - { actual: new Value(e("1.0")).canonicalize().toText(), expected: "1" }, - { actual: new Value(e("0.000")).canonicalize().toText(), expected: "0" }, - { actual: new Value(e("-0.00")).canonicalize().toText(), expected: "0" }, - { actual: new Value(e("3.14000")).canonicalize().toText(), expected: "3.14" }, - { actual: new Value(e("1e+0")).canonicalize().toText(), expected: "1" }, - { actual: new Value(e("1e-0")).canonicalize().toText(), expected: "1" }, - { actual: new Value(e("1e+1")).canonicalize().toText(), expected: "10" }, - { actual: new Value(e("1e-1")).canonicalize().toText(), expected: "0.1" }, + { actual: new Value(e("1.0")).canonicalize().text(), expected: "1" }, + { actual: new Value(e("0.000")).canonicalize().text(), expected: "0" }, + { actual: new Value(e("-0.00")).canonicalize().text(), expected: "0" }, + { actual: new Value(e("3.14000")).canonicalize().text(), expected: "3.14" }, + { actual: new Value(e("1e+0")).canonicalize().text(), expected: "1" }, + { actual: new Value(e("1e-0")).canonicalize().text(), expected: "1" }, + { actual: new Value(e("1e+1")).canonicalize().text(), expected: "10" }, + { actual: new Value(e("1e-1")).canonicalize().text(), expected: "0.1" }, ]; for (const { actual, expected } of cases) { @@ -110,8 +110,8 @@ Deno.test("[module] value", async (test) => { await test.step("should handle empty objects and arrays", () => { const cases = [ - { actual: new Value(e("{}")).canonicalize().toText(), expected: "{}" }, - { actual: new Value(e("[]")).canonicalize().toText(), expected: "[]" }, + { actual: new Value(e("{}")).canonicalize().text(), expected: "{}" }, + { actual: new Value(e("[]")).canonicalize().text(), expected: "[]" }, ]; for (const { actual, expected } of cases) { @@ -121,10 +121,10 @@ Deno.test("[module] value", async (test) => { await test.step("should return scalar values unchanged", () => { const cases = [ - { actual: new Value(e('"a"')).canonicalize().toText(), expected: '"a"' }, - { actual: new Value(e("true")).canonicalize().toText(), expected: "true" }, - { actual: new Value(e("false")).canonicalize().toText(), expected: "false" }, - { actual: new Value(e("null")).canonicalize().toText(), expected: "null" }, + { actual: new Value(e('"a"')).canonicalize().text(), expected: '"a"' }, + { actual: new Value(e("true")).canonicalize().text(), expected: "true" }, + { actual: new Value(e("false")).canonicalize().text(), expected: "false" }, + { actual: new Value(e("null")).canonicalize().text(), expected: "null" }, ]; for (const { actual, expected } of cases) { @@ -186,16 +186,16 @@ Deno.test("[module] value", async (test) => { }); }); - await test.step("[function] toText", async (test) => { + await test.step("[function] text", async (test) => { await test.step("should return the raw bytes decoded as a UTF-8 string", () => { const cases = [ - { actual: new Value(e('"hello"')).toText(), expected: '"hello"' }, - { actual: new Value(e("42")).toText(), expected: "42" }, - { actual: new Value(e("true")).toText(), expected: "true" }, - { actual: new Value(e("false")).toText(), expected: "false" }, - { actual: new Value(e("null")).toText(), expected: "null" }, - { actual: new Value(e('{"a":1}')).toText(), expected: '{"a":1}' }, - { actual: new Value(e("[1,2]")).toText(), expected: "[1,2]" }, + { actual: new Value(e('"hello"')).text(), expected: '"hello"' }, + { actual: new Value(e("42")).text(), expected: "42" }, + { actual: new Value(e("true")).text(), expected: "true" }, + { actual: new Value(e("false")).text(), expected: "false" }, + { actual: new Value(e("null")).text(), expected: "null" }, + { actual: new Value(e('{"a":1}')).text(), expected: '{"a":1}' }, + { actual: new Value(e("[1,2]")).text(), expected: "[1,2]" }, ]; for (const { actual, expected } of cases) { From a9124a854c92294e3aa3d483fe6b0901685916cd Mon Sep 17 00:00:00 2001 From: eden <198768181+lcweden@users.noreply.github.com> Date: Mon, 18 May 2026 14:26:15 +0000 Subject: [PATCH 5/7] test: add e2e and integration tests for streaming, round-trip, path, and correctness --- .github/workflows/ci.yml | 4 +- tests/e2e/correctness.test.ts | 180 +++++++++++++++++++ tests/e2e/path.test.ts | 155 +++++++++++++++++ tests/e2e/round-trip.bench.ts | 49 ------ tests/e2e/round-trip.test.ts | 241 ++++++++++++++++---------- tests/e2e/streaming.test.ts | 94 +++++----- tests/integration/stream-line.test.ts | 68 ++++++++ 7 files changed, 607 insertions(+), 184 deletions(-) create mode 100644 tests/e2e/correctness.test.ts create mode 100644 tests/e2e/path.test.ts delete mode 100644 tests/e2e/round-trip.bench.ts create mode 100644 tests/integration/stream-line.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bd0cd9..b216abf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Setup - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Deno uses: denoland/setup-deno@v2 @@ -28,3 +28,5 @@ jobs: - name: Test run: deno task test + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/tests/e2e/correctness.test.ts b/tests/e2e/correctness.test.ts new file mode 100644 index 0000000..1da1ec9 --- /dev/null +++ b/tests/e2e/correctness.test.ts @@ -0,0 +1,180 @@ +import { KIND } from "#src/common/constants"; +import { JSONTextDecoderStream, JSONTextLineStream, Token } from "#src/index"; +import { assert, assertEquals } from "#std/assert"; + +const GITHUB_TOKEN = Deno.env.get("GITHUB_TOKEN"); +const FIXTURE_BASE = "https://github.com/lcweden/jsontext/releases/download/fixtures"; + +Deno.test("[e2e] correctness", async (test) => { + await test.step("[fixture] edge_cases.json.gz", async (test) => { + await test.step("should have balanced structural tokens", async () => { + const input = `${FIXTURE_BASE}/edge_cases.json.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextDecoderStream(); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let objectBegin = 0, objectEnd = 0, arrayBegin = 0, arrayEnd = 0; + + for await (const token of stream) { + if (token.kind === KIND.OBJECT_BEGIN) objectBegin++; + else if (token.kind === KIND.OBJECT_END) objectEnd++; + else if (token.kind === KIND.ARRAY_BEGIN) arrayBegin++; + else if (token.kind === KIND.ARRAY_END) arrayEnd++; + } + + assertEquals(objectBegin, objectEnd); + assertEquals(arrayBegin, arrayEnd); + }); + + await test.step("should handle 60-level deep nesting without error", async () => { + const input = `${FIXTURE_BASE}/edge_cases.json.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextDecoderStream(); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let currentDepth = 0, maxDepth = 0; + + for await (const token of stream) { + if (token.kind === KIND.OBJECT_BEGIN || token.kind === KIND.ARRAY_BEGIN) { + maxDepth = Math.max(maxDepth, ++currentDepth); + } else if (token.kind === KIND.OBJECT_END || token.kind === KIND.ARRAY_END) { + currentDepth--; + } + } + + assert(maxDepth >= 60); + assert(maxDepth < 10000); + }); + + await test.step("should round-trip string tokens losslessly via asString()", async () => { + const input = `${FIXTURE_BASE}/edge_cases.json.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextDecoderStream(); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + for await (const token of stream) { + if (token.kind !== KIND.STRING) continue; + + const str = token.asString(); + + assertEquals(Token.fromString(str).asString(), str); + } + }); + + await test.step("should decode all number tokens without throwing", async () => { + const input = `${FIXTURE_BASE}/edge_cases.json.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextDecoderStream(); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + for await (const token of stream) { + if (token.kind !== KIND.NUMBER) continue; + + token.asNumber(); + } + }); + + await test.step("should contain at least one 1200-char string", async () => { + const input = `${FIXTURE_BASE}/edge_cases.json.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextDecoderStream(); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let found = false; + + for await (const token of stream) { + if (token.kind === KIND.STRING && token.asString().length === 1200) { + found = true; + } + } + + assert(found); + }); + + await test.step("should canonicalize idempotently", async () => { + const input = `${FIXTURE_BASE}/edge_cases.json.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextLineStream(); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + for await (const value of stream) { + const once = value.canonicalize(); + const twice = once.canonicalize(); + + assertEquals(once.text(), twice.text()); + } + }); + }); +}); diff --git a/tests/e2e/path.test.ts b/tests/e2e/path.test.ts new file mode 100644 index 0000000..0f9fba0 --- /dev/null +++ b/tests/e2e/path.test.ts @@ -0,0 +1,155 @@ +import { KIND } from "#src/common/constants"; +import { JSONTextSelectorStream } from "#src/index"; +import { assert, assertEquals } from "#std/assert"; + +const GITHUB_TOKEN = Deno.env.get("GITHUB_TOKEN"); +const FIXTURE_BASE = "https://github.com/lcweden/jsontext/releases/download/fixtures"; + +Deno.test("[e2e] path", async (test) => { + await test.step("[fixture] json_bus.json.gz", async (test) => { + await test.step("should emit object values from $.features[*]", async () => { + const input = `${FIXTURE_BASE}/json_bus.json.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextSelectorStream("$.features[*]"); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let count = 0; + + for await (const value of stream) { + assertEquals(value.kind, KIND.OBJECT_BEGIN); + count++; + } + + assert(count > 0); + }); + + await test.step("should emit coordinate arrays from $.features[*].geometry.coordinates", async () => { + const input = `${FIXTURE_BASE}/json_bus.json.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextSelectorStream("$.features[*].geometry.coordinates"); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let count = 0; + + for await (const value of stream) { + if (count >= 100) break; + + const coordinates = value.json() as unknown[]; + + assert(Array.isArray(coordinates) && coordinates.length === 2); + count++; + } + + assert(count > 0); + }); + }); + + await test.step("[fixture] www.youtube.com.har.gz", async (test) => { + await test.step("should emit string values from $.log.entries[*].request.url", async () => { + const input = `${FIXTURE_BASE}/www.youtube.com.har.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextSelectorStream("$.log.entries[*].request.url"); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let count = 0; + + for await (const value of stream) { + assertEquals(value.kind, KIND.STRING); + count++; + } + + assert(count > 0); + }); + + await test.step("should emit string values from $..url", async () => { + const input = `${FIXTURE_BASE}/www.youtube.com.har.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextSelectorStream("$..url"); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let count = 0; + + for await (const value of stream) { + assertEquals(value.kind, KIND.STRING); + count++; + } + + assert(count > 0); + }); + + await test.step("should emit no values from $.log.nonexistent[*]", async () => { + const input = `${FIXTURE_BASE}/www.youtube.com.har.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextSelectorStream("$.log.nonexistent[*]"); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let count = 0; + + for await (const _ of stream) { + count++; + } + + assertEquals(count, 0); + }); + }); +}); diff --git a/tests/e2e/round-trip.bench.ts b/tests/e2e/round-trip.bench.ts deleted file mode 100644 index 0819de4..0000000 --- a/tests/e2e/round-trip.bench.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { - JSONTextDecoder, - JSONTextDecoderStream, - JSONTextEncoder, - JSONTextEncoderStream, -} from "#src/index"; - -const HAR_URL = new URL("../../public/example.com.har", import.meta.url); -const bytes = await Deno.readFile(HAR_URL); - -Deno.bench({ - name: "pull", - group: "round-trip", - baseline: true, - fn() { - const decoder = new JSONTextDecoder(bytes); - const encoder = new JSONTextEncoder({ multiline: false, spaceAfterColon: false }); - - decoder.end(); - - let token; - while ((token = decoder.readToken()) !== undefined) { - encoder.writeToken(token); - } - - decoder.checkEOF(); - }, -}); - -Deno.bench({ - name: "stream", - group: "round-trip", - async fn() { - const inputStream = new ReadableStream({ - start(controller) { - controller.enqueue(bytes); - controller.close(); - }, - }); - - const outputStream = inputStream - .pipeThrough(new JSONTextDecoderStream()) - .pipeThrough(new JSONTextEncoderStream({ multiline: false, spaceAfterColon: false })); - - for await (const _ of outputStream) { - // drain - } - }, -}); diff --git a/tests/e2e/round-trip.test.ts b/tests/e2e/round-trip.test.ts index 6a2da33..0f9fba0 100644 --- a/tests/e2e/round-trip.test.ts +++ b/tests/e2e/round-trip.test.ts @@ -1,96 +1,155 @@ -import { - JSONTextDecoder, - JSONTextDecoderStream, - JSONTextEncoder, - JSONTextEncoderStream, - JSONTextSelectorStream, - Token, -} from "#src/index"; -import { decodeText } from "#src/utils/text"; -import { assertEquals } from "#std/assert"; - -const HAR_URL = new URL("../../public/example.com.har", import.meta.url); - -Deno.test("[e2e] round-trip", async (test) => { - await test.step("should decode and re-encode example.com.har to semantically equivalent JSON", async () => { - const bytes = await Deno.readFile(HAR_URL); - const decoder = new JSONTextDecoder(bytes); - const encoder = new JSONTextEncoder({ multiline: false, spaceAfterColon: false }); - - decoder.end(); - - let token; - while ((token = decoder.readToken()) !== undefined) { - encoder.writeToken(token); - } - - decoder.checkEOF(); - - assertEquals( - JSON.parse(decodeText(encoder.bytes())), - JSON.parse(decodeText(bytes)), - ); - }); +import { KIND } from "#src/common/constants"; +import { JSONTextSelectorStream } from "#src/index"; +import { assert, assertEquals } from "#std/assert"; + +const GITHUB_TOKEN = Deno.env.get("GITHUB_TOKEN"); +const FIXTURE_BASE = "https://github.com/lcweden/jsontext/releases/download/fixtures"; + +Deno.test("[e2e] path", async (test) => { + await test.step("[fixture] json_bus.json.gz", async (test) => { + await test.step("should emit object values from $.features[*]", async () => { + const input = `${FIXTURE_BASE}/json_bus.json.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextSelectorStream("$.features[*]"); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let count = 0; + + for await (const value of stream) { + assertEquals(value.kind, KIND.OBJECT_BEGIN); + count++; + } + + assert(count > 0); + }); + + await test.step("should emit coordinate arrays from $.features[*].geometry.coordinates", async () => { + const input = `${FIXTURE_BASE}/json_bus.json.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); - await test.step("should round-trip example.com.har through stream pipeline", async () => { - const chunks: Uint8Array[] = []; - const file = await Deno.open(HAR_URL, { read: true }); - const stream = file.readable - .pipeThrough(new JSONTextDecoderStream()) - .pipeThrough(new JSONTextEncoderStream({ multiline: false, spaceAfterColon: false })); - - for await (const chunk of stream) { - chunks.push(chunk); - } - - const output = new Uint8Array(chunks.reduce((acc, c) => acc + c.length, 0)); - let offset = 0; - - for (const chunk of chunks) { - output.set(chunk, offset); - offset += chunk.length; - } - - assertEquals( - JSON.parse(decodeText(output)), - JSON.parse(decodeText(await Deno.readFile(HAR_URL))), - ); + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextSelectorStream("$.features[*].geometry.coordinates"); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let count = 0; + + for await (const value of stream) { + if (count >= 100) break; + + const coordinates = value.json() as unknown[]; + + assert(Array.isArray(coordinates) && coordinates.length === 2); + count++; + } + + assert(count > 0); + }); }); - await test.step("should round-trip example.com.har through selector pipeline", async () => { - const chunks: Uint8Array[] = []; - const file = await Deno.open(HAR_URL, { read: true }); - const stream = file.readable - .pipeThrough(new JSONTextSelectorStream("$..headers")) - .pipeThrough( - new TransformStream({ - start(controller) { - controller.enqueue(Token.ARRAY_BEGIN); - }, - transform(value, controller) { - for (const token of value.tokens()) { - controller.enqueue(token); - } - }, - flush(controller) { - controller.enqueue(Token.ARRAY_END); - }, - }), - ) - .pipeThrough(new JSONTextEncoderStream({ multiline: true, spaceAfterColon: false })); - - for await (const chunk of stream) { - chunks.push(chunk); - } - - const output = new Uint8Array(chunks.reduce((acc, c) => acc + c.length, 0)); - let offset = 0; - - for (const chunk of chunks) { - output.set(chunk, offset); - offset += chunk.length; - } - - assertEquals(JSON.parse(decodeText(output)).length, 2); + await test.step("[fixture] www.youtube.com.har.gz", async (test) => { + await test.step("should emit string values from $.log.entries[*].request.url", async () => { + const input = `${FIXTURE_BASE}/www.youtube.com.har.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextSelectorStream("$.log.entries[*].request.url"); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let count = 0; + + for await (const value of stream) { + assertEquals(value.kind, KIND.STRING); + count++; + } + + assert(count > 0); + }); + + await test.step("should emit string values from $..url", async () => { + const input = `${FIXTURE_BASE}/www.youtube.com.har.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextSelectorStream("$..url"); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let count = 0; + + for await (const value of stream) { + assertEquals(value.kind, KIND.STRING); + count++; + } + + assert(count > 0); + }); + + await test.step("should emit no values from $.log.nonexistent[*]", async () => { + const input = `${FIXTURE_BASE}/www.youtube.com.har.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } + + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextSelectorStream("$.log.nonexistent[*]"); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); + + let count = 0; + + for await (const _ of stream) { + count++; + } + + assertEquals(count, 0); + }); }); }); diff --git a/tests/e2e/streaming.test.ts b/tests/e2e/streaming.test.ts index 53f912a..9ea90a4 100644 --- a/tests/e2e/streaming.test.ts +++ b/tests/e2e/streaming.test.ts @@ -1,57 +1,65 @@ -import { KIND } from "#src/common/constants"; -import { JSONTextDecoder } from "#src/index"; -import { encodeText } from "#src/utils/text"; -import { assertEquals } from "#std/assert"; - -Deno.test("[e2e] streaming readValue", async (test) => { - await test.step("should parse a JSON array fed one byte at a time using readValue", () => { - const json = JSON.stringify([{ "id": 1, "name": "Alice" }, 42, "hello", [1, 2]]); - const bytes = encodeText(json); - const decoder = new JSONTextDecoder(); - const values: string[] = []; - let started = false; - - for (let i = 0; i < bytes.length; i++) { - decoder.push(bytes.subarray(i, i + 1)); - - if (i === bytes.length - 1) { - decoder.end(); +import { JSONTextDecoderStream } from "#src/index"; +import { assert } from "#std/assert"; + +const GITHUB_TOKEN = Deno.env.get("GITHUB_TOKEN"); +const FIXTURE_BASE = "https://github.com/lcweden/jsontext/releases/download/fixtures"; + +Deno.test("[e2e] streaming", async (test) => { + await test.step("[fixture] json_bus.json.gz", async (test) => { + await test.step("should stream 75 MB without error", async () => { + const input = `${FIXTURE_BASE}/json_bus.json.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); + + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); + + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); } - if (!started) { - if (decoder.readToken() === undefined) { - continue; - } + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextDecoderStream(); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); - started = true; + let tokenCount = 0; + + for await (const _ of stream) { + tokenCount++; } - while (true) { - const kind = decoder.peekKind(); + assert(tokenCount > 0); + }); + }); + + await test.step("[fixture] www.youtube.com.har.gz", async (test) => { + await test.step("should stream 131 MB without error", async () => { + const input = `${FIXTURE_BASE}/www.youtube.com.har.gz`; + const headers = new Headers({ "Accept": "application/octet-stream" }); - if (kind === undefined) { - break; - } + if (GITHUB_TOKEN) { + headers.set("Authorization", `Bearer ${GITHUB_TOKEN}`); + } + + const response = await fetch(input, { headers }); - if (kind === KIND.ARRAY_END) { - decoder.readToken(); - break; - } + if (!response.ok || !response.body) { + throw new Error(`Failed to fetch fixture: ${response.statusText}`); + } - const value = decoder.readValue(); + const decompresser = new DecompressionStream("gzip"); + const decoder = new JSONTextDecoderStream(); + const stream = response.body.pipeThrough(decompresser).pipeThrough(decoder); - if (value === undefined) { - break; - } + let tokenCount = 0; - values.push(value.text()); + for await (const _ of stream) { + tokenCount++; } - } - assertEquals(values.length, 4); - assertEquals(values[0], '{"id":1,"name":"Alice"}'); - assertEquals(values[1], "42"); - assertEquals(values[2], '"hello"'); - assertEquals(values[3], "[1,2]"); + assert(tokenCount > 0); + }); }); }); diff --git a/tests/integration/stream-line.test.ts b/tests/integration/stream-line.test.ts new file mode 100644 index 0000000..0c92d25 --- /dev/null +++ b/tests/integration/stream-line.test.ts @@ -0,0 +1,68 @@ +import { JSONTextLineStream } from "#src/index"; +import { encodeText } from "#src/utils/text"; +import { assertEquals } from "#std/assert"; + +Deno.test("[integration] JSONTextLineStream", async (test) => { + await test.step("should emit one value per JSON line in JSONL input", async () => { + const lines = ["null", "42", '"hello"', "{}", "[]"]; + const stream = new JSONTextLineStream(); + const values = []; + + const writing = (async () => { + const writer = stream.writable.getWriter(); + for (const line of lines) await writer.write(encodeText(line + "\n")); + await writer.close(); + })().catch(() => {}); + + for await (const value of stream.readable) { + values.push(value); + } + + await writing; + + assertEquals(values.length, lines.length); + }); + + await test.step("should emit values split across multiple chunks", async () => { + const json = '{"a":1}{"b":2}{"c":3}'; + const chunkSize = 4; + const stream = new JSONTextLineStream(); + const values = []; + + const writing = (async () => { + const writer = stream.writable.getWriter(); + for (let i = 0; i < json.length; i += chunkSize) { + await writer.write(encodeText(json.slice(i, i + chunkSize))); + } + await writer.close(); + })().catch(() => {}); + + for await (const value of stream.readable) { + values.push(value); + } + + await writing; + + assertEquals(values.length, 3); + }); + + await test.step("should emit a single value without trailing newline", async () => { + const stream = new JSONTextLineStream(); + const values = []; + + const writing = (async () => { + const writer = stream.writable.getWriter(); + await writer.write(encodeText("[1,2,3]")); + await writer.close(); + })().catch(() => {}); + + for await (const value of stream.readable) { + values.push(value); + } + + await writing; + + assertEquals(values.length, 1); + assertEquals(values[0].json(), [1, 2, 3]); + }); +}); From 28d1dfacb9478f3b0ce5b894a3f86409219416f4 Mon Sep 17 00:00:00 2001 From: eden <198768181+lcweden@users.noreply.github.com> Date: Mon, 18 May 2026 15:21:43 +0000 Subject: [PATCH 6/7] docs: enhance public api with JSDoc --- src/api/decoder.ts | 86 +++++++++++++++++++++++ src/api/encoder.ts | 60 ++++++++++++++++ src/common/errors.ts | 12 ++++ src/libs/stream-decoder.ts | 15 ++++ src/libs/stream-encoder.ts | 18 +++++ src/libs/stream-line.ts | 16 +++++ src/libs/stream-selector.ts | 34 ++++++++++ src/modules/token.ts | 132 ++++++++++++++++++++++++++++++++---- src/modules/value.ts | 69 +++++++++++++++++++ 9 files changed, 427 insertions(+), 15 deletions(-) diff --git a/src/api/decoder.ts b/src/api/decoder.ts index ae1a6e1..b5f0a33 100644 --- a/src/api/decoder.ts +++ b/src/api/decoder.ts @@ -7,57 +7,143 @@ import type { DecoderOptions } from "#src/types/options"; type JSONTextDecoderOptions = DecoderOptions; +/** + * Low-level, stateful JSON decoder that processes input incrementally. + * + * Feed byte chunks via {@link push} then consume tokens with + * {@link readToken} / {@link readValue} / {@link skipValue}. + * Call {@link end} when the stream is exhausted to flush any buffered state. + */ class JSONTextDecoder { #decoder: Decoder; + /** + * @param bytes - Initial bytes to pre-load into the decoder. + * @param options - Decoding options. + */ constructor(bytes = new Uint8Array(), options?: JSONTextDecoderOptions) { this.#decoder = new Decoder(bytes, { ...DEFAULT_DECODER_OPTIONS, ...options }); } + /** + * Asserts that the input has been fully consumed. + * + * @throws {SyntacticError} If there are unread bytes remaining. + */ checkEOF(): void { this.#decoder.checkEOF(); } + /** + * The current nesting depth — `0` at top level, incremented inside each + * object or array. + */ depth(): number { return this.#decoder.depth(); } + /** + * Signals that no more input will be pushed. + * + * Validates that any incomplete value is properly terminated. + * + * @throws {SyntacticError} If the input ends in the middle of a value. + */ end(): void { this.#decoder.end(); } + /** + * The byte offset of the next unread byte within the total input seen so far. + * + * @returns The byte offset of the next unread byte, or the total length of all + */ inputOffset(): number { return this.#decoder.inputOffset(); } + /** + * Appends a chunk of bytes to the internal buffer. + * + * @param bytes - The next chunk of JSON-encoded bytes. + */ push(bytes: Uint8Array): void { this.#decoder.push(bytes); } + /** + * Returns the {@link Kind} of the next token without consuming it, + * or `undefined` if no complete token is available yet. + * + * @returns The {@link Kind} of the next token, or `undefined` if no complete token is available yet. + */ peekKind(): Kind | undefined { return this.#decoder.peekKind(); } + /** + * Resets the decoder to its initial state, discarding all buffered input + * and state. + */ reset(): void { this.#decoder.reset(); } + /** + * Reads and returns the next {@link Token} from the buffer, or `undefined` + * if no complete token is available yet. + * + * @returns The next token, or `undefined` if no complete token is available yet. + * @throws {SyntacticError} If invalid JSON syntax is encountered. + */ readToken(): Token | undefined { return this.#decoder.readToken(); } + /** + * Reads and returns the next complete {@link Value} from the buffer, or + * `undefined` if there is not yet enough input to form a complete value. + * + * @returns The next value, or `undefined` if no complete value is available yet. + * @throws {SyntacticError} If invalid JSON syntax is encountered. + */ readValue(): Value | undefined { return this.#decoder.readValue(); } + /** + * Skips over the next complete value without returning it. + * + * @returns `true` if a value was skipped, `false` if no complete value was available yet. + * @throws {SyntacticError} If invalid JSON syntax is encountered. + */ skipValue(): boolean { return this.#decoder.skipValue(); } + /** + * Returns a JSON Pointer string describing a position in the current + * nesting context. + * + * | `where` | Meaning | + * |---------|-------------------------------------------------------------------| + * | `1` | The position of the **next** value to be read (default). | + * | `0` | The position of the **current** container. | + * | `-1` | The position of the **previously** read value. | + * + * @param where - Which position to return. Defaults to `1`. + * @returns A JSON Pointer string, e.g. `"/foo/0"`. + */ stackPointer(where: 0 | 1 | -1 = 1): string { return this.#decoder.stackPointer(where).toString(); } + /** + * Returns a view of the bytes in the internal buffer that have not yet been + * consumed. + * + * @returns A `Uint8Array` view of the unread bytes in the internal buffer. + */ unreadBytes(): Uint8Array { return this.#decoder.unreadBytes(); } diff --git a/src/api/encoder.ts b/src/api/encoder.ts index ac091e4..609938f 100644 --- a/src/api/encoder.ts +++ b/src/api/encoder.ts @@ -6,37 +6,97 @@ import type { EncoderOptions } from "#src/types/options"; type JSONTextEncoderOptions = EncoderOptions; +/** + * Low-level, stateful JSON encoder that produces bytes incrementally. + * + * Write tokens or values via {@link writeToken} / {@link writeValue}, then + * retrieve the accumulated output with {@link bytes}. Call {@link reset} to + * start a new document without creating a new instance. + */ class JSONTextEncoder { #encoder: Encoder; + /** + * @param options - Encoding options. + */ constructor(options?: JSONTextEncoderOptions) { this.#encoder = new Encoder({ ...DEFAULT_ENCODER_OPTIONS, ...options }); } + /** + * Returns the cumulative encoded bytes produced so far. + * + * The buffer is never cleared between calls; use {@link reset} to start fresh. + * + * @returns The cumulative encoded bytes produced so far. + */ bytes(): Uint8Array { return this.#encoder.bytes(); } + /** + * The current nesting depth — `0` at top level, incremented inside each + * object or array. + * + * @returns The current nesting depth. + */ depth(): number { return this.#encoder.depth(); } + /** + * The byte offset of the end of the last token written, equal to the total + * number of bytes produced so far. + * + * @returns The byte offset of the end of the last token written. + */ outputOffset(): number { return this.#encoder.outputOffset(); } + /** + * Resets the encoder to its initial state, clearing the output buffer and + * all structural state. + */ reset(): void { this.#encoder.reset(); } + /** + * Returns a JSON Pointer string describing a position in the current + * nesting context. + * + * | `where` | Meaning | + * |---------|---------------------------------------------------------------| + * | `1` | The position of the **next** value to be written (default). | + * | `0` | The position of the **current** container. | + * | `-1` | The position of the **previously** written value. | + * + * @param where - Which position to return. Defaults to `1`. + * @returns A JSON Pointer string, e.g. `"/foo/0"`. + */ stackPointer(where: 0 | 1 | -1 = 1): string { return this.#encoder.stackPointer(where).toString(); } + /** + * Encodes a single {@link Token} and appends its bytes to the output buffer. + * + * @param token - The token to encode. + * @throws {SyntacticError} If the token is not valid at the current position. + */ writeToken(token: Token): void { this.#encoder.writeToken(token); } + /** + * Encodes a complete {@link Value} and appends its bytes to the output + * buffer. + * + * @param value - The value to encode. + * @throws {SyntacticError} If the value is not valid at the current + * position. + */ writeValue(value: Value): void { this.#encoder.writeValue(value); } diff --git a/src/common/errors.ts b/src/common/errors.ts index 84cad5b..6e96ec6 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -1,7 +1,19 @@ +/** + * Thrown when invalid JSON syntax is encountered during decoding or encoding. + * + * The error message follows the format: + * `" within at offset "`, + * where `within ` is omitted when `pointer` is an empty string. + */ class SyntacticError extends SyntaxError { pointer: string; offset: number; + /** + * @param message - A human-readable description of the syntax error. + * @param pointer - JSON Pointer to the location in the document. + * @param offset - Byte offset at which the error was detected. + */ constructor(message: string, pointer: string, offset: number) { const within = pointer ? ` within ${pointer}` : ""; const at = ` at offset ${offset}`; diff --git a/src/libs/stream-decoder.ts b/src/libs/stream-decoder.ts index e60dd41..da47942 100644 --- a/src/libs/stream-decoder.ts +++ b/src/libs/stream-decoder.ts @@ -8,7 +8,22 @@ type JSONTextDecoderStreamOptions = DecoderOptions & { readableStrategy?: QueuingStrategy; }; +/** + * A `TransformStream` that decodes a stream of `Uint8Array` byte chunks into + * a stream of {@link Token} objects. + * + * Writable side accepts raw JSON bytes (possibly split across multiple chunks). + * Readable side emits one {@link Token} per JSON token in document order. + * + * @example + * const response = await fetch(url); + * const tokens = response.body + * .pipeThrough(new JSONTextDecoderStream()); + */ class JSONTextDecoderStream extends TransformStream { + /** + * @param options - Decoder and queuing strategy options. + */ constructor(options: JSONTextDecoderStreamOptions = {}) { const { writableStrategy, readableStrategy, ...rest } = options; const decoderOptions = { ...DEFAULT_DECODER_OPTIONS, ...rest }; diff --git a/src/libs/stream-encoder.ts b/src/libs/stream-encoder.ts index 3e53581..7950854 100644 --- a/src/libs/stream-encoder.ts +++ b/src/libs/stream-encoder.ts @@ -8,7 +8,25 @@ type JSONTextEncoderStreamOptions = EncoderOptions & { readableStrategy?: QueuingStrategy; }; +/** + * A `TransformStream` that encodes a stream of {@link Token} objects into + * a stream of `Uint8Array` byte chunks. + * + * Writable side accepts {@link Token} objects. Readable side emits the + * corresponding JSON bytes, flushing output as tokens are written. + * + * @example + * const { readable, writable } = new JSONTextEncoderStream(); + * const writer = writable.getWriter(); + * writer.write(Token.ARRAY_BEGIN); + * writer.write(Token.fromNumber(1)); + * writer.write(Token.ARRAY_END); + * writer.close(); + */ class JSONTextEncoderStream extends TransformStream { + /** + * @param options - Encoder and queuing strategy options. + */ constructor(options?: JSONTextEncoderStreamOptions) { const { writableStrategy, readableStrategy, ...rest } = options ?? {}; const encoder = new Encoder({ ...DEFAULT_ENCODER_OPTIONS, ...rest }); diff --git a/src/libs/stream-line.ts b/src/libs/stream-line.ts index b47357f..e0c834a 100644 --- a/src/libs/stream-line.ts +++ b/src/libs/stream-line.ts @@ -8,7 +8,23 @@ type JSONTextLineStreamOptions = DecoderOptions & { readableStrategy?: QueuingStrategy; }; +/** + * A `TransformStream` that decodes a stream of `Uint8Array` byte chunks into + * a stream of complete {@link Value} objects. + * + * Each emitted value corresponds to one top-level JSON value in the input. + * This makes `JSONTextLineStream` well-suited for processing newline-delimited + * JSON (JSONL / JSON Lines) as well as any concatenated-JSON stream. + * + * @example + * const response = await fetch(url); + * const values = response.body + * .pipeThrough(new JSONTextLineStream()); + */ class JSONTextLineStream extends TransformStream { + /** + * @param options - Decoder and queuing strategy options. + */ constructor(options?: JSONTextLineStreamOptions) { const { writableStrategy, readableStrategy, ...rest } = options ?? {}; const decoder = new Decoder(new Uint8Array(), { ...DEFAULT_DECODER_OPTIONS, ...rest }); diff --git a/src/libs/stream-selector.ts b/src/libs/stream-selector.ts index dd4ad61..1e3e552 100644 --- a/src/libs/stream-selector.ts +++ b/src/libs/stream-selector.ts @@ -10,7 +10,41 @@ type JSONTextSelectorStreamOptions = DecoderOptions & { readableStrategy?: QueuingStrategy; }; +/** + * A `TransformStream` that filters a JSON byte stream and emits only the + * {@link Value} objects matching a JSONPath-like query. + * + * Writable side accepts raw JSON bytes. Readable side emits each matched + * value in document order. + * + * This implementation supports a **subset** of JSONPath (RFC 9535): + * + * | Syntax | Description | + * |---|---| + * | `$` | Root node | + * | `$.key` | Named child member (dot notation) | + * | `$.*` | Wildcard — all direct children | + * | `$[0]` | Indexed child (non-negative integer only) | + * | `$['key']` / `$["key"]` | Quoted name child | + * | `$[*]` | Wildcard child (bracket notation) | + * | `$[start:end]` / `$[start:end:step]` | Array slice (non-negative bounds and step only) | + * | `$..key` / `$..*` | Recursive descent | + * + * **Not supported:** negative indices, negative slice bounds, filter + * expressions (`?(...)`), and union selectors (`[0,1]`). + * + * @see https://www.rfc-editor.org/rfc/rfc9535 + * @example + * const response = await fetch(url); + * const values = response.body + * .pipeThrough(new JSONTextSelectorStream('$.items[*]')); + */ class JSONTextSelectorStream extends TransformStream { + /** + * @param query - A JSONPath-like query string. + * @param options - Decoder and queuing strategy options. + * @throws {SyntaxError} If `query` is not a valid query expression. + */ constructor(query: string, options?: JSONTextSelectorStreamOptions) { const { writableStrategy, readableStrategy, ...rest } = options ?? {}; const decoder = new Decoder(new Uint8Array(), { ...DEFAULT_DECODER_OPTIONS, ...rest }); diff --git a/src/modules/token.ts b/src/modules/token.ts index a20c67d..a9b742c 100644 --- a/src/modules/token.ts +++ b/src/modules/token.ts @@ -3,10 +3,25 @@ import type { Kind } from "#src/types/kind"; import { normalize } from "#src/utils/kind"; import { decodeText, encodeText } from "#src/utils/text"; +/** + * Represents a single JSON token. + * + * A token is the smallest unit in a JSON document — either a scalar value + * (`null`, `true`, `false`, a number, or a string) or a structural symbol + * (`{`, `}`, `[`, `]`). Tokens hold their raw UTF-8 bytes and expose typed + * accessor methods for converting to JavaScript primitives. + */ class Token { #bytes: Uint8Array; #kind: Kind; + /** + * Creates a `Token` from raw UTF-8 bytes. + * + * @param bytes - Raw UTF-8 bytes of a single JSON token. Leading whitespace is not permitted. + * @throws {RangeError} If `bytes` is empty. + * @throws {SyntaxError} If the first byte does not correspond to a valid JSON token. + */ constructor(bytes: Uint8Array) { if (!bytes.length) { throw new RangeError("Token must have at least one byte"); @@ -22,14 +37,51 @@ class Token { this.#kind = kind; } + /** The {@link Kind} of this token. */ get kind() { return this.#kind; } + /** The raw UTF-8 bytes of this token. */ get bytes() { return this.#bytes; } + /** Pre-built `null` token. */ + static NULL = Token.fromText("null"); + + /** Pre-built `true` token. */ + static TRUE = Token.fromBoolean(true); + + /** Pre-built `false` token. */ + static FALSE = Token.fromBoolean(false); + + /** Pre-built `{` token. */ + static OBJECT_BEGIN = Token.fromText("{"); + + /** Pre-built `}` token. */ + static OBJECT_END = Token.fromText("}"); + + /** Pre-built `[` token. */ + static ARRAY_BEGIN = Token.fromText("["); + + /** Pre-built `]` token. */ + static ARRAY_END = Token.fromText("]"); + + /** + * Creates a `Token` from a raw JSON text string. + * + * The string must be a valid JSON token (e.g. `"true"`, `"42"`, `'"hello"'`, + * `"{"`) with no surrounding whitespace. + * + * @param value - Raw JSON token text. + * @returns A new `Token` parsed from the given text. + * @throws {SyntaxError} If `value` is not a valid JSON token. + * @example + * const token = Token.fromText("true"); // or Token.fromText(JSON.stringify(true)); + * console.log(token.kind); // "true" + * console.log(token.asBoolean()); // true + */ static fromText(value: string): Token { const encoded = encodeText(value); const token = new Token(encoded); @@ -37,10 +89,25 @@ class Token { return token; } + /** + * Creates a `Token` from a JavaScript boolean. + * + * @param value - The boolean to encode. + * @returns `Token.TRUE` for `true`, `Token.FALSE` for `false`. + */ static fromBoolean(value: boolean): Token { - return value ? Token.fromText("true") : Token.fromText("false"); + return value ? Token.TRUE : Token.FALSE; } + /** + * Creates a `Token` from a JavaScript number. + * + * Non-finite values are encoded as JSON strings: `NaN` → `"NaN"`, + * `Infinity` → `"Infinity"`, `-Infinity` → `"-Infinity"`. + * + * @param value - The number to encode. + * @returns A number token, or a string token for non-finite values. + */ static fromNumber(value: number): Token { if (Number.isNaN(value)) { return Token.fromString("NaN"); @@ -56,6 +123,15 @@ class Token { return token; } + /** + * Creates a `Token` from a JavaScript string. + * + * The string is JSON-encoded — surrounding quotes and escape sequences are + * added automatically. + * + * @param value - The string value to encode. + * @returns A string token whose bytes represent the JSON-encoded form. + */ static fromString(value: string): Token { const encoded = encodeText(JSON.stringify(value)); const token = new Token(encoded); @@ -63,24 +139,21 @@ class Token { return token; } - static NULL = Token.fromText("null"); - - static TRUE = Token.fromBoolean(true); - - static FALSE = Token.fromBoolean(false); - - static OBJECT_BEGIN = Token.fromText("{"); - - static OBJECT_END = Token.fromText("}"); - - static ARRAY_BEGIN = Token.fromText("["); - - static ARRAY_END = Token.fromText("]"); - + /** + * Returns a deep copy of this token with an independent byte array. + * + * @returns A new `Token` backed by a cloned `Uint8Array`. + */ clone(): Token { return new Token(this.bytes.slice()); } + /** + * Returns `true` if this token is a scalar value — `null`, `true`, `false`, + * a number, or a string. + * + * @returns `true` for scalar tokens, `false` for structural tokens. + */ isScalar(): boolean { return ( this.kind === KIND.STRING || @@ -91,6 +164,11 @@ class Token { ); } + /** + * Returns `true` if this token is a structural symbol — `{`, `}`, `[`, or `]`. + * + * @returns `true` for structural tokens, `false` for scalar tokens. + */ isStructural(): boolean { return ( this.kind === KIND.OBJECT_BEGIN || @@ -100,6 +178,12 @@ class Token { ); } + /** + * Decodes this token as a JavaScript string. + * + * @returns The unescaped string value. + * @throws {TypeError} If this token is not of kind {@link KIND.STRING}. + */ asString(): string { if (this.#kind !== KIND.STRING) { throw new TypeError(`invalid JSON token kind: ${this.#kind}`); @@ -111,6 +195,12 @@ class Token { return result; } + /** + * Decodes this token as a JavaScript number. + * + * @returns The numeric value. + * @throws {TypeError} If this token is not of kind {@link KIND.NUMBER}. + */ asNumber(): number { if (this.#kind !== KIND.NUMBER) { throw new TypeError(`invalid JSON token kind: ${this.#kind}`); @@ -122,6 +212,12 @@ class Token { return result; } + /** + * Decodes this token as a JavaScript boolean. + * + * @returns `true` for `true` tokens, `false` for `false` tokens. + * @throws {TypeError} If this token is not of kind {@link KIND.TRUE} or {@link KIND.FALSE}. + */ asBoolean(): boolean { if (this.#kind === KIND.TRUE) { return true; @@ -134,6 +230,12 @@ class Token { throw new TypeError(`invalid JSON token kind: ${this.#kind}`); } + /** + * Asserts that this token is `null` and returns `null`. + * + * @returns `null`. + * @throws {TypeError} If this token is not of kind {@link KIND.NULL}. + */ asNull(): null { if (this.#kind !== KIND.NULL) { throw new TypeError(`invalid JSON token kind: ${this.#kind}`); diff --git a/src/modules/value.ts b/src/modules/value.ts index 397147c..726d6d0 100644 --- a/src/modules/value.ts +++ b/src/modules/value.ts @@ -6,10 +6,29 @@ import { normalize } from "#src/utils/kind"; import { decodeText, encodeText } from "#src/utils/text"; import { compareUTF16, consumeWhitespace } from "#src/utils/wire"; +/** + * Represents a complete JSON value. + * + * A value may be a scalar (`null`, `true`, `false`, a number, or a string) + * or a composite structure (an object or array, including all nested content). + * It holds the raw UTF-8 bytes and provides methods for converting, + * validating, canonicalizing, and iterating over the value. + * + * Unlike {@link Token}, a `Value` may be preceded by whitespace. + */ class Value { #bytes: Uint8Array; #kind: Kind; + /** + * Creates a `Value` from raw UTF-8 bytes. + * + * Leading whitespace is accepted and preserved in {@link bytes}. + * + * @param bytes - Raw UTF-8 bytes of a complete JSON value. + * @throws {RangeError} If `bytes` is empty. + * @throws {SyntaxError} If no valid JSON token is found after skipping leading whitespace. + */ constructor(bytes: Uint8Array) { if (!bytes.length) { throw new RangeError("Value must have at least one byte"); @@ -31,14 +50,25 @@ class Value { this.#kind = kind; } + /** The {@link Kind} of the top-level token of this value. */ get kind() { return this.#kind; } + /** The raw UTF-8 bytes of this value, including any leading whitespace. */ get bytes() { return this.#bytes; } + /** + * Creates a `Value` from any JavaScript value via `JSON.stringify`. + * + * @param input - Any JSON-serialisable value. + * @returns A new `Value` whose bytes are the JSON representation of `input`. + * @example + * const value = Value.from({ a: 1, b: [true, null] }); + * console.log(value.text()); // '{"a":1,"b":[true,null]}' + */ static from(input: unknown): Value { const json = JSON.stringify(input); const encoded = encodeText(json); @@ -46,6 +76,14 @@ class Value { return new Value(encoded); } + /** + * Returns a canonicalized copy of this value. + * + * Canonicalization recursively sorts object keys by UTF-16 code unit order + * and normalizes numbers. The result is deterministic and idempotent. + * + * @returns A new `Value` in canonical form. + */ canonicalize(): Value { const decoder = new Decoder(this.#bytes, { allowDuplicateNames: true }); decoder.end(); @@ -53,10 +91,21 @@ class Value { return new Value(this.#processValue(decoder)); } + /** + * Returns a deep copy of this value with an independent byte array. + * + * @returns A new `Value` backed by a cloned `Uint8Array`. + */ clone(): Value { return new Value(this.#bytes.slice()); } + /** + * Returns `true` if the bytes represent a structurally valid, complete JSON + * value with no trailing content. + * + * @returns `true` if valid, `false` otherwise. Never throws. + */ isValid(): boolean { try { const decoder = new Decoder(this.#bytes, {}); @@ -77,14 +126,34 @@ class Value { } } + /** + * Deserializes this value to a JavaScript value via `JSON.parse`. + * + * @returns The parsed JavaScript value. + */ json(): unknown { return JSON.parse(this.text()); } + /** + * Returns the UTF-8 string representation of this value. + * + * @returns The JSON text of this value. + * @throws If the bytes contain invalid UTF-8 sequences. + */ text(): string { return decodeText(this.bytes, { fatal: true }); } + /** + * Returns a generator that yields each {@link Token} within this value in + * document order. + * + * For scalar values, yields one token. For objects and arrays, yields all + * tokens including structural delimiters, keys, and nested values. + * + * @yields {Token} Tokens in document order. + */ *tokens(): Generator { const decoder = new Decoder(this.bytes, DEFAULT_DECODER_OPTIONS); From d893eb63edadff19659ce269d23affab7fcaf8ea Mon Sep 17 00:00:00 2001 From: eden <198768181+lcweden@users.noreply.github.com> Date: Mon, 18 May 2026 15:36:39 +0000 Subject: [PATCH 7/7] fix: update TRUE and FALSE token creation to use fromText method --- src/modules/token.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/token.ts b/src/modules/token.ts index a9b742c..5d6c571 100644 --- a/src/modules/token.ts +++ b/src/modules/token.ts @@ -51,10 +51,10 @@ class Token { static NULL = Token.fromText("null"); /** Pre-built `true` token. */ - static TRUE = Token.fromBoolean(true); + static TRUE = Token.fromText("true"); /** Pre-built `false` token. */ - static FALSE = Token.fromBoolean(false); + static FALSE = Token.fromText("false"); /** Pre-built `{` token. */ static OBJECT_BEGIN = Token.fromText("{");