diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b216abf..45dde77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,6 @@ name: CI on: - push: - branches: [main] pull_request: branches: [main] workflow_dispatch: @@ -18,6 +16,17 @@ jobs: - name: Setup uses: actions/checkout@v6 + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: lts/* + + - name: Install Dependencies + run: npm ci + + - name: Build + run: npm run build + - name: Setup Deno uses: denoland/setup-deno@v2 with: diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..033a255 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,42 @@ +name: Publish + +on: + release: + types: [published] + workflow_dispatch: + +permissions: + id-token: write + contents: write + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version: lts/* + cache: npm + registry-url: https://registry.npmjs.org/ + + - name: Update npm to latest + run: npm install -g npm@latest + + - name: Install dependencies + run: | + rm -rf node_modules package-lock.json + npm install + + - name: Build + run: npm run build + + - name: Publish to npm + run: npm publish --provenance + + - name: Publish to JSR + run: npx jsr publish diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..23bed05 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +All notable changes are documented in +[GitHub Releases](https://github.com/lcweden/jsontext/releases). diff --git a/README.md b/README.md index e9671d0..dd5eec1 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,307 @@ # JSONText -A streaming JSON parser and serializer for JavaScript. It provides a low-level API for parsing and -generating JSON text, allowing you to process large JSON documents without loading them entirely -into memory. +A streaming JSON parser and serializer for JavaScript, operating at the syntactic layer — processing +JSON tokens and values without any semantic marshaling. + +## Quick Start + +Parse a JSON stream and extract specific values with a JSONPath query: + +```ts +import { JSONTextSelectorStream } from "jsontext"; + +const response = await fetch("https://example.com/data.json"); +const stream = response.body.pipeThrough(new JSONTextSelectorStream("$.items[*]")); + +for await (const value of stream) { + console.log(value.json()); +} +``` + +## Installation + +### NPM + +```bash +npm install jsontext +``` + +## APIs + +### JSONTextDecoderStream + +Transforms a `ReadableStream` into a `ReadableStream`, emitting one `Token` per +JSON token in document order. + +```ts +import { JSONTextDecoderStream } from "jsontext"; + +const response = await fetch("https://example.com/data.json"); + +for await (const token of response.body.pipeThrough(new JSONTextDecoderStream())) { + console.log(token.kind, token.bytes); +} +``` + +### JSONTextEncoderStream + +Transforms a `ReadableStream` into a `ReadableStream`. + +```ts +import { JSONTextEncoderStream, Token } from "jsontext"; + +const { readable, writable } = new JSONTextEncoderStream(); +const writer = writable.getWriter(); + +writer.write(Token.ARRAY_BEGIN); +writer.write(Token.fromNumber(1)); +writer.write(Token.fromNumber(2)); +writer.write(Token.fromNumber(3)); +writer.write(Token.ARRAY_END); +writer.close(); + +const bytes = await new Response(readable).arrayBuffer(); +``` + +### JSONTextSelectorStream + +Transforms a byte stream into a `ReadableStream`, emitting only values matched by a +[JSON Path](https://www.rfc-editor.org/rfc/rfc9535) query. + +```ts +import { JSONTextSelectorStream } from "jsontext"; + +const response = await fetch("https://example.com/data.json"); +const stream = response.body.pipeThrough(new JSONTextSelectorStream("$.features[*].geometry")); + +for await (const value of stream) { + console.log(value.json()); +} +``` + +Supports the following [RFC 9535](https://www.rfc-editor.org/rfc/rfc9535) selectors: Child Segment, +Descendant Segment, Name Selector, Wildcard Selector, Index Selector (positive only), and Array +Slice Selector (positive only). + +### JSONTextLineStream + +Transforms a byte stream into a `ReadableStream`, emitting one complete `Value` per top-level +JSON value. Well-suited for JSONL / JSON Lines and other concatenated-JSON streams. + +```ts +import { JSONTextLineStream } from "jsontext"; + +const response = await fetch("https://example.com/data.jsonl"); +const stream = response.body.pipeThrough(new JSONTextLineStream()); + +for await (const value of stream) { + console.log(value.json()); +} +``` + +## Types + +### Kind + +`Kind` is a string literal type representing the class of a JSON token. All values are available on +the `KIND` constant: + +```ts +import { KIND } from "jsontext"; + +KIND.NULL; // "null" +KIND.TRUE; // "true" +KIND.FALSE; // "false" +KIND.STRING; // "string" +KIND.NUMBER; // "number" +KIND.OBJECT_BEGIN; // "{" +KIND.OBJECT_END; // "}" +KIND.ARRAY_BEGIN; // "[" +KIND.ARRAY_END; // "]" +``` + +### Token + +A `Token` represents a single lexical element — a scalar value (`null`, `true`, `false`, a number, +or a string) or a structural delimiter (`{`, `}`, `[`, `]`). + +**Pre-built tokens:** + +```ts +Token.NULL; // null +Token.TRUE; // true +Token.FALSE; // false +Token.OBJECT_BEGIN; // { +Token.OBJECT_END; // } +Token.ARRAY_BEGIN; // [ +Token.ARRAY_END; // ] +``` + +**Factory methods:** + +```ts +Token.fromBoolean(true); // true +Token.fromNumber(3.14); // 3.14 +Token.fromString("hello"); // "hello" +Token.fromText('"raw"'); // from a raw JSON text string +``` + +### Value + +A `Value` represents a complete JSON value — a scalar or an entire object/array including all nested +content. + +**Factory methods:** + +```ts +Value.from({ name: "Alice", scores: [1, 2, 3] }); +// {"name":"Alice","scores":[1,2,3]} +``` + +## Examples + +The following examples demonstrate how to use the streaming API to perform common transformations on +JSON data. + +### Redacting values + +```javascript +import { JSONTextDecoderStream, JSONTextEncoderStream } from "jsontext"; + +const target = "password"; +const response = await fetch("https://example.com/user.json"); + +response.body + .pipeThrough(new JSONTextDecoderStream()) + .pipeThrough( + new TransformStream({ + transform(token, controller) { + if (token.kind === KIND.STRING && token.asString() === target) { + controller.enqueue(Token.fromString("********")); + } else { + controller.enqueue(token); + } + }, + }), + ) + .pipeThrough(new JSONTextEncoderStream()); +``` + +### Extracting into a new array + +```javascript +import { JSONTextEncoderStream, JSONTextSelectorStream } from "jsontext"; + +const response = await fetch("https://example.com/data.json"); + +response.body + .pipeThrough(new JSONTextSelectorStream("$.items[*]")) + .pipeThrough( + new TransformStream({ + start(controller) { + controller.enqueue(Token.ARRAY_BEGIN); + }, + transform(value, controller) { + for (const token of value.tokens()) { + controller.enqueue(token); + } + }, + flush(controller) { + controller.enqueue(Token.ARRAY_END); + }, + }), + ) + .pipeThrough(new JSONTextEncoderStream()); +``` + +## Core + +For cases where the stream API is not flexible enough, `JSONTextDecoder` and `JSONTextEncoder` +provide a lower-level interface for building custom decoders and encoders. + +### JSONTextDecoder + +```ts +import { JSONTextDecoder } from "jsontext"; + +const decoder = new JSONTextDecoder(); +decoder.push(new TextEncoder().encode('{"name":"Alice"}')); +decoder.end(); + +let token; +while ((token = decoder.readToken()) !== undefined) { + console.log(token.kind); +} + +decoder.checkEOF(); // throws if there are unconsumed bytes or an incomplete value +``` + +| Method | Description | +| ---------------------- | ------------------------------------------------------------------------------------ | +| `push(bytes)` | Append a chunk of raw JSON bytes | +| `end()` | Signal end-of-input; validates the stream is not mid-value | +| `readToken()` | Return the next `Token`, or `undefined` if more input is needed | +| `readValue()` | Return the next complete `Value`, or `undefined` if more input is needed | +| `skipValue()` | Discard the next complete value; returns `true` if a value was skipped | +| `peekKind()` | Inspect the next token kind without consuming it | +| `stackPointer(where?)` | JSON Pointer to the next (`1`), current container (`0`), or previous (`-1`) position | +| `inputOffset()` | Byte offset of the next unread byte | +| `depth()` | Current nesting depth | +| `unreadBytes()` | View of buffered but unconsumed bytes | +| `checkEOF()` | Assert that all input has been consumed | +| `reset()` | Clear all buffered input and internal state | + +### JSONTextEncoder + +```ts +import { JSONTextEncoder, Token } from "jsontext"; + +const encoder = new JSONTextEncoder(); +encoder.writeToken(Token.OBJECT_BEGIN); +encoder.writeToken(Token.fromString("name")); +encoder.writeToken(Token.fromString("Alice")); +encoder.writeToken(Token.OBJECT_END); + +console.log(new TextDecoder().decode(encoder.bytes())); +``` + +| Method | Description | +| ---------------------- | ------------------------------------------------------------------------------------ | +| `writeToken(token)` | Encode a `Token` and append its bytes to the output buffer | +| `writeValue(value)` | Encode a `Value` and append its bytes to the output buffer | +| `bytes()` | All bytes produced so far | +| `outputOffset()` | Byte offset after the last written token | +| `stackPointer(where?)` | JSON Pointer to the next (`1`), current container (`0`), or previous (`-1`) position | +| `depth()` | Current nesting depth | +| `reset()` | Clear the output buffer and all internal state | + +## Options + +### Decoder + +| Option | Default | Description | +| --------------------- | ------- | ------------------------------------------------------------- | +| `allowDuplicateNames` | `false` | Allow duplicate object member names | +| `allowInvalidUTF8` | `false` | Replace invalid UTF-8 bytes with `U+FFFD` instead of erroring | + +### Encoder + +All decoder options, plus: + +| Option | Default | Description | +| ------------------------ | ------- | ------------------------------------------------------------ | +| `escapeForHTML` | `false` | Escape `<`, `>`, `&` as `\uXXXX` for safe HTML embedding | +| `escapeForJS` | `false` | Escape `U+2028` and `U+2029` for safe JavaScript embedding | +| `canonicalizeRawNumbers` | `false` | Normalize numbers per RFC 8785 §3.2.2.3 | +| `spaceAfterColon` | `true` | Emit a space after `:` in objects | +| `spaceAfterComma` | `false` | Emit a space after `,` | +| `multiline` | `true` | Expand output across multiple indented lines | +| `indent` | `"\t"` | Indentation string (implies `multiline`) | +| `indentPrefix` | `""` | Prefix prepended to each indented line (implies `multiline`) | ## Acknowledgements -This project is heavily inspired by Go +This project is heavily inspired by Go's [`encoding/json/jsontext`](https://pkg.go.dev/encoding/json/jsontext) standard library. The API and internal design are closely modeled after it, with adjustments made to fit JavaScript's language features and ecosystem. diff --git a/deno.json b/deno.json index 5e12755..9967d5a 100644 --- a/deno.json +++ b/deno.json @@ -1,4 +1,10 @@ { + "name": "@lcweden/jsontext", + "version": "0.1.0", + "license": "MIT", + "exports": { + ".": "./src/index.ts" + }, "tasks": { "bench": "deno bench --allow-all", "check": "deno check && deno fmt --check && deno lint", @@ -26,6 +32,11 @@ "exclude": [ "node_modules/", "public/", - "dist/" - ] + "dist/", + "vite.config.ts" + ], + "publish": { + "include": ["src/", "README.md", "LICENSE", "deno.json"], + "exclude": ["tests/", ".github/", ".vscode/", "dist/", "public/"] + } } diff --git a/package.json b/package.json index 0c28cd3..f15f8f0 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,41 @@ { "name": "jsontext", - "private": true, - "version": "0.0.0", + "version": "0.1.0", + "description": "A streaming JSON parser and serializer for JavaScript.", + "keywords": ["json", "stream", "decoder", "encoder"], "type": "module", + "sideEffects": false, + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, "imports": { - "#src/*": "./src/*.ts" + "#src/*": { + "types": "./dist/*.d.ts", + "default": "./src/*.ts" + } + }, + "files": ["dist/**/*", "!dist/**/*.map"], + "author": { + "name": "lcweden", + "url": "https://github.com/lcweden" + }, + "license": "MIT", + "homepage": "https://github.com/lcweden/jsontext", + "repository": { + "type": "git", + "url": "git+https://github.com/lcweden/jsontext.git" + }, + "bugs": { + "url": "https://github.com/lcweden/jsontext/issues" }, "scripts": { - "build": "tsc && vite build", + "build": "vite build && tsc -p tsconfig.build.json", "bench": "deno task bench", "dev": "vite", "format": "deno task format", diff --git a/src/api/decoder.ts b/src/api/decoder.ts index b5f0a33..89f2fda 100644 --- a/src/api/decoder.ts +++ b/src/api/decoder.ts @@ -21,7 +21,7 @@ class JSONTextDecoder { * @param bytes - Initial bytes to pre-load into the decoder. * @param options - Decoding options. */ - constructor(bytes = new Uint8Array(), options?: JSONTextDecoderOptions) { + constructor(bytes: Uint8Array = new Uint8Array(), options?: JSONTextDecoderOptions) { this.#decoder = new Decoder(bytes, { ...DEFAULT_DECODER_OPTIONS, ...options }); } diff --git a/src/libs/stream-selector.ts b/src/libs/stream-selector.ts index 1e3e552..8d782d1 100644 --- a/src/libs/stream-selector.ts +++ b/src/libs/stream-selector.ts @@ -18,20 +18,14 @@ type JSONTextSelectorStreamOptions = DecoderOptions & { * value in document order. * * This implementation supports a **subset** of JSONPath (RFC 9535): + * **supported:** * - * | Syntax | Description | - * |---|---| - * | `$` | Root node | - * | `$.key` | Named child member (dot notation) | - * | `$.*` | Wildcard — all direct children | - * | `$[0]` | Indexed child (non-negative integer only) | - * | `$['key']` / `$["key"]` | Quoted name child | - * | `$[*]` | Wildcard child (bracket notation) | - * | `$[start:end]` / `$[start:end:step]` | Array slice (non-negative bounds and step only) | - * | `$..key` / `$..*` | Recursive descent | - * - * **Not supported:** negative indices, negative slice bounds, filter - * expressions (`?(...)`), and union selectors (`[0,1]`). + * - Child Segment + * - Descendant Segment + * - Name Selector + * - Wildcard Selector + * - Index Selector + * - Array Slice Selector * * @see https://www.rfc-editor.org/rfc/rfc9535 * @example diff --git a/src/modules/token.ts b/src/modules/token.ts index 5d6c571..ad98d5f 100644 --- a/src/modules/token.ts +++ b/src/modules/token.ts @@ -38,35 +38,35 @@ class Token { } /** The {@link Kind} of this token. */ - get kind() { + get kind(): Kind { return this.#kind; } /** The raw UTF-8 bytes of this token. */ - get bytes() { + get bytes(): Uint8Array { return this.#bytes; } /** Pre-built `null` token. */ - static NULL = Token.fromText("null"); + static NULL: Token = Token.fromText("null"); /** Pre-built `true` token. */ - static TRUE = Token.fromText("true"); + static TRUE: Token = Token.fromText("true"); /** Pre-built `false` token. */ - static FALSE = Token.fromText("false"); + static FALSE: Token = Token.fromText("false"); /** Pre-built `{` token. */ - static OBJECT_BEGIN = Token.fromText("{"); + static OBJECT_BEGIN: Token = Token.fromText("{"); /** Pre-built `}` token. */ - static OBJECT_END = Token.fromText("}"); + static OBJECT_END: Token = Token.fromText("}"); /** Pre-built `[` token. */ - static ARRAY_BEGIN = Token.fromText("["); + static ARRAY_BEGIN: Token = Token.fromText("["); /** Pre-built `]` token. */ - static ARRAY_END = Token.fromText("]"); + static ARRAY_END: Token = Token.fromText("]"); /** * Creates a `Token` from a raw JSON text string. diff --git a/src/modules/value.ts b/src/modules/value.ts index 726d6d0..d865d2f 100644 --- a/src/modules/value.ts +++ b/src/modules/value.ts @@ -51,12 +51,12 @@ class Value { } /** The {@link Kind} of the top-level token of this value. */ - get kind() { + get kind(): Kind { return this.#kind; } /** The raw UTF-8 bytes of this value, including any leading whitespace. */ - get bytes() { + get bytes(): Uint8Array { return this.#bytes; } diff --git a/tests/e2e/streaming.test.ts b/tests/e2e/streaming.test.ts index 9ea90a4..5c31d91 100644 --- a/tests/e2e/streaming.test.ts +++ b/tests/e2e/streaming.test.ts @@ -6,7 +6,7 @@ const FIXTURE_BASE = "https://github.com/lcweden/jsontext/releases/download/fixt Deno.test("[e2e] streaming", async (test) => { await test.step("[fixture] json_bus.json.gz", async (test) => { - await test.step("should stream 75 MB without error", async () => { + await test.step("should stream without error", async () => { const input = `${FIXTURE_BASE}/json_bus.json.gz`; const headers = new Headers({ "Accept": "application/octet-stream" }); @@ -35,7 +35,7 @@ Deno.test("[e2e] streaming", async (test) => { }); await test.step("[fixture] www.youtube.com.har.gz", async (test) => { - await test.step("should stream 131 MB without error", async () => { + await test.step("should stream without error", async () => { const input = `${FIXTURE_BASE}/www.youtube.com.har.gz`; const headers = new Headers({ "Accept": "application/octet-stream" }); diff --git a/tsconfig.build.json b/tsconfig.build.json new file mode 100644 index 0000000..4468d28 --- /dev/null +++ b/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "declaration": true, + "emitDeclarationOnly": true, + "rootDir": "./src", + "declarationDir": "./dist", + "declarationMap": true + }, + "include": ["src"] +} diff --git a/vite.config.ts b/vite.config.ts new file mode 100644 index 0000000..ba569b7 --- /dev/null +++ b/vite.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from "vite"; + +export default defineConfig({ + publicDir: false, + build: { + lib: { + entry: new URL("src/index.ts", import.meta.url).pathname, + fileName: "index", + formats: ["es"], + }, + sourcemap: "hidden", + }, +});