docs/imporve documentaions

docs/performance.md

@@ -0,0 +1,111 @@
+# Performance
+
+This section focuses on memory performance. When processing huge files, the goal is to keep the
+memory baseline flat and GC pauses to an absolute minimum, entirely independent of the input size.
+
+> [!NOTE]
+> The following examples are run on `Node.js` using a 1 GB JSON file. Performance profiling is
+> generated via `clinic`.
+
+## Passthrough
+
+This scenario demonstrates the absolute base cost of parsing. We use the core `JSONTextDecoder` to
+read chunks from a 1 GB file, tokenize them, and immediately discard the tokens.
+
+```javascript
+import { createReadStream } from "node:fs";
+import { JSONTextDecoder } from "jsontext";
+
+const decoder = new JSONTextDecoder();
+const stream = createReadStream("data.json");
+
+for await (const chunk of stream) {
+  decoder.push(chunk);
+
+  while (decoder.readToken() !== undefined) {
+    /** Drain */
+  }
+}
+
+decoder.end();
+decoder.checkEOF();
+```
+
+![Passthrough Result](https://github.com/user-attachments/assets/6d8d795b-ba11-41c1-8993-ac5e15088524)
+
+## Round Trip
+
+This scenario represents a full I/O cycle. We stream bytes from the 1 GB file, decode them into
+Tokens using the core `JSONTextDecoder`, immediately feed those tokens into `JSONTextEncoder`, and
+write the re-encoded bytes to a destination `/dev/null`.
+
+```javascript
+import { createReadStream, createWriteStream } from "node:fs";
+import { JSONTextDecoder, JSONTextEncoder } from "jsontext";
+
+const input = createReadStream("data.json");
+const output = createWriteStream("/dev/null");
+const decoder = new JSONTextDecoder();
+const encoder = new JSONTextEncoder();
+
+for await (const chunk of input) {
+  decoder.push(chunk);
+
+  for (let token; (token = decoder.readToken()) !== undefined;) {
+    encoder.writeToken(token);
+  }
+
+  const bytes = encoder.takeBytes();
+
+  if (bytes.length > 0) {
+    output.write(bytes);
+  }
+}
+
+decoder.end();
+decoder.checkEOF();
+output.end();
+```
+
+![Round Trip Result](https://github.com/user-attachments/assets/f8c6fc35-0227-40c3-98a2-c9503a366299)
+
+> [!IMPORTANT]
+> Using `JSONTextDecoderStream` and `JSONTextEncoderStream` directly in Node.js requires `.toWeb()`
+> to convert to Web Streams, which adds an extra buffering layer and can push Heap Used up to 300 MB
+> before triggering GC in this scenario.
+
+## Query
+
+This scenario demonstrates a data querying use case. We use `JSONTextSelectorStream` with a
+descendant JSON Path expression `$..id` to scan the entire 1 GB file. For each match, we call
+`json()` to decode the value into a JavaScript object, and keep a count of the total matches.
+
+Since `JSONTextSelectorStream` is a Web Streams `TransformStream`, we use `.toWeb()` to bridge
+`Node.js` streams.
+
+```javascript
+import { JSONTextSelectorStream } from "jsontext";
+import { createReadStream } from "node:fs";
+import { Readable } from "node:stream";
+
+const stream = createReadStream("data.json");
+const selector = new JSONTextSelectorStream("$..id");
+let count = 0;
+
+for await (const value of Readable.toWeb(stream).pipeThrough(selector)) {
+  value.json();
+  count++;
+}
+
+console.log(`Total values: ${count}`);
+// Total values: 565255 for the 1 GB file used in this example
+```
+
+![Query Result](https://github.com/user-attachments/assets/2a4e679f-e76f-43f5-bece-487d9a925b91)
+
+> [!TIP]
+> `JSONTextSelectorStream` only emits matched values, so the frequency of `.enqueue()` calls is low
+> bounded by the number of matches, not the number of tokens. This makes the microtask overhead from
+> Web Streams acceptable here. In the Round Trip scenario every token triggers an `.enqueue()`,
+> which creates enough microtask pressure to push Heap Used to 300 MB and trigger GC. That is why we
+> use the core APIs directly there instead.

package.json

@@ -2,8 +2,22 @@
   "name": "jsontext",
   "version": "0.2.1",
   "license": "MIT",
-  "description": "State machine for incremental JSON processing.",
-  "keywords": ["json", "stream"],
+  "description": "A state machine for incremental JSON processing.",
+  "keywords": [
+    "json",
+    "json-path",
+    "json-pointer",
+    "jsonl",
+    "jsonlines",
+    "parser",
+    "encoder",
+    "decoder",
+    "stream",
+    "streaming",
+    "web-streams",
+    "ndjson",
+    "state-machine"
+  ],
   "type": "module",
   "types": "./dist/index.d.ts",
   "imports": {

src/api/decoder.ts

@@ -5,6 +5,11 @@ import type Value from "#src/modules/value";
 import type { Kind } from "#src/types/kind";
 import type { DecoderOptions } from "#src/types/options";
 
+/**
+ * Options for {@link JSONTextDecoder}.
+ *
+ * @public
+ */
 type JSONTextDecoderOptions = DecoderOptions;
 
 /**

src/api/encoder.ts

@@ -4,6 +4,11 @@ import type Token from "#src/modules/token";
 import type Value from "#src/modules/value";
 import type { EncoderOptions } from "#src/types/options";
 
+/**
+ * Options for {@link JSONTextEncoder}.
+ *
+ * @public
+ */
 type JSONTextEncoderOptions = EncoderOptions;
 
 /**

src/libs/stream-decoder.ts

@@ -3,8 +3,15 @@ import Decoder from "#src/modules/decoder";
 import type Token from "#src/modules/token";
 import type { DecoderOptions } from "#src/types/options";
 
+/**
+ * Options for {@link JSONTextDecoderStream}.
+ *
+ * @public
+ */
 type JSONTextDecoderStreamOptions = DecoderOptions & {
+  /** Queuing strategy for the writable side. */
   writableStrategy?: QueuingStrategy<Uint8Array>;
+  /** Queuing strategy for the readable side. */
   readableStrategy?: QueuingStrategy<Token>;
 };
 

src/libs/stream-encoder.ts

@@ -3,8 +3,15 @@ import Encoder from "#src/modules/encoder";
 import type Token from "#src/modules/token";
 import type { EncoderOptions } from "#src/types/options";
 
+/**
+ * Options for {@link JSONTextEncoderStream}.
+ *
+ * @public
+ */
 type JSONTextEncoderStreamOptions = EncoderOptions & {
+  /** Queuing strategy for the writable side. */
   writableStrategy?: QueuingStrategy<Token>;
+  /** Queuing strategy for the readable side. */
   readableStrategy?: QueuingStrategy<Uint8Array>;
 };
 

src/libs/stream-line.ts

@@ -3,8 +3,15 @@ import Decoder from "#src/modules/decoder";
 import type Value from "#src/modules/value";
 import type { DecoderOptions } from "#src/types/options";
 
+/**
+ * Options for {@link JSONTextLineStream}.
+ *
+ * @public
+ */
 type JSONTextLineStreamOptions = DecoderOptions & {
+  /** Queuing strategy for the writable side. */
   writableStrategy?: QueuingStrategy<Uint8Array>;
+  /** Queuing strategy for the readable side. */
   readableStrategy?: QueuingStrategy<Value>;
 };
 

src/libs/stream-selector.ts

@@ -6,8 +6,15 @@ import Value from "#src/modules/value";
 import type { DecoderOptions } from "#src/types/options";
 import { encodeText } from "#src/utils/text";
 
+/**
+ * Options for {@link JSONTextSelectorStream}.
+ *
+ * @public
+ */
 type JSONTextSelectorStreamOptions = DecoderOptions & {
+  /** Queuing strategy for the writable side. */
   writableStrategy?: QueuingStrategy<Uint8Array>;
+  /** Queuing strategy for the readable side. */
   readableStrategy?: QueuingStrategy<Value>;
 };
 

src/modules/token.ts

@@ -155,7 +155,7 @@ class Token {
    * @returns A new `Token` backed by a cloned `Uint8Array`.
    */
   clone(): Token {
-    return new Token(this.bytes.slice());
+    return new Token(this.#bytes.slice());
   }
 
   /**
@@ -166,11 +166,11 @@ class Token {
    */
   isScalar(): boolean {
     return (
-      this.kind === KIND.STRING ||
-      this.kind === KIND.NUMBER ||
-      this.kind === KIND.TRUE ||
-      this.kind === KIND.FALSE ||
-      this.kind === KIND.NULL
+      this.#kind === KIND.STRING ||
+      this.#kind === KIND.NUMBER ||
+      this.#kind === KIND.TRUE ||
+      this.#kind === KIND.FALSE ||
+      this.#kind === KIND.NULL
     );
   }
 

src/modules/value.ts

@@ -88,7 +88,7 @@ class Value {
   /**
    * Returns a canonicalized copy of this value.
    *
-   * Canonicalization recursively sorts object keys by UTF-16 code unit order
+   * JSON Canonicalization Scheme (RFC 8785) recursively sorts object keys by UTF-16 code unit order
    * and normalizes numbers. The result is deterministic and idempotent.
    *
    * @returns A new `Value` in canonical form.
@@ -158,7 +158,7 @@ class Value {
    * @throws {TypeError} If the bytes contain invalid UTF-8 sequences.
    */
   text(): string {
-    return decodeText(this.bytes, true);
+    return decodeText(this.#bytes, true);
   }
 
   /**
@@ -172,7 +172,7 @@ class Value {
    * @throws {SyntacticError} If the bytes do not represent valid JSON.
    */
   *tokens(): Generator<Token> {
-    const decoder = new Decoder(this.bytes, DEFAULT_DECODER_OPTIONS);
+    const decoder = new Decoder(this.#bytes, DEFAULT_DECODER_OPTIONS);
 
     while (true) {
       const token = decoder.readToken();

src/types/kind.ts

@@ -1,3 +1,9 @@
 import type { KIND } from "#src/common/constants";
 
+/**
+ * A union type of all possible JSON token kind values, corresponding to the
+ * string discriminants in {@link KIND}.
+ *
+ * @public
+ */
 export type Kind = typeof KIND[keyof typeof KIND];

src/types/options.ts

@@ -1,17 +1,42 @@
+/**
+ * Base options for {@link DecoderOptions} and {@link EncoderOptions}.
+ *
+ * @internal
+ */
 export type BaseOptions = {
+  /** Allow duplicate object key names. By default, duplicate names throw a `SyntacticError`. */
   allowDuplicateNames?: boolean;
+  /** Allow invalid UTF-8 byte sequences. By default, invalid sequences throw a `TypeError`. */
   allowInvalidUTF8?: boolean;
 };
 
+/**
+ * Options for {@link Decoder}.
+ *
+ * @internal
+ */
 export type DecoderOptions = BaseOptions;
 
+/**
+ * Options for {@link Encoder}.
+ *
+ * @internal
+ */
 export type EncoderOptions = {
+  /** Escape `<`, `>`, and `&` for safe embedding in HTML. */
   escapeForHTML?: boolean;
+  /** Escape `\u2028` and `\u2029` for safe embedding in JavaScript string literals. */
   escapeForJS?: boolean;
+  /** Normalize number tokens to their canonical decimal form. */
   canonicalizeRawNumbers?: boolean;
+  /** Emit a space after each `:` separator in objects. */
   spaceAfterColon?: boolean;
+  /** Emit a space after each `,` separator in arrays and objects. */
   spaceAfterComma?: boolean;
+  /** Emit each value on its own line with indentation. */
   multiline?: boolean;
+  /** Indentation string used per nesting level when multiline is enabled. Defaults to two spaces. */
   indent?: string;
+  /** Prefix prepended to every indented line when multiline is enabled. */
   indentPrefix?: string;
 } & BaseOptions;