From 5bb2d56543d09cd39038167feedfa75ec7daa097 Mon Sep 17 00:00:00 2001 From: Elia <83713217+eliahilse@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:07:24 +0200 Subject: [PATCH 1/2] fix(bench,web): realistic Brotli baselines front and center MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The q11 row was the offline ceiling, not something any edge runs on dynamic responses — Cloudflare's documented dynamic level is 4. The site panel now compares against edge q4, the harness adds q6 (ngx_brotli default), and q11 stays in the harness labelled as the static-asset ceiling. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_hf1 --- apps/bench/README.md | 7 +++++-- apps/bench/src/run.ts | 5 +++++ apps/web/app/benchmark.tsx | 12 ++++++------ apps/web/app/page.tsx | 9 +++++---- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/apps/bench/README.md b/apps/bench/README.md index 9d7d35b..7028ba9 100644 --- a/apps/bench/README.md +++ b/apps/bench/README.md @@ -20,8 +20,11 @@ cd apps/bench && bun run bench - Deterministic seeds; payloads identical across contenders, verified by round-trip deep-equality before any timing. -- gzip level 6. Brotli q4 (labelled: latency-oriented dynamic setting) and q11 - (offline ceiling — report, don't pretend it's a dynamic choice), TEXT mode +- gzip level 6 (zlib/Express default). Brotli q4 — what edges actually run on + dynamic responses (Cloudflare's documented dynamic level) and the primary + baseline for any realistic claim; q6 (ngx_brotli's default) for + origin-compression setups; q11 is the offline ceiling for pre-compressed + static assets — report it, never present it as a dynamic choice. TEXT mode + size hint for JSON inputs, GENERIC for binary. - `hyperfly+br4` is included because transport compression exists in real deployments; it is a valid configuration, not an admission of failure. diff --git a/apps/bench/src/run.ts b/apps/bench/src/run.ts index 1a12266..a572026 100644 --- a/apps/bench/src/run.ts +++ b/apps/bench/src/run.ts @@ -49,6 +49,11 @@ function contenders(payload: unknown, row: AnyCodec, col: AnyCodec, proto: Proto encode: () => brotli(enc.encode(JSON.stringify(payload)), 4, constants.BROTLI_MODE_TEXT), decode: (b) => JSON.parse(dec.decode(brotliDecompressSync(b))), }, + { + name: "json+br6", + encode: () => brotli(enc.encode(JSON.stringify(payload)), 6, constants.BROTLI_MODE_TEXT), + decode: (b) => JSON.parse(dec.decode(brotliDecompressSync(b))), + }, { name: "json+br11", encode: () => brotli(enc.encode(JSON.stringify(payload)), 11, constants.BROTLI_MODE_TEXT), diff --git a/apps/web/app/benchmark.tsx b/apps/web/app/benchmark.tsx index e4fa3d8..0aad98c 100644 --- a/apps/web/app/benchmark.tsx +++ b/apps/web/app/benchmark.tsx @@ -23,12 +23,12 @@ const PAYLOADS: Payload[] = [ rows: [ { label: "JSON", bytes: 88209, kind: "baseline" }, { label: "JSON + gzip", bytes: 21705, kind: "generic" }, - { label: "JSON + Brotli (max)", bytes: 14716, kind: "generic" }, + { label: "JSON + Brotli — edge q4", bytes: 21315, kind: "generic" }, { label: "Protobuf", bytes: 56996, kind: "binary" }, { label: "Hyperfly · columnar", bytes: 12628, kind: "hyperfly" }, { label: "Hyperfly + Brotli", bytes: 7917, kind: "profile" }, ], - note: "Columns ride separately: timestamps become deltas, and prices that are exact decimals travel as integer mantissas instead of eight raw bytes. No entropy coder is involved yet — this is layout alone beating Brotli's best effort.", + note: "Columns ride separately: timestamps become deltas, and prices that are exact decimals travel as integer mantissas instead of eight raw bytes. No entropy coder is involved yet — layout alone, uncompressed, undercuts what the edge actually serves. It clears Brotli's offline q11 ceiling too; the harness in the repo has the receipts.", }, { route: "GET /v1/devices", @@ -36,12 +36,12 @@ const PAYLOADS: Payload[] = [ rows: [ { label: "JSON", bytes: 113443, kind: "baseline" }, { label: "JSON + gzip", bytes: 17026, kind: "generic" }, - { label: "JSON + Brotli (max)", bytes: 12609, kind: "generic" }, + { label: "JSON + Brotli — edge q4", bytes: 16823, kind: "generic" }, { label: "Protobuf", bytes: 29056, kind: "binary" }, { label: "Hyperfly · columnar", bytes: 17981, kind: "hyperfly" }, { label: "Hyperfly + Brotli", bytes: 9446, kind: "profile" }, ], - note: "An enum with six members is an index, not a string. Bounded integers ship as offsets from their declared minimum, booleans pack into bitmaps, and transport compression stacks on top of all of it.", + note: "An enum with six members is an index, not a string. Bounded integers ship as offsets from their declared minimum, booleans pack into bitmaps. Standalone this sits at parity with edge Brotli — the win here comes from stacking, because columnar bytes compress far better than JSON does.", }, { route: "GET /v1/feed", @@ -49,12 +49,12 @@ const PAYLOADS: Payload[] = [ rows: [ { label: "JSON", bytes: 22245, kind: "baseline" }, { label: "JSON + gzip", bytes: 7775, kind: "generic" }, - { label: "JSON + Brotli (max)", bytes: 6576, kind: "generic" }, + { label: "JSON + Brotli — edge q4", bytes: 7691, kind: "generic" }, { label: "Protobuf", bytes: 15232, kind: "binary" }, { label: "Hyperfly", bytes: 14560, kind: "hyperfly" }, { label: "Hyperfly + Brotli", bytes: 7225, kind: "profile" }, ], - note: "Structure compresses; prose does not. When the payload is mostly human text, Brotli keeps the crown — Hyperfly removes the envelope and steps aside. Route-trained dictionaries are what change this picture, and they do not exist yet.", + note: "Structure compresses; prose does not. On mostly-text payloads Hyperfly lands at a wash with edge Brotli, and Brotli's offline ceiling still wins outright. Route-trained dictionaries are what change this picture, and they do not exist yet.", }, ]; diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx index 03a05cc..bfb42b1 100644 --- a/apps/web/app/page.tsx +++ b/apps/web/app/page.tsx @@ -164,10 +164,11 @@ export default function Home() {

Measured with the TypeScript reference implementation on deterministic synthetic - corpora — reproduce with `bun run bench` in the repo. Results depend entirely on the - payload: structure compresses, prose does not, and the feed tab above is the loss - case shown on purpose. Baselines: gzip −6, Brotli q11 (its offline ceiling; the - dynamic q4 setting is larger), Protobuf with proper enums and int64. + corpora — reproduce with `bun run bench` in the repo. The Brotli row is q4, the + level edges actually run on dynamic responses; q6 (nginx's default) and the q11 + offline ceiling are in the harness, and columnar clears even q11 on the candles + corpus. Protobuf gets proper enums and int64. Results depend entirely on the + payload: structure compresses, prose does not, and the feed tab is the honest wash.

From bab7045b11f150af3c1cfde7c9df9ca1f80bb8ce Mon Sep 17 00:00:00 2001 From: Elia <83713217+eliahilse@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:15:50 +0200 Subject: [PATCH 2/2] feat: packed string columns + nested-struct flattening (columnar@2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit String columns gain a mode byte: plain, or packed — per-value lengths plus one raw-deflate stream of the concatenated UTF-8, so prose compresses with shared context across every row and unpacks bit-exactly. The hooks are pluggable (node:zlib resolved via process.getBuiltinModule so browsers can import the core; a decoder without inflate rejects packed columns as unsupported and the protocol falls back to JSON). Deflate output is not canonical across libraries, so the re-encode guarantee is scoped to one implementation for this mode; any spec-valid stream decodes everywhere, and the golden decode vector uses a hand-built stored block to stay inflater-agnostic. Eligibility extends to required non-nullable nested structs, flattened depth-first into leaf columns — which is what lets the feed corpus (nested authors) reach the packed columns at all. The plan bumps to columnar@2 (v1 never shipped). Harness: feed-50 hf-col is now 6,513 B standalone, ahead of json+br11 at 6,576 B; devices-500 11,945 B — uncompressed columnar clears the q11 ceiling on all three corpora. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_hf1 --- apps/bench/README.md | 2 +- apps/web/app/benchmark.tsx | 14 +- apps/web/app/page.tsx | 7 +- packages/hyperfly/src/canonical.ts | 4 +- packages/hyperfly/src/codec.ts | 12 +- packages/hyperfly/src/columnar.ts | 215 ++++++++++++++++-- packages/hyperfly/src/decode.ts | 19 +- packages/hyperfly/src/encode.ts | 11 +- packages/hyperfly/src/errors.ts | 1 + packages/hyperfly/src/index.ts | 3 +- packages/hyperfly/src/pack.ts | 31 +++ packages/hyperfly/test/columnar.test.ts | 49 ++++ ...lan-columnar-v1.md => plan-columnar-v2.md} | 47 ++-- spec/vectors/columnar.json | 210 ++++++++++++++--- spec/vectors/fingerprints.json | 20 +- 15 files changed, 539 insertions(+), 106 deletions(-) create mode 100644 packages/hyperfly/src/pack.ts rename spec/{plan-columnar-v1.md => plan-columnar-v2.md} (63%) diff --git a/apps/bench/README.md b/apps/bench/README.md index 7028ba9..8da3bae 100644 --- a/apps/bench/README.md +++ b/apps/bench/README.md @@ -14,7 +14,7 @@ cd apps/bench && bun run bench |---|---|---| | candles | numeric OHLCV rows, f64-heavy | weak under plan `row`; the flagship under plan `columnar` (delta timestamps, scaled-decimal prices) | | devices | enums, bounded ints, booleans | favorable for schema-only encoding under either plan | -| feed | prose bodies, ids, names | honest loss case — Brotli eats text, schemas don't; the nested author struct keeps the posts array on the row path even under `columnar` | +| feed | prose bodies, ids, names | thinnest margin — text columns deflate inside the codec (packed string mode), nested authors flatten into leaf columns | ## Fairness rules diff --git a/apps/web/app/benchmark.tsx b/apps/web/app/benchmark.tsx index 0aad98c..03e35be 100644 --- a/apps/web/app/benchmark.tsx +++ b/apps/web/app/benchmark.tsx @@ -26,7 +26,7 @@ const PAYLOADS: Payload[] = [ { label: "JSON + Brotli — edge q4", bytes: 21315, kind: "generic" }, { label: "Protobuf", bytes: 56996, kind: "binary" }, { label: "Hyperfly · columnar", bytes: 12628, kind: "hyperfly" }, - { label: "Hyperfly + Brotli", bytes: 7917, kind: "profile" }, + { label: "Hyperfly + Brotli", bytes: 7915, kind: "profile" }, ], note: "Columns ride separately: timestamps become deltas, and prices that are exact decimals travel as integer mantissas instead of eight raw bytes. No entropy coder is involved yet — layout alone, uncompressed, undercuts what the edge actually serves. It clears Brotli's offline q11 ceiling too; the harness in the repo has the receipts.", }, @@ -38,10 +38,10 @@ const PAYLOADS: Payload[] = [ { label: "JSON + gzip", bytes: 17026, kind: "generic" }, { label: "JSON + Brotli — edge q4", bytes: 16823, kind: "generic" }, { label: "Protobuf", bytes: 29056, kind: "binary" }, - { label: "Hyperfly · columnar", bytes: 17981, kind: "hyperfly" }, - { label: "Hyperfly + Brotli", bytes: 9446, kind: "profile" }, + { label: "Hyperfly · columnar", bytes: 11945, kind: "hyperfly" }, + { label: "Hyperfly + Brotli", bytes: 9726, kind: "profile" }, ], - note: "An enum with six members is an index, not a string. Bounded integers ship as offsets from their declared minimum, booleans pack into bitmaps. Standalone this sits at parity with edge Brotli — the win here comes from stacking, because columnar bytes compress far better than JSON does.", + note: "An enum with six members is an index, not a string. Bounded integers ship as offsets from their declared minimum, booleans pack into bitmaps, and repetitive id columns deflate inside the codec — so the uncompressed wire already undercuts what the edge serves.", }, { route: "GET /v1/feed", @@ -51,10 +51,10 @@ const PAYLOADS: Payload[] = [ { label: "JSON + gzip", bytes: 7775, kind: "generic" }, { label: "JSON + Brotli — edge q4", bytes: 7691, kind: "generic" }, { label: "Protobuf", bytes: 15232, kind: "binary" }, - { label: "Hyperfly", bytes: 14560, kind: "hyperfly" }, - { label: "Hyperfly + Brotli", bytes: 7225, kind: "profile" }, + { label: "Hyperfly · columnar", bytes: 6513, kind: "hyperfly" }, + { label: "Hyperfly + Brotli", bytes: 6443, kind: "profile" }, ], - note: "Structure compresses; prose does not. On mostly-text payloads Hyperfly lands at a wash with edge Brotli, and Brotli's offline ceiling still wins outright. Route-trained dictionaries are what change this picture, and they do not exist yet.", + note: "Prose does not vanish under a schema — so text columns pack through deflate inside the codec, with shared context across every row, and unpack bit-exactly. Structure travels as columns around them. The all-text route now lands ahead of Brotli's offline ceiling instead of behind it.", }, ]; diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx index bfb42b1..4fc16d1 100644 --- a/apps/web/app/page.tsx +++ b/apps/web/app/page.tsx @@ -166,9 +166,10 @@ export default function Home() { Measured with the TypeScript reference implementation on deterministic synthetic corpora — reproduce with `bun run bench` in the repo. The Brotli row is q4, the level edges actually run on dynamic responses; q6 (nginx's default) and the q11 - offline ceiling are in the harness, and columnar clears even q11 on the candles - corpus. Protobuf gets proper enums and int64. Results depend entirely on the - payload: structure compresses, prose does not, and the feed tab is the honest wash. + offline ceiling are in the harness — and uncompressed columnar output clears even + q11 on all three corpora. Protobuf gets proper enums and int64. Results still + depend on the payload; the feed tab is where the margin is thinnest, and it says + why.

diff --git a/packages/hyperfly/src/canonical.ts b/packages/hyperfly/src/canonical.ts index f688233..07e2147 100644 --- a/packages/hyperfly/src/canonical.ts +++ b/packages/hyperfly/src/canonical.ts @@ -59,8 +59,10 @@ export function serializeNode(node: IRNode): string { export type PlanLayout = "row" | "columnar"; +const PLAN_VERSION: Record = { row: 1, columnar: 2 }; + export function serializeArtifact(ir: IRNode, layout: PlanLayout = "row"): string { - return `{"wire":1,"plan":{"layout":"${layout}","version":1},"ir":${serializeNode(ir)}}`; + return `{"wire":1,"plan":{"layout":"${layout}","version":${PLAN_VERSION[layout]}},"ir":${serializeNode(ir)}}`; } export function fingerprintOf(artifact: string): Uint8Array { diff --git a/packages/hyperfly/src/codec.ts b/packages/hyperfly/src/codec.ts index 3d5061f..db5c199 100644 --- a/packages/hyperfly/src/codec.ts +++ b/packages/hyperfly/src/codec.ts @@ -1,4 +1,5 @@ import { fingerprintOf, serializeArtifact, toHex, type PlanLayout } from "./canonical.js"; +import { defaultPackHooks } from "./pack.js"; import { decodeNode } from "./decode.js"; import { encodeNode } from "./encode.js"; import { DecodeError, FingerprintMismatchError } from "./errors.js"; @@ -10,9 +11,15 @@ export const MAGIC = new Uint8Array([0x68, 0x66]); export const WIRE_VERSION = 1; export const HEADER_SIZE = 19; +export interface PackHooks { + deflate?: (data: Uint8Array) => Uint8Array; + inflate?: (data: Uint8Array, maxOutputLength: number) => Uint8Array; +} + export interface CompileOptions { limits?: Partial; plan?: PlanLayout; + pack?: PackHooks | false; } export interface Codec { @@ -34,16 +41,17 @@ export function compileIR(ir: IRNode, options: CompileOptions = {}) const fingerprint = toHex(fingerprintBytes); const limits: DecodeLimits = { ...DEFAULT_LIMITS, ...options.limits }; const columnar = plan === "columnar"; + const pack = options.pack === false ? {} : (options.pack ?? defaultPackHooks()); const encodeBody = (value: T): Uint8Array => { const w = new Writer(); - encodeNode(w, ir, value, "$", 0, { maxDepth: limits.maxDepth, columnar }); + encodeNode(w, ir, value, "$", 0, { maxDepth: limits.maxDepth, columnar, deflate: pack.deflate }); return w.finish(); }; const decodeBody = (bytes: Uint8Array): T => { const r = new Reader(bytes, limits); - const value = decodeNode(r, ir, "$", 0, columnar); + const value = decodeNode(r, ir, "$", 0, columnar, pack.inflate); r.expectEnd(); return value as T; }; diff --git a/packages/hyperfly/src/columnar.ts b/packages/hyperfly/src/columnar.ts index 13a5613..c9ca2dc 100644 --- a/packages/hyperfly/src/columnar.ts +++ b/packages/hyperfly/src/columnar.ts @@ -1,7 +1,7 @@ -import { decodeNode } from "./decode.js"; -import { encodeNode, typeAcceptsNull, writeBitmap, type EncodeCtx } from "./encode.js"; +import { decodeNode, type Inflate } from "./decode.js"; +import { encodeNode, typeAcceptsNull, utf8Bytes, writeBitmap, type EncodeCtx } from "./encode.js"; import { DecodeError, EncodeError } from "./errors.js"; -import type { IRNode } from "./ir.js"; +import type { IRField, IRNode } from "./ir.js"; import { readBitmap } from "./decode.js"; import type { Reader } from "./reader.js"; import { INT_MAX, INT_MIN, readUleb, ulebLen, unzigzag, writeUleb, zigzag } from "./varint.js"; @@ -13,12 +13,36 @@ type IntNode = Extract; const COLUMN_KINDS = new Set(["bool", "int", "float64", "string", "bytes", "enum", "literal"]); +interface Leaf { + segs: readonly string[]; + field: IRField; +} + +/** + * Depth-first leaf columns in declared order. Nested structs flatten only when + * required and non-nullable, so every leaf inherits its row's participation. + */ +export function flattenLeaves(element: StructNode): Leaf[] | null { + const out: Leaf[] = []; + const walk = (node: StructNode, segs: string[]): boolean => { + if (node.fields.length === 0) return false; + for (const f of node.fields) { + if (f.type.kind === "struct") { + if (f.optional || f.nullable) return false; + if (!walk(f.type, [...segs, f.name])) return false; + } else if (COLUMN_KINDS.has(f.type.kind)) { + out.push({ segs: [...segs, f.name], field: f }); + } else { + return false; + } + } + return true; + }; + return walk(element, []) ? out : null; +} + export function columnarEligible(node: ArrayNode): boolean { - return ( - node.element.kind === "struct" && - node.element.fields.length > 0 && - node.element.fields.every((f) => COLUMN_KINDS.has(f.type.kind)) - ); + return node.element.kind === "struct" && flattenLeaves(node.element) !== null; } const scratch = new DataView(new ArrayBuffer(8)); @@ -260,6 +284,110 @@ function decodeFloatColumn(r: Reader, count: number, path: string): number[] { return out; } +function encodeStringColumn(w: Writer, values: unknown[], path: string, ctx: EncodeCtx): void { + if (values.length === 0) { + w.u8(0); + return; + } + const bytes = values.map((v, i) => utf8Bytes(v, `${path}[${i}]`)); + const plainCost = bytes.reduce((n, b) => n + ulebLen(BigInt(b.length)) + b.length, 0); + + let packed: Uint8Array | null = null; + let packedCost = Infinity; + if (ctx.deflate) { + const total = bytes.reduce((n, b) => n + b.length, 0); + const concat = new Uint8Array(total); + let offset = 0; + for (const b of bytes) { + concat.set(b, offset); + offset += b.length; + } + packed = ctx.deflate(concat); + packedCost = + bytes.reduce((n, b) => n + ulebLen(BigInt(b.length)), 0) + + ulebLen(BigInt(packed.length)) + + packed.length; + } + + if (packed && packedCost < plainCost) { + w.u8(1); + for (const b of bytes) writeUleb(w, BigInt(b.length)); + writeUleb(w, BigInt(packed.length)); + w.bytes(packed); + return; + } + w.u8(0); + for (const b of bytes) { + writeUleb(w, BigInt(b.length)); + w.bytes(b); + } +} + +const utf8Strict = new TextDecoder("utf-8", { fatal: true }); + +function decodeStringColumn(r: Reader, count: number, path: string, inflate?: Inflate): string[] { + const mode = r.u8(); + if (mode > 1) throw new DecodeError("marker", `${path}: invalid string column mode 0x${mode.toString(16)}`); + const out: string[] = new Array(count); + if (count === 0) return out; + + const decodeSlice = (bytes: Uint8Array, i: number): string => { + try { + return utf8Strict.decode(bytes); + } catch { + throw new DecodeError("utf8", `${path}[${i}]: invalid UTF-8`); + } + }; + + if (mode === 0) { + for (let i = 0; i < count; i++) { + const raw = readUleb(r); + if (raw > BigInt(r.limits.maxByteLength)) { + throw new DecodeError("limit", `${path}[${i}]: string length exceeds limit`); + } + out[i] = decodeSlice(r.bytes(Number(raw)), i); + } + return out; + } + + const lengths: number[] = new Array(count); + let total = 0; + for (let i = 0; i < count; i++) { + const raw = readUleb(r); + if (raw > BigInt(r.limits.maxByteLength)) { + throw new DecodeError("limit", `${path}[${i}]: string length exceeds limit`); + } + lengths[i] = Number(raw); + total += lengths[i]!; + if (total > r.limits.maxByteLength) { + throw new DecodeError("limit", `${path}: packed column total exceeds limit`); + } + } + const blobLenRaw = readUleb(r); + if (blobLenRaw > BigInt(r.limits.maxByteLength)) { + throw new DecodeError("limit", `${path}: packed blob exceeds limit`); + } + const blob = r.bytes(Number(blobLenRaw)); + if (!inflate) { + throw new DecodeError("unsupported", `${path}: packed string column requires an inflate hook`); + } + let inflated: Uint8Array; + try { + inflated = inflate(blob, total); + } catch { + throw new DecodeError("packed", `${path}: packed blob failed to inflate`); + } + if (inflated.length !== total) { + throw new DecodeError("packed", `${path}: packed blob inflates to ${inflated.length} bytes, expected ${total}`); + } + let offset = 0; + for (let i = 0; i < count; i++) { + out[i] = decodeSlice(inflated.subarray(offset, offset + lengths[i]!), i); + offset += lengths[i]!; + } + return out; +} + function encodeBoolColumn(w: Writer, values: unknown[], path: string): void { const bits = values.map((v, i) => { if (typeof v !== "boolean") throw new EncodeError("type", `${path}[${i}]: expected boolean`); @@ -298,16 +426,36 @@ export function encodeColumnarArray( return row as Record; }); - for (const field of element.fields) { - const fieldPath = `${path}[].${field.name}`; - const states: RowState[] = rows.map((row, i) => { - const v = row[field.name]; + const leaves = flattenLeaves(element)!; + + const containerOf = (row: Record, segs: readonly string[], i: number): Record => { + let obj: Record = row; + for (let d = 0; d < segs.length - 1; d++) { + const v = obj[segs[d]!]; + if (typeof v !== "object" || v === null || Array.isArray(v)) { + throw new EncodeError( + v === undefined ? "required" : "type", + `${path}[${i}].${segs.slice(0, d + 1).join(".")}: expected object`, + ); + } + obj = v as Record; + } + return obj; + }; + + for (const leaf of leaves) { + const field = leaf.field; + const leafName = leaf.segs[leaf.segs.length - 1]!; + const dotted = leaf.segs.join("."); + const fieldPath = `${path}[].${dotted}`; + const values: unknown[] = rows.map((row, i) => containerOf(row, leaf.segs, i)[leafName]); + const states: RowState[] = values.map((v, i) => { const absent = v === undefined; if (absent && !field.optional) { - throw new EncodeError("required", `${path}[${i}].${field.name}: required field missing`); + throw new EncodeError("required", `${path}[${i}].${dotted}: required field missing`); } if (v === null && !field.nullable && !typeAcceptsNull(field.type)) { - throw new EncodeError("type", `${path}[${i}].${field.name}: null for non-nullable field`); + throw new EncodeError("type", `${path}[${i}].${dotted}: null for non-nullable field`); } return { present: !absent, isNull: !absent && v === null }; }); @@ -320,7 +468,7 @@ export function encodeColumnarArray( const s = states[i]!; if (!s.present) continue; if (s.isNull && field.nullable) continue; - participating.push(rows[i]![field.name]); + participating.push(values[i]); } switch (field.type.kind) { @@ -337,6 +485,9 @@ export function encodeColumnarArray( case "bool": encodeBoolColumn(w, participating, fieldPath); break; + case "string": + encodeStringColumn(w, participating, fieldPath, ctx); + break; default: for (let i = 0; i < participating.length; i++) { encodeNode(w, field.type, participating[i], `${fieldPath}[${i}]`, depth + 2, ctx); @@ -350,6 +501,7 @@ export function decodeColumnarArray( node: ArrayNode, path: string, depth: number, + inflate?: Inflate, ): Record[] { const element = node.element as StructNode; let count: number; @@ -364,9 +516,21 @@ export function decodeColumnarArray( } const out: Record[] = Array.from({ length: count }, () => ({})); + const leaves = flattenLeaves(element)!; - for (const field of element.fields) { - const fieldPath = `${path}[].${field.name}`; + const containerOf = (row: Record, segs: readonly string[]): Record => { + let obj = row; + for (let d = 0; d < segs.length - 1; d++) { + const seg = segs[d]!; + obj = (obj[seg] ??= {}) as Record; + } + return obj; + }; + + for (const leaf of leaves) { + const field = leaf.field; + const leafName = leaf.segs[leaf.segs.length - 1]!; + const fieldPath = `${path}[].${leaf.segs.join(".")}`; const presence = field.optional ? readBitmap(r, count, fieldPath) : null; const nulls = field.nullable ? readBitmap(r, count, fieldPath) : null; @@ -375,11 +539,11 @@ export function decodeColumnarArray( const present = presence ? presence[i]! : true; const isNull = nulls ? nulls[i]! : false; if (!present) { - if (isNull) throw new DecodeError("bitmap", `${path}[${i}].${field.name}: null bit set for absent field`); + if (isNull) throw new DecodeError("bitmap", `${path}[${i}].${leafName}: null bit set for absent field`); continue; } if (isNull) { - out[i]![field.name] = null; + containerOf(out[i]!, leaf.segs)[leafName] = null; continue; } slots.push(i); @@ -388,22 +552,27 @@ export function decodeColumnarArray( switch (field.type.kind) { case "int": { const values = decodeIntColumn(r, field.type as IntNode, slots.length, fieldPath); - slots.forEach((row, j) => (out[row]![field.name] = values[j]!)); + slots.forEach((row, j) => (containerOf(out[row]!, leaf.segs)[leafName] = values[j]!)); break; } case "float64": { const values = decodeFloatColumn(r, slots.length, fieldPath); - slots.forEach((row, j) => (out[row]![field.name] = values[j]!)); + slots.forEach((row, j) => (containerOf(out[row]!, leaf.segs)[leafName] = values[j]!)); break; } case "bool": { const values = readBitmap(r, slots.length, fieldPath); - slots.forEach((row, j) => (out[row]![field.name] = values[j]!)); + slots.forEach((row, j) => (containerOf(out[row]!, leaf.segs)[leafName] = values[j]!)); + break; + } + case "string": { + const values = decodeStringColumn(r, slots.length, fieldPath, inflate); + slots.forEach((row, j) => (containerOf(out[row]!, leaf.segs)[leafName] = values[j]!)); break; } default: { for (const row of slots) { - out[row]![field.name] = decodeNode(r, field.type, `${path}[${row}].${field.name}`, depth + 2, false); + containerOf(out[row]!, leaf.segs)[leafName] = decodeNode(r, field.type, `${path}[${row}].${leafName}`, depth + 2, false); } } } diff --git a/packages/hyperfly/src/decode.ts b/packages/hyperfly/src/decode.ts index e043ab8..3f9b9df 100644 --- a/packages/hyperfly/src/decode.ts +++ b/packages/hyperfly/src/decode.ts @@ -43,7 +43,16 @@ export function readBitmap(r: Reader, count: number, path: string): boolean[] { return bits; } -export function decodeNode(r: Reader, node: IRNode, path: string, depth: number, columnar: boolean): unknown { +export type Inflate = (data: Uint8Array, maxOutputLength: number) => Uint8Array; + +export function decodeNode( + r: Reader, + node: IRNode, + path: string, + depth: number, + columnar: boolean, + inflate?: Inflate, +): unknown { if (depth > r.limits.maxDepth) fail("depth", path, `nesting deeper than ${r.limits.maxDepth}`); switch (node.kind) { @@ -87,15 +96,15 @@ export function decodeNode(r: Reader, node: IRNode, path: string, depth: number, const marker = r.u8(); if (marker === 0) return null; if (marker !== 1) fail("marker", path, `invalid nullable marker 0x${marker.toString(16)}`); - return decodeNode(r, node.inner, path, depth + 1, columnar); + return decodeNode(r, node.inner, path, depth + 1, columnar, inflate); } case "array": { if (columnar && columnarEligible(node)) { - return decodeColumnarArray(r, node, path, depth); + return decodeColumnarArray(r, node, path, depth, inflate); } const count = node.length ?? readCount(r, r.limits.maxItems, "array count", path); const out = new Array(count); - for (let i = 0; i < count; i++) out[i] = decodeNode(r, node.element, `${path}[${i}]`, depth + 1, columnar); + for (let i = 0; i < count; i++) out[i] = decodeNode(r, node.element, `${path}[${i}]`, depth + 1, columnar, inflate); return out; } case "struct": { @@ -117,7 +126,7 @@ export function decodeNode(r: Reader, node: IRNode, path: string, depth: number, out[field.name] = null; continue; } - out[field.name] = decodeNode(r, field.type, `${path}.${field.name}`, depth + 1, columnar); + out[field.name] = decodeNode(r, field.type, `${path}.${field.name}`, depth + 1, columnar, inflate); } return out; } diff --git a/packages/hyperfly/src/encode.ts b/packages/hyperfly/src/encode.ts index 0e86353..2dfc156 100644 --- a/packages/hyperfly/src/encode.ts +++ b/packages/hyperfly/src/encode.ts @@ -26,12 +26,19 @@ function checkSurrogates(s: string, path: string): void { export interface EncodeCtx { maxDepth: number; columnar: boolean; + deflate?: (data: Uint8Array) => Uint8Array; } export function typeAcceptsNull(node: IRNode): boolean { return node.kind === "nullable" || (node.kind === "literal" && node.value === null); } +export function utf8Bytes(value: unknown, path: string): Uint8Array { + if (typeof value !== "string") fail("type", path, "expected string"); + checkSurrogates(value, path); + return encoder.encode(value); +} + function encodeInt(w: Writer, node: Extract, value: unknown, path: string): void { if (typeof value !== "number" || !Number.isSafeInteger(value)) { fail("type", path, `expected a safe integer, got ${typeof value === "number" ? value : typeof value}`); @@ -72,9 +79,7 @@ export function encodeNode(w: Writer, node: IRNode, value: unknown, path: string return; } case "string": { - if (typeof value !== "string") fail("type", path, "expected string"); - checkSurrogates(value, path); - const bytes = encoder.encode(value); + const bytes = utf8Bytes(value, path); writeUleb(w, BigInt(bytes.length)); w.bytes(bytes); return; diff --git a/packages/hyperfly/src/errors.ts b/packages/hyperfly/src/errors.ts index b7b9538..ab07e9a 100644 --- a/packages/hyperfly/src/errors.ts +++ b/packages/hyperfly/src/errors.ts @@ -14,6 +14,7 @@ export type ErrorCode = | "header" | "fingerprint" | "ir" + | "packed" | "unsupported"; export class HyperflyError extends Error { diff --git a/packages/hyperfly/src/index.ts b/packages/hyperfly/src/index.ts index c84bbc5..df129b8 100644 --- a/packages/hyperfly/src/index.ts +++ b/packages/hyperfly/src/index.ts @@ -1,4 +1,5 @@ -export { compileIR, HEADER_SIZE, MAGIC, WIRE_VERSION, type Codec, type CompileOptions } from "./codec.js"; +export { compileIR, HEADER_SIZE, MAGIC, WIRE_VERSION, type Codec, type CompileOptions, type PackHooks } from "./codec.js"; +export { defaultPackHooks } from "./pack.js"; export { serializeArtifact, serializeNode, fingerprintOf, toHex, type PlanLayout } from "./canonical.js"; export { columnarEligible } from "./columnar.js"; export { validateIR, type IRField, type IRNode, type LiteralValue } from "./ir.js"; diff --git a/packages/hyperfly/src/pack.ts b/packages/hyperfly/src/pack.ts new file mode 100644 index 0000000..aec2c50 --- /dev/null +++ b/packages/hyperfly/src/pack.ts @@ -0,0 +1,31 @@ +import type { PackHooks } from "./codec.js"; + +interface ZlibLike { + deflateRawSync(data: Uint8Array, options?: { level?: number }): Uint8Array; + inflateRawSync(data: Uint8Array, options?: { maxOutputLength?: number }): Uint8Array; +} + +/** + * node:zlib resolved at runtime so the module stays importable in browsers, + * where packing needs explicit hooks (DecompressionStream is async and cannot + * back the sync codec API). + */ +function builtinZlib(): ZlibLike | null { + const get = (globalThis as { process?: { getBuiltinModule?: (id: string) => unknown } }).process + ?.getBuiltinModule; + if (typeof get !== "function") return null; + try { + return (get("node:zlib") as ZlibLike) ?? null; + } catch { + return null; + } +} + +export function defaultPackHooks(): PackHooks { + const zlib = builtinZlib(); + if (!zlib) return {}; + return { + deflate: (data) => new Uint8Array(zlib.deflateRawSync(data, { level: 6 })), + inflate: (data, maxOutputLength) => new Uint8Array(zlib.inflateRawSync(data, { maxOutputLength })), + }; +} diff --git a/packages/hyperfly/test/columnar.test.ts b/packages/hyperfly/test/columnar.test.ts index 8fcecc0..838463c 100644 --- a/packages/hyperfly/test/columnar.test.ts +++ b/packages/hyperfly/test/columnar.test.ts @@ -51,6 +51,55 @@ describe("columnar golden vectors: invalid encode", () => { } }); +describe("columnar packed vectors: decode-only", () => { + for (const v of vectors.packedDecode) { + test(v.name, () => { + const codec = compileIR(v.ir as IRNode, { plan: "columnar" }); + expect(codec.decodeBody(fromHex(v.hex))).toEqual(v.value); + }); + } + + test("packed input without an inflate hook fails closed", () => { + const v = vectors.packedDecode[0]!; + const codec = compileIR(v.ir as IRNode, { plan: "columnar", pack: false }); + try { + codec.decodeBody(fromHex(v.hex)); + throw new Error("expected failure"); + } catch (err) { + expect((err as HyperflyError).code).toBe("unsupported"); + } + }); +}); + +describe("packed string columns", () => { + const IR: IRNode = { + kind: "array", + element: { kind: "struct", fields: [{ name: "body", type: { kind: "string" } }] }, + }; + const prose = Array.from({ length: 40 }, (_, i) => ({ + body: `the quick brown fox jumps over the lazy dog and files report number ${i} about the same fox again`, + })); + + test("prose columns pack, shrink, and round-trip", () => { + const packedCodec = compileIR(IR, { plan: "columnar" }); + const plainCodec = compileIR(IR, { plan: "columnar", pack: false }); + const packed = packedCodec.encodeBody(prose); + const plain = plainCodec.encodeBody(prose); + expect(packed.length).toBeLessThan(plain.length * 0.5); + expect(packedCodec.decodeBody(packed)).toEqual(prose); + expect(plainCodec.decodeBody(plain)).toEqual(prose); + const again = packedCodec.encodeBody(packedCodec.decodeBody(packed)); + expect(Buffer.from(again).equals(Buffer.from(packed))).toBe(true); + }); + + test("tiny strings stay plain even with hooks available", () => { + const codec = compileIR(IR, { plan: "columnar" }); + const value = [{ body: "a" }, { body: "b" }]; + const plain = compileIR(IR, { plan: "columnar", pack: false }).encodeBody(value); + expect(Buffer.from(codec.encodeBody(value)).equals(Buffer.from(plain))).toBe(true); + }); +}); + describe("plan separation", () => { const IR: IRNode = { kind: "array", diff --git a/spec/plan-columnar-v1.md b/spec/plan-columnar-v2.md similarity index 63% rename from spec/plan-columnar-v1.md rename to spec/plan-columnar-v2.md index aee2052..0c31378 100644 --- a/spec/plan-columnar-v1.md +++ b/spec/plan-columnar-v2.md @@ -1,20 +1,24 @@ -# Hyperfly plan `columnar` — v1 +# Hyperfly plan `columnar` — v2 Status: draft. Extends `spec/wire-v0.md`; everything there (envelope, varints, bitmaps, scalar encodings, limits, canonical serialization) applies unchanged. -The artifact is `{"wire":1,"plan":{"layout":"columnar","version":1},"ir":…}` — +The artifact is `{"wire":1,"plan":{"layout":"columnar","version":2},"ir":…}` — a different fingerprint than the row plan for the same IR, so the two never -mix on the wire. +mix on the wire. (v1 lacked string column modes and was never released; no v1 +artifact exists in the wild.) ## 1. Scope Under this plan, every **eligible** `array` node is encoded column-wise. -Eligible: the element is a `struct` with at least one field, and every field -type is one of `bool`, `int`, `float64`, `string`, `bytes`, `enum`, -`literal`. Field-level `optional`/`nullable` flags are allowed. Everything -else — including arrays whose elements nest structs or arrays — falls back to -the row encoding from wire-v0 §4.8/§4.10. Eligibility is a pure function of -the IR, so both sides always agree. +Eligible: the element is a `struct` whose fields are primitives (`bool`, +`int`, `float64`, `string`, `bytes`, `enum`, `literal`) or required, +non-nullable structs of the same shape, recursively, with at least one leaf. +Nested structs flatten into leaf columns, depth-first in declared order, so a +leaf's participation is exactly its row's; leaf-level `optional`/`nullable` +flags keep their bitmaps. Anything else — arrays or nullable/optional structs +inside the element — sends the whole array down the row encoding from +wire-v0 §4.8/§4.10. Eligibility is a pure function of the IR, so both sides +always agree. ## 2. Layout of an eligible array @@ -32,7 +36,16 @@ the IR, so both sides always agree. - **literal** — zero bytes. - **enum** — `uvarint` index per value. -- **string / bytes** — `uvarint` length + bytes per value (as wire-v0). +- **bytes** — `uvarint` length + bytes per value (as wire-v0). +- **string** — one mode byte, then: + - `0x00` plain: `uvarint` length + strict UTF-8 bytes per value. + - `0x01` packed: `uvarint` byte length per value in row order, then a + `uvarint` blob length and a raw-deflate (RFC 1951) stream of the + concatenated UTF-8 bytes. The inflated size MUST equal the sum of the + declared lengths; each slice MUST be strict UTF-8; per-value and total + lengths obey the decoder byte limits. A decoder without an inflate + capability MUST reject packed columns as unsupported (and the protocol + layer falls back to JSON) rather than guess. - **bool** — one bitmap of `k` bits (padding rules from wire-v0 §3.5). - **int** — one mode byte, then: - `0x00` raw: each value in its wire-v0 form (`uvarint(v - min)` when `min` @@ -61,16 +74,18 @@ the IR, so both sides always agree. division of the same integers. Mantissas outside the v0 integer domain MUST be rejected. -When `k = 0`, int and float columns still emit their mode byte (`0x00`); -other columns emit nothing. +When `k = 0`, int, float, and string columns still emit their mode byte +(`0x00`); other columns emit nothing. ## 4. Encoder mode choice Encoders MUST pick the mode with the smaller encoded size, choosing `0x00` on -ties, so a decode → encode round trip is byte-identical. Decoders accept -either mode regardless of which is smaller — canonicality is an encoder -obligation, checked by the re-encode property, not a decode-time -recomputation. +ties, so a decode → encode round trip is byte-identical. Decoders accept any +valid mode — canonicality is an encoder obligation, checked by the re-encode +property, not a decode-time recomputation. For packed string columns the +obligation is scoped to one implementation: deflate output is not canonical +across libraries, so byte-identical re-encode holds within an +implementation+version, while any spec-valid stream decodes everywhere. ## 5. Rationale (non-normative) diff --git a/spec/vectors/columnar.json b/spec/vectors/columnar.json index 7c25849..20873ab 100644 --- a/spec/vectors/columnar.json +++ b/spec/vectors/columnar.json @@ -155,7 +155,7 @@ "name": "bc" } ], - "hex": "0201000161026263" + "hex": "020100000161026263" }, { "name": "col-empty-array", @@ -203,39 +203,6 @@ ], "hex": "01" }, - { - "name": "col-ineligible-falls-back-to-row", - "ir": { - "kind": "array", - "element": { - "kind": "struct", - "fields": [ - { - "name": "p", - "type": { - "kind": "struct", - "fields": [ - { - "name": "q", - "type": { - "kind": "int" - } - } - ] - } - } - ] - } - }, - "value": [ - { - "p": { - "q": 7 - } - } - ], - "hex": "010e" - }, { "name": "col-literal-column-zero-bytes", "ir": { @@ -380,6 +347,114 @@ } ], "hex": "0202011e4f" + }, + { + "name": "col-string-plain-mode", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "value": [ + { + "s": "hi" + }, + { + "s": "" + } + ], + "hex": "020002686900" + }, + { + "name": "col-nested-struct-flattens", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "a", + "type": { + "kind": "int" + } + }, + { + "name": "p", + "type": { + "kind": "struct", + "fields": [ + { + "name": "q", + "type": { + "kind": "string" + } + }, + { + "name": "r", + "type": { + "kind": "bool" + } + } + ] + } + } + ] + } + }, + "value": [ + { + "a": 1, + "p": { + "q": "x", + "r": true + } + }, + { + "a": 2, + "p": { + "q": "y", + "r": false + } + } + ], + "hex": "02000204000178017901" + }, + { + "name": "col-array-field-falls-back-to-row", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "arr", + "type": { + "kind": "array", + "element": { + "kind": "int" + } + } + } + ] + } + }, + "value": [ + { + "arr": [ + 1 + ] + } + ], + "hex": "010102" } ], "invalidDecode": [ @@ -574,6 +649,44 @@ }, "hex": "010300ffffffffffffff7f", "error": "range" + }, + { + "name": "col-string-bad-mode", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "hex": "0102", + "error": "marker" + }, + { + "name": "col-string-packed-length-mismatch", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "hex": "0201020208010300fcff686979", + "error": "packed" } ], "invalidEncode": [ @@ -648,5 +761,34 @@ ], "error": "type" } + ], + "packedDecode": [ + { + "name": "col-string-packed-stored-block", + "description": "decode-only: blob is a deflate-raw stored block, valid for any inflater", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "value": [ + { + "s": "hi" + }, + { + "s": "yo" + } + ], + "hex": "0201020209010400fbff6869796f" + } ] } diff --git a/spec/vectors/fingerprints.json b/spec/vectors/fingerprints.json index 539a34c..78d54b8 100644 --- a/spec/vectors/fingerprints.json +++ b/spec/vectors/fingerprints.json @@ -125,8 +125,8 @@ "ir": { "kind": "bool" }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":1},\"ir\":{\"kind\":\"bool\"}}", - "fingerprint": "0e65773a352ea24e70d440c4d2e550a7" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":2},\"ir\":{\"kind\":\"bool\"}}", + "fingerprint": "96f5b1aed1b1d4a4dee316623c08acc7" }, { "name": "bounded-int@columnar", @@ -136,8 +136,8 @@ "min": 0, "max": 100 }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":1},\"ir\":{\"kind\":\"int\",\"min\":0,\"max\":100}}", - "fingerprint": "73a409953164507704018d5d2433c2bf" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":2},\"ir\":{\"kind\":\"int\",\"min\":0,\"max\":100}}", + "fingerprint": "6be791f84b580ee7dd78db739b143595" }, { "name": "enum@columnar", @@ -151,8 +151,8 @@ "1d" ] }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":1},\"ir\":{\"kind\":\"enum\",\"members\":[\"1m\",\"5m\",\"1h\",\"1d\"]}}", - "fingerprint": "f0d4a1f72dfda483ea06f3fd4b5e3770" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":2},\"ir\":{\"kind\":\"enum\",\"members\":[\"1m\",\"5m\",\"1h\",\"1d\"]}}", + "fingerprint": "31ac340d52c50c2d65e23c1d031fede4" }, { "name": "candles-response@columnar", @@ -224,8 +224,8 @@ } ] }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":1},\"ir\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"route\",\"type\":{\"kind\":\"literal\",\"value\":\"candles\"}},{\"name\":\"candles\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"t\",\"type\":{\"kind\":\"int\",\"min\":0}},{\"name\":\"o\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"h\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"l\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"c\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"v\",\"type\":{\"kind\":\"float64\"}}]}}},{\"name\":\"cursor\",\"type\":{\"kind\":\"string\"},\"optional\":true}]}}", - "fingerprint": "3b6cffb9377bcaacc89704dd9c4fce0b" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":2},\"ir\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"route\",\"type\":{\"kind\":\"literal\",\"value\":\"candles\"}},{\"name\":\"candles\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"t\",\"type\":{\"kind\":\"int\",\"min\":0}},{\"name\":\"o\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"h\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"l\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"c\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"v\",\"type\":{\"kind\":\"float64\"}}]}}},{\"name\":\"cursor\",\"type\":{\"kind\":\"string\"},\"optional\":true}]}}", + "fingerprint": "b7c09d113bd1134bdf0f0910d86996db" }, { "name": "escaping@columnar", @@ -234,8 +234,8 @@ "kind": "literal", "value": "a\"b\\c" }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":1},\"ir\":{\"kind\":\"literal\",\"value\":\"a\\\"b\\\\c\"}}", - "fingerprint": "ddd3cd318f6e0629b903629fa1603893" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":2},\"ir\":{\"kind\":\"literal\",\"value\":\"a\\\"b\\\\c\"}}", + "fingerprint": "94f508736ee8b0eee4562a5b3d4eda07" } ] }