diff --git a/apps/bench/src/profiles.ts b/apps/bench/src/profiles.ts new file mode 100644 index 0000000..12ffa79 --- /dev/null +++ b/apps/bench/src/profiles.ts @@ -0,0 +1,65 @@ +import { brotliCompressSync, constants } from "node:zlib"; +import { compile, toIR } from "hyperfly/zod"; +import { train } from "hyperfly"; +import { DeviceResponse } from "./corpora/devices.js"; +import { FeedResponse } from "./corpora/feed.js"; +import { devicesTraffic, feedTraffic } from "./traffic.js"; + +const enc = new TextEncoder(); +const br = (bytes: Uint8Array, mode: number) => + brotliCompressSync(bytes, { + params: { + [constants.BROTLI_PARAM_QUALITY]: 4, + [constants.BROTLI_PARAM_MODE]: mode, + [constants.BROTLI_PARAM_SIZE_HINT]: bytes.length, + }, + }).length; + +/** + * Profiles are trained on one slice of sampled traffic and measured on a held-out + * slice, so what is reported is generalization rather than memorization. + */ +export function runProfileSuite(): void { + const suites = [ + { name: "devices", schema: DeviceResponse, traffic: devicesTraffic(60, 50, 0xd7) }, + { name: "feed", schema: FeedResponse, traffic: feedTraffic(60, 12, 0xf7) }, + ] as const; + + console.log("\nprofiles — trained on 80% of sampled responses, measured on the held-out 20%"); + console.log(" corpus json json+br4 columnar col+br4 profiled prof+br4 dict"); + + for (const suite of suites) { + const ir = toIR(suite.schema as never); + const profile = train(ir, suite.traffic.train); + const columnar = compile(suite.schema as never, { plan: "columnar" }); + const profiled = compile(suite.schema as never, { plan: "columnar", profile }); + + let json = 0; + let jsonBr = 0; + let col = 0; + let colBr = 0; + let prof = 0; + let profBr = 0; + + for (const response of suite.traffic.holdout) { + const j = enc.encode(JSON.stringify(response)); + json += j.length; + jsonBr += br(j, constants.BROTLI_MODE_TEXT); + const c = columnar.encode(response as never); + col += c.length; + colBr += br(c, constants.BROTLI_MODE_GENERIC); + const p = profiled.encode(response as never); + prof += p.length; + profBr += br(p, constants.BROTLI_MODE_GENERIC); + if (!Bun.deepEquals(profiled.decode(p), response, false)) { + throw new Error(`${suite.name}: profiled round-trip mismatch`); + } + } + + const entries = profile?.shared.columns.reduce((n, c) => n + c.dict.length, 0) ?? 0; + const cell = (v: number) => `${String(v).padStart(7)} `; + console.log( + ` ${suite.name.padEnd(9)} ${cell(json)} ${cell(jsonBr)} ${cell(col)} ${cell(colBr)} ${cell(prof)} ${cell(profBr)} ${entries}`, + ); + } +} diff --git a/apps/bench/src/run.ts b/apps/bench/src/run.ts index a572026..bc4a311 100644 --- a/apps/bench/src/run.ts +++ b/apps/bench/src/run.ts @@ -2,6 +2,7 @@ import { brotliCompressSync, brotliDecompressSync, constants, gzipSync, gunzipSy import { decode as cborDecode, encode as cborEncode } from "cbor-x"; import { compile } from "hyperfly/zod"; import { pack, unpack } from "msgpackr"; +import { runProfileSuite } from "./profiles.js"; import { candlesProto, devicesProto, feedProto, type ProtoCodec } from "./proto.js"; import { CandleResponse, candlesPayload } from "./corpora/candles.js"; import { DeviceResponse, devicesPayload } from "./corpora/devices.js"; @@ -207,6 +208,8 @@ for (const suite of SUITES) { all.push(...runCorpus(suite.name, suite.schema, suite.payload, suite.proto, suite.warmup, suite.samples)); } +runProfileSuite(); + await Bun.write( new URL("../results/results.json", import.meta.url).pathname, JSON.stringify( diff --git a/apps/bench/src/traffic.ts b/apps/bench/src/traffic.ts new file mode 100644 index 0000000..6e1bbc6 --- /dev/null +++ b/apps/bench/src/traffic.ts @@ -0,0 +1,111 @@ +import { intIn, mulberry32, pick } from "./prng.js"; +import { DeviceResponse } from "./corpora/devices.js"; +import { FeedResponse } from "./corpora/feed.js"; +import type { z } from "zod"; + +/** + * Sampled traffic for one route: many responses drawn from a stable universe, the + * way a real endpoint behaves. Profiles are trained on one slice and measured on a + * held-out slice, so the numbers reflect generalization rather than memorization. + */ +export interface Traffic { + train: T[]; + holdout: T[]; +} + +const REGIONS = ["us-east", "us-west", "eu-central", "eu-west", "ap-south", "ap-northeast", "sa-east", "af-south"] as const; +const STATUSES = ["online", "online", "online", "online", "offline", "degraded", "provisioning", "unknown"] as const; +const TAGS = ["fleet-a", "fleet-b", "fleet-a", "pilot", "lab", null, null, null]; + +/** A fixed device fleet: the same ids and tags recur across every response. */ +export function devicesTraffic( + responses: number, + perResponse: number, + seed: number, +): Traffic> { + const rng = mulberry32(seed); + const fleet = Array.from({ length: 400 }, (_, i) => ({ + id: `dev-${String(intIn(rng, 0, 99)).padStart(2, "0")}-${String(i).padStart(5, "0")}`, + region: pick(rng, REGIONS), + tag: pick(rng, TAGS), + firmwareMajor: intIn(rng, 1, 4), + firmwareMinor: intIn(rng, 0, 27), + })); + + const base = 1754000000000; + const all = Array.from({ length: responses }, (_, r) => ({ + route: "devices" as const, + page: 0, + devices: Array.from({ length: perResponse }, () => { + const unit = fleet[intIn(rng, 0, fleet.length - 1)]!; + return { + ...unit, + status: pick(rng, STATUSES), + battery: intIn(rng, 0, 100), + rssi: intIn(rng, -120, 0), + uptimeSec: intIn(rng, 0, 40000000), + tempC: Math.round((15 + rng() * 45) * 10) / 10, + alarms: rng() < 0.85 ? 0 : intIn(rng, 1, 12), + shadowSynced: rng() < 0.93, + lastSeen: base + r * 60000 - intIn(rng, 0, 86400000), + }; + }), + })); + + const split = Math.floor(all.length * 0.8); + return { train: all.slice(0, split), holdout: all.slice(split) }; +} + +const HANDLES = Array.from({ length: 120 }, (_, i) => { + const first = ["ada", "linus", "grace", "alan", "edsger", "barbara", "donald", "radia", "ken", "margaret"][i % 10]!; + const last = ["hopper", "torvalds", "lovelace", "turing", "dijkstra", "liskov", "knuth", "perlman", "thompson", "hamilton"][ + Math.floor(i / 10) % 10 + ]!; + return { handle: `@${first}${last}${i}`, name: `${first[0]!.toUpperCase()}${first.slice(1)} ${last[0]!.toUpperCase()}${last.slice(1)}` }; +}); + +const LEXICON = ( + "the of and to in is that it was for on are as with his they at be this have from or had by hot word " + + "but what some we can out other were all there when up use your how said an each she which do their time " + + "deploy latency cluster rollout incident postmortem throughput regression release migration schema payload" +).split(" "); + +/** A recurring cast of authors posting about a recurring vocabulary. */ +export function feedTraffic( + responses: number, + perResponse: number, + seed: number, +): Traffic> { + const rng = mulberry32(seed); + const base = 1754500000000; + const hex = (n: number) => Array.from({ length: n }, () => Math.floor(rng() * 16).toString(16)).join(""); + const authors = HANDLES.map((h) => ({ ...h, id: hex(12), verified: rng() < 0.2 })); + + const all = Array.from({ length: responses }, () => ({ + route: "feed" as const, + posts: Array.from({ length: perResponse }, () => { + const author = authors[intIn(rng, 0, authors.length - 1)]!; + const sentences = Array.from({ length: intIn(rng, 1, 3) }, () => { + const words = Array.from({ length: intIn(rng, 8, 24) }, () => pick(rng, LEXICON)); + const s = words.join(" "); + return s.charAt(0).toUpperCase() + s.slice(1) + "."; + }); + return { + id: hex(16), + author, + body: sentences.join(" "), + lang: pick(rng, ["en", "en", "en", "de", "fr", "es", "ja"] as const), + likes: intIn(rng, 0, 50000), + replies: intIn(rng, 0, 2000), + reposts: intIn(rng, 0, 8000), + createdAt: base - intIn(rng, 0, 604800000), + inReplyTo: null, + }; + }), + })); + + const split = Math.floor(all.length * 0.8); + return { train: all.slice(0, split), holdout: all.slice(split) }; +} + +export { DeviceResponse, FeedResponse }; diff --git a/packages/hyperfly/src/canonical.ts b/packages/hyperfly/src/canonical.ts index 07e2147..21e1e0e 100644 --- a/packages/hyperfly/src/canonical.ts +++ b/packages/hyperfly/src/canonical.ts @@ -1,4 +1,5 @@ import type { IRField, IRNode, LiteralValue } from "./ir.js"; +import type { SharedProfile } from "./profile.js"; import { sha256 } from "./sha256.js"; /** Spec §5: not generic JSON canonicalization — key order and escaping are fixed here. */ @@ -59,10 +60,22 @@ export function serializeNode(node: IRNode): string { export type PlanLayout = "row" | "columnar"; -const PLAN_VERSION: Record = { row: 1, columnar: 2 }; +const PLAN_VERSION: Record = { row: 1, columnar: 3 }; -export function serializeArtifact(ir: IRNode, layout: PlanLayout = "row"): string { - return `{"wire":1,"plan":{"layout":"${layout}","version":${PLAN_VERSION[layout]}},"ir":${serializeNode(ir)}}`; +export function serializeShared(shared: SharedProfile): string { + const columns = shared.columns.map( + (c) => `{"leaf":${c.leaf},"dict":[${c.dict.map(escapeString).join(",")}]}`, + ); + return `{"columns":[${columns.join(",")}]}`; +} + +export function serializeArtifact( + ir: IRNode, + layout: PlanLayout = "row", + profile?: { shared: SharedProfile }, +): string { + const head = `{"wire":1,"plan":{"layout":"${layout}","version":${PLAN_VERSION[layout]}},"ir":${serializeNode(ir)}`; + return profile ? `${head},"profile":${serializeShared(profile.shared)}}` : `${head}}`; } export function fingerprintOf(artifact: string): Uint8Array { diff --git a/packages/hyperfly/src/codec.ts b/packages/hyperfly/src/codec.ts index a78d477..4d7278c 100644 --- a/packages/hyperfly/src/codec.ts +++ b/packages/hyperfly/src/codec.ts @@ -1,8 +1,9 @@ import { fingerprintOf, serializeArtifact, toHex, type PlanLayout } from "./canonical.js"; import { defaultPackHooks } from "./pack.js"; +import { indexProfile, validateProfile, type Profile } from "./profile.js"; import { decodeNode } from "./decode.js"; import { encodeNode } from "./encode.js"; -import { DecodeError, FingerprintMismatchError } from "./errors.js"; +import { DecodeError, FingerprintMismatchError, HyperflyError } from "./errors.js"; import { validateIR, type IRNode } from "./ir.js"; import { DEFAULT_LIMITS, Reader, type DecodeLimits } from "./reader.js"; import { Writer } from "./writer.js"; @@ -20,6 +21,7 @@ export interface CompileOptions { limits?: Partial; plan?: PlanLayout; pack?: PackHooks | false; + profile?: Profile; } export interface Codec { @@ -27,6 +29,7 @@ export interface Codec { readonly artifact: string; readonly fingerprint: string; readonly plan: PlanLayout; + readonly profile?: Profile; encode(value: T): Uint8Array; decode(bytes: Uint8Array): T; encodeBody(value: T): Uint8Array; @@ -47,7 +50,17 @@ export function compileIR(ir: IRNode, options: CompileOptions = {}) // fixed at compile time and the schema behind it must not drift ir = deepFreeze(structuredClone(ir)); const plan: PlanLayout = options.plan ?? "row"; - const artifact = serializeArtifact(ir, plan); + const profile = options.profile; + if (profile) { + if (plan !== "columnar") { + throw new HyperflyError("ir", "profiles apply to the columnar plan only"); + } + validateProfile(ir, profile); + } + // the profile is fixed by the fingerprint exactly as the IR is + const frozenProfile = profile ? deepFreeze(structuredClone(profile)) : undefined; + const profileIndex = indexProfile(frozenProfile); + const artifact = serializeArtifact(ir, plan, frozenProfile); const fingerprintBytes = fingerprintOf(artifact); const fingerprint = toHex(fingerprintBytes); const limits: DecodeLimits = { ...DEFAULT_LIMITS, ...options.limits }; @@ -63,13 +76,14 @@ export function compileIR(ir: IRNode, options: CompileOptions = {}) columnar, deflate: pack.deflate, canInflate: pack.inflate !== undefined, + profile: profileIndex, }); return w.finish(); }; const decodeBody = (bytes: Uint8Array): T => { const r = new Reader(bytes, limits); - const value = decodeNode(r, ir, "$", 0, columnar, pack.inflate); + const value = decodeNode(r, ir, "$", 0, columnar, pack.inflate, profileIndex); r.expectEnd(); return value as T; }; @@ -79,6 +93,7 @@ export function compileIR(ir: IRNode, options: CompileOptions = {}) artifact, fingerprint, plan, + profile: frozenProfile, encodeBody, decodeBody, encode(value: T): Uint8Array { diff --git a/packages/hyperfly/src/columnar.ts b/packages/hyperfly/src/columnar.ts index 57c3177..1169fae 100644 --- a/packages/hyperfly/src/columnar.ts +++ b/packages/hyperfly/src/columnar.ts @@ -2,6 +2,7 @@ import { boundByInput, decodeNode, readBitmap, type Inflate } from "./decode.js" import { encodeNode, typeAcceptsNull, utf8Bytes, writeBitmap, type EncodeCtx } from "./encode.js"; import { DecodeError, EncodeError } from "./errors.js"; import type { IRField, IRNode } from "./ir.js"; +import type { ProfileIndex } from "./profile.js"; import type { Reader } from "./reader.js"; import { INT_MAX, INT_MIN, readUleb, ulebLen, unzigzag, writeUleb, zigzag } from "./varint.js"; import type { Writer } from "./writer.js"; @@ -12,7 +13,7 @@ type IntNode = Extract; const COLUMN_KINDS = new Set(["bool", "int", "float64", "string", "bytes", "enum", "literal"]); -interface Leaf { +export interface Leaf { segs: readonly string[]; field: IRField; } @@ -296,7 +297,13 @@ function decodeFloatColumn(r: Reader, count: number, path: string): number[] { return out; } -function encodeStringColumn(w: Writer, values: unknown[], path: string, ctx: EncodeCtx): void { +function encodeStringColumn( + w: Writer, + values: unknown[], + path: string, + ctx: EncodeCtx, + ordinal: number, +): void { if (values.length === 0) { w.u8(0); return; @@ -304,6 +311,19 @@ function encodeStringColumn(w: Writer, values: unknown[], path: string, ctx: Enc const bytes = values.map((v, i) => utf8Bytes(v, `${path}[${i}]`)); const plainCost = bytes.reduce((n, b) => n + ulebLen(BigInt(b.length)) + b.length, 0); + // dictionary: one byte per hit, escape + plain encoding per miss + const dict = ctx.profile.dictOf(ordinal); + let dictCost = Infinity; + let codes: number[] | null = null; + if (dict) { + codes = values.map((v) => ctx.profile.codeOf(ordinal, v as string) ?? 0); + dictCost = 0; + for (let i = 0; i < codes.length; i++) { + dictCost += ulebLen(BigInt(codes[i]!)); + if (codes[i] === 0) dictCost += ulebLen(BigInt(bytes[i]!.length)) + bytes[i]!.length; + } + } + let packed: Uint8Array | null = null; let packedCost = Infinity; if (ctx.deflate && ctx.canInflate) { @@ -321,25 +341,46 @@ function encodeStringColumn(w: Writer, values: unknown[], path: string, ctx: Enc packed.length; } - if (packed && packedCost < plainCost) { - w.u8(1); - for (const b of bytes) writeUleb(w, BigInt(b.length)); - writeUleb(w, BigInt(packed.length)); - w.bytes(packed); + // smallest wins; ties go to the lowest flags byte, which is also the most + // reproducible encoding (plain, then dictionary, then library-dependent deflate) + const best = Math.min(plainCost, dictCost, packed ? packedCost : Infinity); + if (best === plainCost) { + w.u8(0x00); + for (const b of bytes) { + writeUleb(w, BigInt(b.length)); + w.bytes(b); + } return; } - w.u8(0); - for (const b of bytes) { - writeUleb(w, BigInt(b.length)); - w.bytes(b); + if (best === dictCost && codes) { + w.u8(0x01); + for (let i = 0; i < codes.length; i++) { + writeUleb(w, BigInt(codes[i]!)); + if (codes[i] === 0) { + writeUleb(w, BigInt(bytes[i]!.length)); + w.bytes(bytes[i]!); + } + } + return; } + w.u8(0x02); + for (const b of bytes) writeUleb(w, BigInt(b.length)); + writeUleb(w, BigInt(packed!.length)); + w.bytes(packed!); } const utf8Strict = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); -function decodeStringColumn(r: Reader, count: number, path: string, inflate?: Inflate): string[] { +function decodeStringColumn( + r: Reader, + count: number, + path: string, + inflate: Inflate | undefined, + profile: ProfileIndex, + ordinal: number, +): string[] { const mode = r.u8(); - if (mode > 1) throw new DecodeError("marker", `${path}: invalid string column mode 0x${mode.toString(16)}`); + if (mode > 2) throw new DecodeError("marker", `${path}: invalid string column flags 0x${mode.toString(16)}`); if (count === 0 && mode !== 0) throw new DecodeError("marker", `${path}: empty column must use mode 0x00`); const out: string[] = new Array(count); if (count === 0) return out; @@ -363,6 +404,29 @@ function decodeStringColumn(r: Reader, count: number, path: string, inflate?: In return out; } + if (mode === 0x01) { + const dict = profile.dictOf(ordinal); + if (!dict) { + throw new DecodeError("unsupported", `${path}: dictionary column requires a profile for this leaf`); + } + for (let i = 0; i < count; i++) { + const code = readUleb(r); + if (code === 0n) { + const len = readUleb(r); + if (len > BigInt(r.limits.maxByteLength)) { + throw new DecodeError("limit", `${path}[${i}]: string length exceeds limit`); + } + out[i] = decodeSlice(r.bytes(Number(len)), i); + } else { + if (code > BigInt(dict.length)) { + throw new DecodeError("range", `${path}[${i}]: dictionary code ${code} out of range`); + } + out[i] = dict[Number(code) - 1]!; + } + } + return out; + } + const lengths: number[] = new Array(count); let total = 0; for (let i = 0; i < count; i++) { @@ -421,6 +485,7 @@ export function encodeColumnarArray( path: string, depth: number, ctx: EncodeCtx, + ordinalBase: number, ): void { if (!Array.isArray(value)) throw new EncodeError("type", `${path}: expected array`); const element = node.element as StructNode; @@ -457,7 +522,7 @@ export function encodeColumnarArray( return obj; }; - for (const leaf of leaves) { + for (const [leafIndex, leaf] of leaves.entries()) { const field = leaf.field; const leafName = leaf.segs[leaf.segs.length - 1]!; const dotted = leaf.segs.join("."); @@ -512,11 +577,11 @@ export function encodeColumnarArray( encodeBoolColumn(w, participating, fieldPath); break; case "string": - encodeStringColumn(w, participating, fieldPath, ctx); + encodeStringColumn(w, participating, fieldPath, ctx, ordinalBase + leafIndex); break; default: for (let i = 0; i < participating.length; i++) { - encodeNode(w, field.type, participating[i], `${fieldPath}[${i}]`, depth + 2, ctx); + encodeNode(w, field.type, participating[i], `${fieldPath}[${i}]`, depth + 2, ctx, ordinalBase + leafIndex); } } } @@ -527,7 +592,9 @@ export function decodeColumnarArray( node: ArrayNode, path: string, depth: number, - inflate?: Inflate, + inflate: Inflate | undefined, + profile: ProfileIndex, + ordinalBase: number, ): Record[] { const element = node.element as StructNode; let count: number; @@ -558,7 +625,7 @@ export function decodeColumnarArray( return obj; }; - for (const leaf of leaves) { + for (const [leafIndex, leaf] of leaves.entries()) { const field = leaf.field; const leafName = leaf.segs[leaf.segs.length - 1]!; const fieldPath = `${path}[].${leaf.segs.join(".")}`; @@ -610,13 +677,13 @@ export function decodeColumnarArray( break; } case "string": { - const values = decodeStringColumn(r, slots.length, fieldPath, inflate); + const values = decodeStringColumn(r, slots.length, fieldPath, inflate, profile, ordinalBase + leafIndex); slots.forEach((row, j) => (containerOf(out[row]!, leaf.segs)[leafName] = values[j]!)); break; } default: { for (const row of slots) { - containerOf(out[row]!, leaf.segs)[leafName] = decodeNode(r, field.type, `${path}[${row}].${leafName}`, depth + 2, false); + containerOf(out[row]!, leaf.segs)[leafName] = decodeNode(r, field.type, `${path}[${row}].${leafName}`, depth + 2, false, inflate, profile, ordinalBase + leafIndex); } } } diff --git a/packages/hyperfly/src/decode.ts b/packages/hyperfly/src/decode.ts index c07220c..6589547 100644 --- a/packages/hyperfly/src/decode.ts +++ b/packages/hyperfly/src/decode.ts @@ -1,6 +1,7 @@ import { columnarEligible, decodeColumnarArray } from "./columnar.js"; import { DecodeError } from "./errors.js"; import { hasPayload, type IRNode } from "./ir.js"; +import { columnCount, type ProfileIndex } from "./profile.js"; import type { Reader } from "./reader.js"; import { INT_MAX, INT_MIN, readUleb, unzigzag } from "./varint.js"; @@ -64,7 +65,9 @@ export function decodeNode( path: string, depth: number, columnar: boolean, - inflate?: Inflate, + inflate: Inflate | undefined, + profile: ProfileIndex, + column = 0, ): unknown { if (depth > r.limits.maxDepth) fail("depth", path, `nesting deeper than ${r.limits.maxDepth}`); @@ -109,11 +112,11 @@ export function decodeNode( const marker = r.u8(); if (marker === 0) return null; if (marker !== 1) fail("marker", path, `invalid nullable marker 0x${marker.toString(16)}`); - return decodeNode(r, node.inner, path, depth + 1, columnar, inflate); + return decodeNode(r, node.inner, path, depth + 1, columnar, inflate, profile, column); } case "array": { if (columnar && columnarEligible(node)) { - return decodeColumnarArray(r, node, path, depth, inflate); + return decodeColumnarArray(r, node, path, depth, inflate, profile, column); } const count = node.length ?? readCount(r, r.limits.maxItems, "array count", path); if (count > r.limits.maxItems) { @@ -121,7 +124,7 @@ export function decodeNode( } boundByInput(r, count, node.element, path); const out = new Array(count); - for (let i = 0; i < count; i++) out[i] = decodeNode(r, node.element, `${path}[${i}]`, depth + 1, columnar, inflate); + for (let i = 0; i < count; i++) out[i] = decodeNode(r, node.element, `${path}[${i}]`, depth + 1, columnar, inflate, profile, column); return out; } case "struct": { @@ -131,10 +134,14 @@ export function decodeNode( const nulls = readBitmap(r, nullableCount, path); let pi = 0; let ni = 0; + let fieldColumn = column; // a field may legitimately be named constructor/toString/valueOf; assigning those on a // prototypeful object would hit inherited accessors instead of creating own properties const out: Record = {}; for (const field of node.fields) { + // columns advance in declared field order, matching the §6.1 walk exactly + const base = fieldColumn; + fieldColumn += columnCount(field.type); const present = field.optional ? presence[pi++]! : true; const isNull = field.nullable ? nulls[ni++]! : false; if (!present) { @@ -145,7 +152,7 @@ export function decodeNode( out[field.name] = null; continue; } - out[field.name] = decodeNode(r, field.type, `${path}.${field.name}`, depth + 1, columnar, inflate); + out[field.name] = decodeNode(r, field.type, `${path}.${field.name}`, depth + 1, columnar, inflate, profile, base); } return out; } diff --git a/packages/hyperfly/src/encode.ts b/packages/hyperfly/src/encode.ts index 376a2fb..2ae1180 100644 --- a/packages/hyperfly/src/encode.ts +++ b/packages/hyperfly/src/encode.ts @@ -1,4 +1,5 @@ import { columnarEligible, encodeColumnarArray } from "./columnar.js"; +import { columnCount, type ProfileIndex } from "./profile.js"; import { EncodeError, type ErrorCode } from "./errors.js"; import type { IRNode } from "./ir.js"; import { INT_MAX, INT_MIN, writeUleb, zigzag } from "./varint.js"; @@ -31,6 +32,7 @@ export interface EncodeCtx { deflate?: (data: Uint8Array) => Uint8Array; /** packing is only canonical when the same codec can also inflate what it wrote */ canInflate: boolean; + profile: ProfileIndex; } export function typeAcceptsNull(node: IRNode): boolean { @@ -64,7 +66,15 @@ export function writeBitmap(w: Writer, bits: boolean[]): void { } } -export function encodeNode(w: Writer, node: IRNode, value: unknown, path: string, depth: number, ctx: EncodeCtx): void { +export function encodeNode( + w: Writer, + node: IRNode, + value: unknown, + path: string, + depth: number, + ctx: EncodeCtx, + column = 0, +): void { if (depth > ctx.maxDepth) fail("depth", path, `nesting deeper than ${ctx.maxDepth}`); switch (node.kind) { @@ -114,12 +124,12 @@ export function encodeNode(w: Writer, node: IRNode, value: unknown, path: string return; } w.u8(1); - encodeNode(w, node.inner, value, path, depth + 1, ctx); + encodeNode(w, node.inner, value, path, depth + 1, ctx, column); return; } case "array": { if (ctx.columnar && columnarEligible(node)) { - encodeColumnarArray(w, node, value, path, depth, ctx); + encodeColumnarArray(w, node, value, path, depth, ctx, column); return; } if (!Array.isArray(value)) fail("type", path, "expected array"); @@ -130,7 +140,7 @@ export function encodeNode(w: Writer, node: IRNode, value: unknown, path: string writeUleb(w, BigInt(value.length)); } for (let i = 0; i < value.length; i++) { - encodeNode(w, node.element, value[i], `${path}[${i}]`, depth + 1, ctx); + encodeNode(w, node.element, value[i], `${path}[${i}]`, depth + 1, ctx, column); } return; } @@ -158,11 +168,15 @@ export function encodeNode(w: Writer, node: IRNode, value: unknown, path: string }); writeBitmap(w, presence); writeBitmap(w, nulls); + // columns advance in declared field order, matching the §6.1 walk exactly + let fieldColumn = column; node.fields.forEach((field, i) => { const v = snapshot[i]; + const base = fieldColumn; + fieldColumn += columnCount(field.type); if (v === undefined) return; if (v === null && field.nullable) return; - encodeNode(w, field.type, v, `${path}.${field.name}`, depth + 1, ctx); + encodeNode(w, field.type, v, `${path}.${field.name}`, depth + 1, ctx, base); }); return; } diff --git a/packages/hyperfly/src/index.ts b/packages/hyperfly/src/index.ts index df129b8..ca825e7 100644 --- a/packages/hyperfly/src/index.ts +++ b/packages/hyperfly/src/index.ts @@ -2,6 +2,15 @@ export { compileIR, HEADER_SIZE, MAGIC, WIRE_VERSION, type Codec, type CompileOp export { defaultPackHooks } from "./pack.js"; export { serializeArtifact, serializeNode, fingerprintOf, toHex, type PlanLayout } from "./canonical.js"; export { columnarEligible } from "./columnar.js"; +export { train, type TrainOptions } from "./train.js"; +export { + enumerateColumns, + validateProfile, + MAX_DICT_ENTRIES, + type Profile, + type ProfileColumn, + type SharedProfile, +} from "./profile.js"; export { validateIR, type IRField, type IRNode, type LiteralValue } from "./ir.js"; export { DEFAULT_LIMITS, type DecodeLimits } from "./reader.js"; export { INT_MAX, INT_MIN } from "./varint.js"; diff --git a/packages/hyperfly/src/ir.ts b/packages/hyperfly/src/ir.ts index f8df602..956d47d 100644 --- a/packages/hyperfly/src/ir.ts +++ b/packages/hyperfly/src/ir.ts @@ -30,7 +30,7 @@ function isSafeInt(v: unknown): v is number { return typeof v === "number" && Number.isSafeInteger(v); } -function hasLoneSurrogate(s: string): boolean { +export function hasLoneSurrogate(s: string): boolean { for (let i = 0; i < s.length; i++) { const c = s.charCodeAt(i); if (c >= 0xd800 && c <= 0xdbff) { diff --git a/packages/hyperfly/src/profile.ts b/packages/hyperfly/src/profile.ts new file mode 100644 index 0000000..2917295 --- /dev/null +++ b/packages/hyperfly/src/profile.ts @@ -0,0 +1,139 @@ +import { columnarEligible, flattenLeaves } from "./columnar.js"; +import { HyperflyError } from "./errors.js"; +import { hasLoneSurrogate } from "./ir.js"; +import type { IRNode } from "./ir.js"; + +export interface ProfileColumn { + leaf: number; + dict: readonly string[]; +} + +export interface SharedProfile { + columns: readonly ProfileColumn[]; +} + +export interface Profile { + version: 1; + shared: SharedProfile; + /** Advisory encoder guidance; never part of the artifact or the fingerprint. */ + hints?: Record; +} + +export const MAX_DICT_ENTRIES = 16383; + +export interface ColumnRef { + ordinal: number; + kind: IRNode["kind"]; +} + +/** + * Spec §6.1: one total enumeration of every columnar leaf in the schema. + * Ordinals, not textual paths — field names may contain dots and brackets, so a + * dotted path can bind two different leaves to the same key. + */ +export function enumerateColumns(ir: IRNode): ColumnRef[] { + const out: ColumnRef[] = []; + const walk = (node: IRNode): void => { + switch (node.kind) { + case "array": { + if (columnarEligible(node)) { + const leaves = flattenLeaves(node.element as Extract); + if (leaves) { + for (const leaf of leaves) out.push({ ordinal: out.length, kind: leaf.field.type.kind }); + return; + } + } + walk(node.element); + return; + } + case "nullable": + walk(node.inner); + return; + case "struct": + for (const f of node.fields) walk(f.type); + return; + default: + return; + } + }; + walk(ir); + return out; +} + +/** + * Columnar leaves under this node. A pure function of the subtree, so two schema + * positions sharing one node object still count the same — which is why column + * bases are threaded positionally rather than looked up by node identity. + */ +export function columnCount(node: IRNode): number { + switch (node.kind) { + case "array": { + if (columnarEligible(node)) { + const leaves = flattenLeaves(node.element as Extract); + if (leaves) return leaves.length; + } + return columnCount(node.element); + } + case "nullable": + return columnCount(node.inner); + case "struct": + return node.fields.reduce((n, f) => n + columnCount(f.type), 0); + default: + return 0; + } +} + +export function validateProfile(ir: IRNode, profile: Profile): void { + const fail = (message: string): never => { + throw new HyperflyError("ir", `profile: ${message}`); + }; + + if (profile.version !== 1) fail(`unsupported profile version ${profile.version}`); + const columns = enumerateColumns(ir); + let previous = -1; + + for (const column of profile.shared.columns) { + if (!Number.isSafeInteger(column.leaf) || column.leaf < 0 || column.leaf >= columns.length) { + fail(`leaf ${column.leaf} is not a column in this schema`); + } + if (column.leaf <= previous) fail("columns must be sorted by ascending leaf and unique"); + previous = column.leaf; + if (columns[column.leaf]!.kind !== "string") fail(`leaf ${column.leaf} is not a string column`); + if (column.dict.length === 0 || column.dict.length > MAX_DICT_ENTRIES) { + fail(`leaf ${column.leaf}: a dictionary holds 1 to ${MAX_DICT_ENTRIES} entries`); + } + const seen = new Set(); + for (const entry of column.dict) { + if (typeof entry !== "string") fail(`leaf ${column.leaf}: entries must be strings`); + if (hasLoneSurrogate(entry)) { + fail(`leaf ${column.leaf}: entry contains a lone surrogate and has no portable encoding`); + } + if (seen.has(entry)) fail(`leaf ${column.leaf}: duplicate entry gives one value two codes`); + seen.add(entry); + } + } +} + +/** Maps a leaf ordinal to its dictionary, plus the reverse index used at encode. */ +export interface ProfileIndex { + dictOf(ordinal: number): readonly string[] | undefined; + codeOf(ordinal: number, value: string): number | undefined; +} + +export function indexProfile(profile: Profile | undefined): ProfileIndex { + if (!profile) { + return { dictOf: () => undefined, codeOf: () => undefined }; + } + const dicts = new Map(); + const codes = new Map>(); + for (const column of profile.shared.columns) { + dicts.set(column.leaf, column.dict); + const lookup = new Map(); + column.dict.forEach((entry, i) => lookup.set(entry, i + 1)); + codes.set(column.leaf, lookup); + } + return { + dictOf: (ordinal) => dicts.get(ordinal), + codeOf: (ordinal, value) => codes.get(ordinal)?.get(value), + }; +} diff --git a/packages/hyperfly/src/train.ts b/packages/hyperfly/src/train.ts new file mode 100644 index 0000000..db9b597 --- /dev/null +++ b/packages/hyperfly/src/train.ts @@ -0,0 +1,123 @@ +import { MAX_DICT_ENTRIES, columnCount, enumerateColumns, type Profile, type ProfileColumn } from "./profile.js"; +import { columnarEligible, flattenLeaves } from "./columnar.js"; +import { hasLoneSurrogate, type IRNode } from "./ir.js"; + +export interface TrainOptions { + /** A value must appear at least this often across the samples to be considered. */ + minOccurrences?: number; + maxEntries?: number; +} + +const encoder = new TextEncoder(); + +function ulebLen(value: number): number { + let n = 1; + let v = value; + while (v > 0x7f) { + v >>= 7; + n++; + } + return n; +} + +/** UTF-8 byte order — JS string comparison is UTF-16 code-unit order and disagrees above the BMP. */ +function compareUtf8(a: string, b: string): number { + const x = encoder.encode(a); + const y = encoder.encode(b); + const n = Math.min(x.length, y.length); + for (let i = 0; i < n; i++) { + if (x[i] !== y[i]) return x[i]! - y[i]!; + } + return x.length - y.length; +} + +function collect(node: IRNode, value: unknown, base: number, counts: Map>): void { + if (value === undefined || value === null) return; + switch (node.kind) { + case "array": { + if (!Array.isArray(value)) return; + if (columnarEligible(node)) { + const leaves = flattenLeaves(node.element as Extract); + if (!leaves) return; + leaves.forEach((leaf, i) => { + if (leaf.field.type.kind !== "string") return; + const ordinal = base + i; + let bucket = counts.get(ordinal); + if (!bucket) counts.set(ordinal, (bucket = new Map())); + for (const row of value) { + let holder: unknown = row; + for (const seg of leaf.segs.slice(0, -1)) { + if (typeof holder !== "object" || holder === null) return; + holder = (holder as Record)[seg]; + } + if (typeof holder !== "object" || holder === null) continue; + const v = (holder as Record)[leaf.segs[leaf.segs.length - 1]!]; + if (typeof v === "string") bucket.set(v, (bucket.get(v) ?? 0) + 1); + } + }); + return; + } + for (const item of value) collect(node.element, item, base, counts); + return; + } + case "nullable": + collect(node.inner, value, base, counts); + return; + case "struct": { + if (typeof value !== "object" || value === null) return; + let fieldColumn = base; + for (const f of node.fields) { + const fieldBase = fieldColumn; + fieldColumn += columnCount(f.type); + collect(f.type, (value as Record)[f.name], fieldBase, counts); + } + return; + } + default: + return; + } +} + +/** + * Reference trainer. Explicitly non-normative (spec §6.5): any document meeting + * §6.2 is a valid profile, and the artifact pins the exact bytes, so + * implementations need not agree on how one is produced. + * + * A dictionary hit costs its code and saves `len(uvarint) + len(utf8)`; a miss + * costs the escape byte plus the plain encoding. Entries are ordered by + * frequency so code length depends only on position, which keeps the objective + * linear instead of self-referential, and are capped at MAX_DICT_ENTRIES. + */ +export function train(ir: IRNode, samples: readonly unknown[], options: TrainOptions = {}): Profile | undefined { + const minOccurrences = options.minOccurrences ?? 2; + const maxEntries = Math.min(options.maxEntries ?? MAX_DICT_ENTRIES, MAX_DICT_ENTRIES); + const counts = new Map>(); + for (const sample of samples) collect(ir, sample, 0, counts); + + const kinds = enumerateColumns(ir); + const columns: ProfileColumn[] = []; + + for (const [ordinal, bucket] of [...counts.entries()].sort((a, b) => a[0] - b[0])) { + if (kinds[ordinal]?.kind !== "string") continue; + // Frequency order first, so the shortest codes land on the most frequent values. + // Code length then depends only on position, which keeps the objective linear + // instead of self-referential, and lets a trailing entry be dropped safely. + const ranked = [...bucket.entries()] + .filter(([value, n]) => n >= minOccurrences && !hasLoneSurrogate(value)) + .sort((a, b) => b[1] - a[1] || compareUtf8(a[0], b[0])) + .slice(0, maxEntries); + + const kept: string[] = []; + for (const [value, n] of ranked) { + const bytes = encoder.encode(value).length; + const plain = bytes + ulebLen(bytes); + const code = ulebLen(kept.length + 1); + if (n * (plain - code) <= 0) break; + kept.push(value); + } + + if (kept.length > 0) columns.push({ leaf: ordinal, dict: kept }); + } + + return columns.length > 0 ? { version: 1, shared: { columns } } : undefined; +} diff --git a/packages/hyperfly/test/columnar.test.ts b/packages/hyperfly/test/columnar.test.ts index ce49858..142b063 100644 --- a/packages/hyperfly/test/columnar.test.ts +++ b/packages/hyperfly/test/columnar.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import vectors from "../../../spec/vectors/columnar.json" with { type: "json" }; import { z } from "zod"; -import { compileIR, FingerprintMismatchError, toHex, type IRNode } from "../src/index.js"; +import { compileIR, FingerprintMismatchError, toHex, train, type IRNode } from "../src/index.js"; import { HyperflyError } from "../src/errors.js"; import { compile } from "../src/zod.js"; @@ -259,3 +259,134 @@ describe("empty column canonicality", () => { } }); }); + +describe("profiles: dictionary columns", () => { + const IR: IRNode = { + kind: "array", + element: { kind: "struct", fields: [{ name: "s", type: { kind: "string" } }] }, + }; + const profile = { + version: 1 as const, + shared: { columns: [{ leaf: 0, dict: ["online", "offline"] }] }, + }; + + test("hits become codes, misses escape, and it round-trips", () => { + const codec = compileIR(IR, { plan: "columnar", profile, pack: false }); + const value = [{ s: "online" }, { s: "novel" }, { s: "offline" }]; + const body = codec.encodeBody(value); + expect(toHex(body)).toBe("03010100056e6f76656c02"); + expect(codec.decodeBody(body)).toEqual(value); + }); + + test("a profile changes the fingerprint and the artifact", () => { + const bare = compileIR(IR, { plan: "columnar" }); + const withProfile = compileIR(IR, { plan: "columnar", profile }); + expect(withProfile.fingerprint).not.toBe(bare.fingerprint); + expect(withProfile.artifact).toContain('"profile":{"columns":[{"leaf":0,"dict":["online","offline"]}]}'); + expect(() => bare.decode(withProfile.encode([{ s: "online" }]))).toThrow(); + }); + + test("a decoder without the profile refuses dictionary columns", () => { + const codec = compileIR(IR, { plan: "columnar", profile, pack: false }); + const body = codec.encodeBody([{ s: "online" }]); + const bare = compileIR(IR, { plan: "columnar", pack: false }); + expect(() => bare.decodeBody(body)).toThrow("dictionary column requires a profile"); + }); + + test("an out-of-range code is rejected", () => { + const codec = compileIR(IR, { plan: "columnar", profile, pack: false }); + expect(() => codec.decodeBody(fromHex("010109"))).toThrow("out of range"); + }); + + test("invalid profiles are rejected at compile", () => { + const bad = (columns: unknown) => + compileIR(IR, { plan: "columnar", profile: { version: 1, shared: { columns } } as never }); + expect(() => bad([{ leaf: 9, dict: ["a"] }])).toThrow("not a column"); + expect(() => bad([{ leaf: 0, dict: ["a", "a"] }])).toThrow("duplicate entry"); + expect(() => bad([{ leaf: 0, dict: [] }])).toThrow("1 to"); + expect(() => + compileIR(IR, { plan: "row", profile: { version: 1, shared: { columns: [] } } }), + ).toThrow("columnar plan only"); + }); +}); + +describe("profiled golden vectors", () => { + for (const v of vectors.profiled.valid) { + test(v.name, () => { + const codec = compileIR(v.ir as IRNode, { plan: "columnar", profile: v.profile as never, pack: false }); + expect(toHex(codec.encodeBody(v.value))).toBe(v.hex); + expect(codec.decodeBody(fromHex(v.hex))).toEqual(v.value); + }); + } + + for (const v of vectors.profiled.invalidDecode) { + test(v.name, () => { + const codec = compileIR(v.ir as IRNode, { plan: "columnar", profile: v.profile as never, pack: false }); + try { + codec.decodeBody(fromHex(v.hex)); + throw new Error("expected failure"); + } catch (err) { + expect((err as HyperflyError).code as string).toBe(v.error); + } + }); + } + + for (const v of vectors.profiled.requiresProfile) { + test(v.name, () => { + const codec = compileIR(v.ir as IRNode, { plan: "columnar", pack: false }); + try { + codec.decodeBody(fromHex(v.hex)); + throw new Error("expected failure"); + } catch (err) { + expect((err as HyperflyError).code as string).toBe(v.error); + } + }); + } +}); + +describe("profiles: aliased schema nodes", () => { + // A golden vector cannot express this: loading IR from JSON always yields distinct + // objects, so only an in-memory schema that reuses one node reaches the hazard. + const arr: IRNode = { + kind: "array", + element: { kind: "struct", fields: [{ name: "s", type: { kind: "string" } }] }, + }; + const IR: IRNode = { + kind: "struct", + fields: [ + { name: "a", type: arr }, + { name: "b", type: arr }, + ], + }; + const profile = { + version: 1 as const, + shared: { + columns: [ + { leaf: 0, dict: ["red", "green"] }, + { leaf: 1, dict: ["green", "red"] }, + ], + }, + }; + + test("one node object at two positions still gets distinct ordinals", () => { + const codec = compileIR(IR, { plan: "columnar", profile, pack: false }); + const value = { a: [{ s: "red" }], b: [{ s: "red" }] }; + // "red" is code 1 under leaf 0 and code 2 under leaf 1 + expect(toHex(codec.encodeBody(value))).toBe("010101010102"); + expect(codec.decodeBody(codec.encodeBody(value))).toEqual(value); + }); + + test("the trainer assigns the same ordinals the codec reads", () => { + const samples = [ + { a: [{ s: "aa" }, { s: "aa" }], b: [{ s: "bb" }, { s: "bb" }] }, + { a: [{ s: "aa" }], b: [{ s: "bb" }] }, + ]; + const trained = train(IR, samples); + expect(trained?.shared.columns).toEqual([ + { leaf: 0, dict: ["aa"] }, + { leaf: 1, dict: ["bb"] }, + ]); + const codec = compileIR(IR, { plan: "columnar", profile: trained, pack: false }); + expect(codec.decodeBody(codec.encodeBody(samples[0]!))).toEqual(samples[0]!); + }); +}); diff --git a/packages/hyperfly/test/fingerprints.test.ts b/packages/hyperfly/test/fingerprints.test.ts index 4798a60..8884480 100644 --- a/packages/hyperfly/test/fingerprints.test.ts +++ b/packages/hyperfly/test/fingerprints.test.ts @@ -5,7 +5,7 @@ import { fingerprintOf, serializeArtifact, toHex, type IRNode, type PlanLayout } describe("fingerprint vectors", () => { for (const c of vectors.cases) { test(c.name, () => { - const canonical = serializeArtifact(c.ir as IRNode, c.plan as PlanLayout); + const canonical = serializeArtifact(c.ir as IRNode, c.plan as PlanLayout, (c as { profile?: never }).profile); expect(canonical).toBe(c.canonical); expect(toHex(fingerprintOf(canonical))).toBe(c.fingerprint); }); diff --git a/packages/hyperfly/test/train.test.ts b/packages/hyperfly/test/train.test.ts new file mode 100644 index 0000000..7194657 --- /dev/null +++ b/packages/hyperfly/test/train.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from "bun:test"; +import { compileIR, train, MAX_DICT_ENTRIES, type IRNode } from "../src/index.js"; + +const ROWS: IRNode = { + kind: "array", + element: { kind: "struct", fields: [{ name: "s", type: { kind: "string" } }] }, +}; + +const samples = (values: string[][]) => values.map((vs) => vs.map((s) => ({ s }))); + +describe("reference trainer", () => { + test("keeps repeated values and drops one-offs", () => { + const profile = train(ROWS, samples([["alpha", "alpha"], ["alpha", "beta"], ["beta", "unique"]])); + expect(profile?.shared.columns).toEqual([{ leaf: 0, dict: ["alpha", "beta"] }]); + }); + + test("returns nothing when there is no repetition to exploit", () => { + expect(train(ROWS, samples([["a"], ["b"], ["c"]]))).toBeUndefined(); + }); + + test("orders by frequency so the shortest codes reach the most common values", () => { + const profile = train(ROWS, samples([["rare", "rare", "common", "common", "common", "common"]])); + expect(profile?.shared.columns[0]!.dict[0]).toBe("common"); + }); + + test("a trained profile shrinks held-out data and round-trips", () => { + const trainSet = samples([["online", "offline", "online"], ["online", "online", "degraded"]]); + const profile = train(ROWS, trainSet); + const bare = compileIR(ROWS, { plan: "columnar", pack: false }); + const profiled = compileIR(ROWS, { plan: "columnar", profile, pack: false }); + const holdout = [{ s: "online" }, { s: "offline" }, { s: "novel" }]; + expect(profiled.encodeBody(holdout).length).toBeLessThan(bare.encodeBody(holdout).length); + expect(profiled.decodeBody(profiled.encodeBody(holdout))).toEqual(holdout); + }); + + test("output always compiles, including at the entry ceiling", () => { + const many = Array.from({ length: MAX_DICT_ENTRIES + 50 }, (_, i) => `v${i}`); + const profile = train(ROWS, samples([many, many])); + expect(profile!.shared.columns[0]!.dict.length).toBeLessThanOrEqual(MAX_DICT_ENTRIES); + expect(() => compileIR(ROWS, { plan: "columnar", profile })).not.toThrow(); + }); + + test("nested and multi-array schemas train to the right ordinals", () => { + const ir: IRNode = { + kind: "struct", + fields: [ + { name: "a", type: ROWS }, + { + name: "b", + type: { + kind: "array", + element: { + kind: "struct", + fields: [{ name: "inner", type: { kind: "struct", fields: [{ name: "t", type: { kind: "string" } }] } }], + }, + }, + }, + ], + }; + const sample = { a: [{ s: "aa" }, { s: "aa" }], b: [{ inner: { t: "bb" } }, { inner: { t: "bb" } }] }; + const profile = train(ir, [sample, sample]); + expect(profile?.shared.columns).toEqual([ + { leaf: 0, dict: ["aa"] }, + { leaf: 1, dict: ["bb"] }, + ]); + const codec = compileIR(ir, { plan: "columnar", profile, pack: false }); + expect(codec.decodeBody(codec.encodeBody(sample))).toEqual(sample); + }); + + test("never proposes an entry the codec would refuse", () => { + const profile = train(ROWS, samples([["\ud800x", "\ud800x"], ["ok", "ok"]])); + expect(profile?.shared.columns[0]!.dict).toEqual(["ok"]); + }); +}); diff --git a/packages/hyperfly/test/zod.test.ts b/packages/hyperfly/test/zod.test.ts index 20ef571..957496a 100644 --- a/packages/hyperfly/test/zod.test.ts +++ b/packages/hyperfly/test/zod.test.ts @@ -156,7 +156,7 @@ describe("cross-adapter parity (retro)", () => { }); // identical string is asserted in python/tests/test_cross_adapter.py expect(serializeArtifact(toIR(Row), "columnar")).toBe( - '{"wire":1,"plan":{"layout":"columnar","version":2},"ir":' + + '{"wire":1,"plan":{"layout":"columnar","version":3},"ir":' + '{"kind":"struct","fields":[' + '{"name":"id","type":{"kind":"string"}},' + '{"name":"kind","type":{"kind":"literal","value":"a"}},' + diff --git a/python/src/hyperfly/_codec.py b/python/src/hyperfly/_codec.py index ffe21f0..cf33520 100644 --- a/python/src/hyperfly/_codec.py +++ b/python/src/hyperfly/_codec.py @@ -6,8 +6,17 @@ import zlib from typing import Any -from ._ir import LEAF_KINDS, fingerprint_of, has_payload, serialize_artifact, validate_ir +from ._ir import ( + LEAF_KINDS, + column_count, + fingerprint_of, + has_payload, + serialize_artifact, + validate_ir, + validate_profile, +) from ._wire import ( + HyperflyError, DEFAULT_LIMITS, INT_MAX, INT_MIN, @@ -179,9 +188,24 @@ def _sig_bytes(x: int) -> int: class _Ctx: - __slots__ = ("max_depth", "max_items", "max_byte_length", "columnar", "deflate", "inflate") - - def __init__(self, limits: Limits, columnar: bool, deflate, inflate) -> None: + __slots__ = ( + "max_depth", + "max_items", + "max_byte_length", + "columnar", + "deflate", + "inflate", + "dicts", + "codes", + ) + + def __init__(self, limits: Limits, columnar: bool, deflate, inflate, profile=None) -> None: + self.dicts: dict[int, list[str]] = {} + self.codes: dict[int, dict[str, int]] = {} + if profile is not None: + for column in profile["shared"]["columns"]: + self.dicts[column["leaf"]] = column["dict"] + self.codes[column["leaf"]] = {v: i + 1 for i, v in enumerate(column["dict"])} self.max_depth = limits.max_depth self.max_items = limits.max_items self.max_byte_length = limits.max_byte_length @@ -190,7 +214,7 @@ def __init__(self, limits: Limits, columnar: bool, deflate, inflate) -> None: self.inflate = inflate -def _encode_node(out: bytearray, node: dict[str, Any], value: Any, path: str, depth: int, ctx: _Ctx) -> None: +def _encode_node(out: bytearray, node: dict[str, Any], value: Any, path: str, depth: int, ctx: _Ctx, column: int = 0) -> None: if depth > ctx.max_depth: _efail("depth", path, f"nesting deeper than {ctx.max_depth}") kind = node["kind"] @@ -232,10 +256,10 @@ def _encode_node(out: bytearray, node: dict[str, Any], value: Any, path: str, de out.append(0) else: out.append(1) - _encode_node(out, node["inner"], value, path, depth + 1, ctx) + _encode_node(out, node["inner"], value, path, depth + 1, ctx, column) elif kind == "array": if ctx.columnar and _columnar_eligible(node): - _encode_columnar(out, node, value, path, depth, ctx) + _encode_columnar(out, node, value, path, depth, ctx, column) return if type(value) is not list: _efail("type", path, "expected array") @@ -248,7 +272,7 @@ def _encode_node(out: bytearray, node: dict[str, Any], value: Any, path: str, de else: write_uleb(out, len(value)) for i, item in enumerate(value): - _encode_node(out, node["element"], item, f"{path}[{i}]", depth + 1, ctx) + _encode_node(out, node["element"], item, f"{path}[{i}]", depth + 1, ctx, column) elif kind == "struct": if not isinstance(value, dict): _efail("type", path, "expected object") @@ -267,13 +291,16 @@ def _encode_node(out: bytearray, node: dict[str, Any], value: Any, path: str, de nulls.append(not absent and v is None) write_bitmap(out, presence) write_bitmap(out, nulls) + field_column = column for f in node["fields"]: + base = field_column + field_column += column_count(f["type"]) if f["name"] not in value: continue v = value[f["name"]] if v is None and f.get("nullable"): continue - _encode_node(out, f["type"], v, f"{path}.{f['name']}", depth + 1, ctx) + _encode_node(out, f["type"], v, f"{path}.{f['name']}", depth + 1, ctx, base) else: _efail("type", path, f"unknown kind {kind}") @@ -344,33 +371,52 @@ def _encode_float_column(out: bytearray, values: list[Any], path: str) -> None: write_uleb(out, zigzag(m)) -def _encode_string_column(out: bytearray, values: list[Any], path: str, ctx: _Ctx) -> None: +def _encode_string_column(out: bytearray, values: list[Any], path: str, ctx: _Ctx, ordinal: int) -> None: if not values: out.append(0) return encoded = [_utf8(v, f"{path}[{i}]") for i, v in enumerate(values)] plain_cost = sum(uleb_len(len(b)) + len(b) for b in encoded) + codes = None + dict_cost = math.inf + lookup = ctx.codes.get(ordinal) + if lookup is not None: + codes = [lookup.get(v, 0) for v in values] + dict_cost = sum( + uleb_len(c) + (uleb_len(len(encoded[i])) + len(encoded[i]) if c == 0 else 0) + for i, c in enumerate(codes) + ) + packed = None packed_cost = math.inf if ctx.deflate is not None and ctx.inflate is not None: packed = ctx.deflate(b"".join(encoded)) packed_cost = sum(uleb_len(len(b)) for b in encoded) + uleb_len(len(packed)) + len(packed) - if packed is not None and packed_cost < plain_cost: - out.append(1) - for b in encoded: - write_uleb(out, len(b)) - write_uleb(out, len(packed)) - out += packed - else: - out.append(0) + best = min(plain_cost, dict_cost, packed_cost) + if best == plain_cost: + out.append(0x00) for b in encoded: write_uleb(out, len(b)) out += b + return + if best == dict_cost and codes is not None: + out.append(0x01) + for i, c in enumerate(codes): + write_uleb(out, c) + if c == 0: + write_uleb(out, len(encoded[i])) + out += encoded[i] + return + out.append(0x02) + for b in encoded: + write_uleb(out, len(b)) + write_uleb(out, len(packed)) + out += packed -def _encode_columnar(out: bytearray, node: dict[str, Any], value: Any, path: str, depth: int, ctx: _Ctx) -> None: +def _encode_columnar(out: bytearray, node: dict[str, Any], value: Any, path: str, depth: int, ctx: _Ctx, ordinal_base: int = 0) -> None: element = node["element"] if type(value) is not list: _efail("type", path, "expected array") @@ -400,7 +446,7 @@ def container(row: dict[str, Any], segs: tuple[str, ...], i: int) -> dict[str, A obj = nxt return obj - for segs, field in leaves: + for leaf_index, (segs, field) in enumerate(leaves): dotted = ".".join(segs) leaf = segs[-1] field_path = f"{path}[].{dotted}" @@ -442,13 +488,13 @@ def container(row: dict[str, Any], segs: tuple[str, ...], i: int) -> dict[str, A _efail("type", f"{field_path}[{i}]", "expected boolean") write_bitmap(out, list(participating)) elif kind == "string": - _encode_string_column(out, participating, field_path, ctx) + _encode_string_column(out, participating, field_path, ctx, ordinal_base + leaf_index) else: for i, v in enumerate(participating): _encode_node(out, t, v, f"{field_path}[{i}]", depth + 2, ctx) -def _decode_node(r: Reader, node: dict[str, Any], path: str, depth: int, ctx: _Ctx) -> Any: +def _decode_node(r: Reader, node: dict[str, Any], path: str, depth: int, ctx: _Ctx, column: int = 0) -> Any: if depth > ctx.max_depth: _dfail("depth", path, f"nesting deeper than {ctx.max_depth}") kind = node["kind"] @@ -496,17 +542,17 @@ def _decode_node(r: Reader, node: dict[str, Any], path: str, depth: int, ctx: _C return None if marker != 1: _dfail("marker", path, f"invalid nullable marker 0x{marker:x}") - return _decode_node(r, node["inner"], path, depth + 1, ctx) + return _decode_node(r, node["inner"], path, depth + 1, ctx, column) if kind == "array": if ctx.columnar and _columnar_eligible(node): - return _decode_columnar(r, node, path, depth, ctx) + return _decode_columnar(r, node, path, depth, ctx, column) length = node.get("length") if length is None: length = read_uleb(r) if length > r.limits.max_items: _dfail("limit", path, f"array count {length} exceeds limit {r.limits.max_items}") _bound_by_input(r, length, node["element"], path) - return [_decode_node(r, node["element"], f"{path}[{i}]", depth + 1, ctx) for i in range(length)] + return [_decode_node(r, node["element"], f"{path}[{i}]", depth + 1, ctx, column) for i in range(length)] if kind == "struct": optional = [f for f in node["fields"] if f.get("optional")] nullable = [f for f in node["fields"] if f.get("nullable")] @@ -514,7 +560,10 @@ def _decode_node(r: Reader, node: dict[str, Any], path: str, depth: int, ctx: _C nulls = read_bitmap(r, len(nullable), path) pi = ni = 0 out: dict[str, Any] = {} + field_column = column for f in node["fields"]: + base = field_column + field_column += column_count(f["type"]) present = True if f.get("optional"): present = presence[pi] @@ -530,7 +579,7 @@ def _decode_node(r: Reader, node: dict[str, Any], path: str, depth: int, ctx: _C if is_null: out[f["name"]] = None continue - out[f["name"]] = _decode_node(r, f["type"], f"{path}.{f['name']}", depth + 1, ctx) + out[f["name"]] = _decode_node(r, f["type"], f"{path}.{f['name']}", depth + 1, ctx, base) return out _dfail("marker", path, f"unknown kind {kind}") @@ -620,10 +669,10 @@ def mantissa(m: int, i: int) -> float: return out -def _decode_string_column(r: Reader, count: int, path: str, ctx: _Ctx) -> list[str]: +def _decode_string_column(r: Reader, count: int, path: str, ctx: _Ctx, ordinal: int = 0) -> list[str]: mode = r.u8() - if mode > 1: - _dfail("marker", path, f"invalid string column mode 0x{mode:x}") + if mode > 2: + _dfail("marker", path, f"invalid string column flags 0x{mode:x}") if count == 0: if mode != 0: _dfail("marker", path, "empty column must use mode 0x00") @@ -645,6 +694,24 @@ def decode_slice(data: bytes, i: int) -> str: out.append(decode_slice(r.take(n), i)) return out + if mode == 0x01: + entries = ctx.dicts.get(ordinal) + if entries is None: + _dfail("unsupported", path, "dictionary column requires a profile for this leaf") + out = [] + for i in range(count): + code = read_uleb(r) + if code == 0: + n = read_uleb(r) + if n > r.limits.max_byte_length: + _dfail("limit", f"{path}[{i}]", "string length exceeds limit") + out.append(decode_slice(r.take(n), i)) + else: + if code > len(entries): + _dfail("range", f"{path}[{i}]", f"dictionary code {code} out of range") + out.append(entries[code - 1]) + return out + lengths = [] total = 0 for i in range(count): @@ -675,7 +742,7 @@ def decode_slice(data: bytes, i: int) -> str: return out -def _decode_columnar(r: Reader, node: dict[str, Any], path: str, depth: int, ctx: _Ctx) -> list[dict[str, Any]]: +def _decode_columnar(r: Reader, node: dict[str, Any], path: str, depth: int, ctx: _Ctx, ordinal_base: int = 0) -> list[dict[str, Any]]: element = node["element"] length = node.get("length") if length is None: @@ -694,7 +761,7 @@ def container(row: dict[str, Any], segs: tuple[str, ...]) -> dict[str, Any]: obj = obj.setdefault(seg, {}) return obj - for segs, field in leaves: + for leaf_index, (segs, field) in enumerate(leaves): leaf = segs[-1] field_path = f"{path}[].{'.'.join(segs)}" # nested structs are required and non-nullable: materialize the container chain at @@ -732,7 +799,7 @@ def container(row: dict[str, Any], segs: tuple[str, ...]) -> dict[str, Any]: elif kind == "bool": values = read_bitmap(r, len(slots), field_path) elif kind == "string": - values = _decode_string_column(r, len(slots), field_path, ctx) + values = _decode_string_column(r, len(slots), field_path, ctx, ordinal_base + leaf_index) else: values = [ _decode_node(r, t, f"{path}[{row}].{leaf}", depth + 2, ctx) for row in slots @@ -763,13 +830,18 @@ def _default_inflate(data: bytes, max_output_length: int) -> bytes: class Codec: - def __init__(self, ir: dict[str, Any], plan: str, limits: Limits, pack) -> None: + def __init__(self, ir: dict[str, Any], plan: str, limits: Limits, pack, profile=None) -> None: validate_ir(ir) + if profile is not None: + if plan != "columnar": + raise HyperflyError("ir", "profiles apply to the columnar plan only") + validate_profile(ir, profile) # isolate from later caller mutation: the fingerprint is fixed at compile time ir = copy.deepcopy(ir) self._ir = ir self.plan = plan - self.artifact = serialize_artifact(ir, plan) + self._profile = copy.deepcopy(profile) if profile is not None else None + self.artifact = serialize_artifact(ir, plan, self._profile) self._fp = fingerprint_of(self.artifact) self.fingerprint = self._fp.hex() self._limits = limits @@ -779,7 +851,12 @@ def __init__(self, ir: dict[str, Any], plan: str, limits: Limits, pack) -> None: deflate, inflate = _default_deflate, _default_inflate else: deflate, inflate = pack.get("deflate"), pack.get("inflate") - self._ctx = _Ctx(limits, plan == "columnar", deflate, inflate) + self._ctx = _Ctx(limits, plan == "columnar", deflate, inflate, self._profile) + + @property + def profile(self) -> dict[str, Any] | None: + """A copy: the compiled profile is fixed by the fingerprint and never mutated.""" + return copy.deepcopy(self._profile) if self._profile is not None else None @property def ir(self) -> dict[str, Any]: @@ -814,5 +891,12 @@ def decode(self, data: bytes) -> Any: return self.decode_body(data[HEADER_SIZE:]) -def compile_ir(ir: dict[str, Any], *, plan: str = "row", limits: Limits | None = None, pack=None) -> Codec: - return Codec(ir, plan, limits or DEFAULT_LIMITS, pack) +def compile_ir( + ir: dict[str, Any], + *, + plan: str = "row", + limits: Limits | None = None, + pack=None, + profile: dict[str, Any] | None = None, +) -> Codec: + return Codec(ir, plan, limits or DEFAULT_LIMITS, pack, profile) diff --git a/python/src/hyperfly/_ir.py b/python/src/hyperfly/_ir.py index feb0a9e..c91e924 100644 --- a/python/src/hyperfly/_ir.py +++ b/python/src/hyperfly/_ir.py @@ -6,7 +6,7 @@ from ._wire import INT_MAX, INT_MIN, HyperflyError LEAF_KINDS = frozenset({"bool", "int", "float64", "string", "bytes", "enum", "literal"}) -_PLAN_VERSION = {"row": 1, "columnar": 2} +_PLAN_VERSION = {"row": 1, "columnar": 3} def _fail(path: str, message: str) -> None: @@ -166,9 +166,102 @@ def serialize_node(node: dict[str, Any]) -> str: return '{"kind":"struct","fields":[' + ",".join(fields) + "]}" -def serialize_artifact(ir: dict[str, Any], layout: str = "row") -> str: +MAX_DICT_ENTRIES = 16383 + + +def serialize_shared(shared: dict[str, Any]) -> str: + columns = [ + '{"leaf":' + str(c["leaf"]) + ',"dict":[' + ",".join(_esc(e) for e in c["dict"]) + "]}" + for c in shared["columns"] + ] + return '{"columns":[' + ",".join(columns) + "]}" + + +def serialize_artifact(ir: dict[str, Any], layout: str = "row", profile: dict[str, Any] | None = None) -> str: version = _PLAN_VERSION[layout] - return f'{{"wire":1,"plan":{{"layout":"{layout}","version":{version}}},"ir":{serialize_node(ir)}}}' + head = f'{{"wire":1,"plan":{{"layout":"{layout}","version":{version}}},"ir":{serialize_node(ir)}' + if profile is None: + return head + "}" + return head + ',"profile":' + serialize_shared(profile["shared"]) + "}" + + +def enumerate_columns(ir: dict[str, Any]) -> list[str]: + """Spec 6.1: the kind of every columnar leaf in the schema, in ordinal order.""" + out: list[str] = [] + _walk_columns(ir, out, None) + return out + + +def column_count(node: dict[str, Any]) -> int: + """Columnar leaves under this node. A pure function of the subtree, so two schema + positions sharing one node object still count the same — which is why column bases + are threaded positionally rather than looked up by node identity.""" + from ._codec import _columnar_eligible, _flatten_leaves + + kind = node["kind"] + if kind == "array": + if _columnar_eligible(node): + leaves = _flatten_leaves(node["element"]) + if leaves is not None: + return len(leaves) + return column_count(node["element"]) + if kind == "nullable": + return column_count(node["inner"]) + if kind == "struct": + return sum(column_count(f["type"]) for f in node["fields"]) + return 0 + + +def _walk_columns(node: dict[str, Any], out: list[str], bases: None = None) -> None: + from ._codec import _columnar_eligible, _flatten_leaves + + kind = node["kind"] + if kind == "array": + if _columnar_eligible(node): + leaves = _flatten_leaves(node["element"]) + if leaves is not None: + for _segs, field in leaves: + out.append(field["type"]["kind"]) + return + _walk_columns(node["element"], out, bases) + return + if kind == "nullable": + _walk_columns(node["inner"], out, bases) + return + if kind == "struct": + for f in node["fields"]: + _walk_columns(f["type"], out, bases) + return + + +def validate_profile(ir: dict[str, Any], profile: dict[str, Any]) -> None: + def fail(message: str) -> None: + raise HyperflyError("ir", f"profile: {message}") + + if profile.get("version") != 1: + fail(f"unsupported profile version {profile.get('version')}") + kinds = enumerate_columns(ir) + previous = -1 + for column in profile["shared"]["columns"]: + leaf = column["leaf"] + if type(leaf) is not int or leaf < 0 or leaf >= len(kinds): + fail(f"leaf {leaf} is not a column in this schema") + if leaf <= previous: + fail("columns must be sorted by ascending leaf and unique") + previous = leaf + if kinds[leaf] != "string": + fail(f"leaf {leaf} is not a string column") + entries = column["dict"] + if not entries or len(entries) > MAX_DICT_ENTRIES: + fail(f"leaf {leaf}: a dictionary holds 1 to {MAX_DICT_ENTRIES} entries") + seen: set[str] = set() + for entry in entries: + if type(entry) is not str: + fail(f"leaf {leaf}: entries must be strings") + _check_string(entry, f"leaf {leaf}", "dictionary entry") + if entry in seen: + fail(f"leaf {leaf}: duplicate entry gives one value two codes") + seen.add(entry) def fingerprint_of(artifact: str) -> bytes: diff --git a/python/tests/test_cross_adapter.py b/python/tests/test_cross_adapter.py index 5395ea3..d84bc34 100644 --- a/python/tests/test_cross_adapter.py +++ b/python/tests/test_cross_adapter.py @@ -26,7 +26,7 @@ class Row(BaseModel): EXPECTED_ARTIFACT = ( - '{"wire":1,"plan":{"layout":"columnar","version":2},"ir":' + '{"wire":1,"plan":{"layout":"columnar","version":3},"ir":' '{"kind":"struct","fields":[' '{"name":"id","type":{"kind":"string"}},' '{"name":"kind","type":{"kind":"literal","value":"a"}},' diff --git a/python/tests/test_fingerprints.py b/python/tests/test_fingerprints.py index adb4f9f..00da860 100644 --- a/python/tests/test_fingerprints.py +++ b/python/tests/test_fingerprints.py @@ -7,6 +7,6 @@ def test_fingerprints_match_reference(): assert CASES, "fingerprint vectors missing" for case in CASES: - canonical = serialize_artifact(case["ir"], case["plan"]) + canonical = serialize_artifact(case["ir"], case["plan"], case.get("profile")) assert canonical == case["canonical"], case["name"] assert fingerprint_of(canonical).hex() == case["fingerprint"], case["name"] diff --git a/python/tests/test_retro.py b/python/tests/test_retro.py index d4beb75..8e54411 100644 --- a/python/tests/test_retro.py +++ b/python/tests/test_retro.py @@ -191,3 +191,31 @@ def deflate(data: bytes) -> bytes: codec = compile_ir(ir, plan="columnar", pack={"deflate": deflate}) assert codec.decode_body(codec.encode_body(rows)) == rows + + +def test_aliased_array_nodes_get_distinct_ordinals(): + # A golden vector cannot express this: IR loaded from JSON always has distinct + # objects, so only an in-memory schema reusing one node reaches the hazard. + arr = {"kind": "array", "element": {"kind": "struct", "fields": [{"name": "s", "type": {"kind": "string"}}]}} + ir = {"kind": "struct", "fields": [{"name": "a", "type": arr}, {"name": "b", "type": arr}]} + profile = { + "version": 1, + "shared": {"columns": [ + {"leaf": 0, "dict": ["red", "green"]}, + {"leaf": 1, "dict": ["green", "red"]}, + ]}, + } + codec = compile_ir(ir, plan="columnar", profile=profile, pack=False) + value = {"a": [{"s": "red"}], "b": [{"s": "red"}]} + assert codec.encode_body(value).hex() == "010101010102" + assert codec.decode_body(codec.encode_body(value)) == value + + +def test_codec_profile_is_isolated_from_mutation(): + ir = {"kind": "array", "element": {"kind": "struct", "fields": [{"name": "s", "type": {"kind": "string"}}]}} + profile = {"version": 1, "shared": {"columns": [{"leaf": 0, "dict": ["online", "offline"]}]}} + codec = compile_ir(ir, plan="columnar", profile=profile, pack=False) + body = codec.encode_body([{"s": "online"}]) + codec.profile["shared"]["columns"][0]["dict"][0] = "HIJACKED" + profile["shared"]["columns"][0]["dict"][0] = "HIJACKED" + assert codec.decode_body(body) == [{"s": "online"}] diff --git a/python/tests/test_vectors.py b/python/tests/test_vectors.py index 7704012..c908030 100644 --- a/python/tests/test_vectors.py +++ b/python/tests/test_vectors.py @@ -87,3 +87,29 @@ def test_packed_without_inflate_fails_closed(): with pytest.raises(HyperflyError) as err: codec.decode_body(bytes.fromhex(vector["hex"])) assert err.value.code == "unsupported" + + +PROFILED = COLUMNAR["profiled"] + + +@pytest.mark.parametrize("vector", PROFILED["valid"], ids=lambda v: v["name"]) +def test_profiled_valid(vector): + codec = compile_ir(vector["ir"], plan="columnar", profile=vector["profile"], pack=False) + assert codec.encode_body(vector["value"]).hex() == vector["hex"], vector["name"] + assert deep_eq(codec.decode_body(bytes.fromhex(vector["hex"])), vector["value"]), vector["name"] + + +@pytest.mark.parametrize("vector", PROFILED["invalidDecode"], ids=lambda v: v["name"]) +def test_profiled_invalid_decode(vector): + codec = compile_ir(vector["ir"], plan="columnar", profile=vector["profile"], pack=False) + with pytest.raises(HyperflyError) as err: + codec.decode_body(bytes.fromhex(vector["hex"])) + assert err.value.code == vector["error"], vector["name"] + + +@pytest.mark.parametrize("vector", PROFILED["requiresProfile"], ids=lambda v: v["name"]) +def test_profiled_requires_profile(vector): + codec = compile_ir(vector["ir"], plan="columnar", pack=False) + with pytest.raises(HyperflyError) as err: + codec.decode_body(bytes.fromhex(vector["hex"])) + assert err.value.code == vector["error"], vector["name"] diff --git a/rust/src/codec.rs b/rust/src/codec.rs index c0dac37..0014b8f 100644 --- a/rust/src/codec.rs +++ b/rust/src/codec.rs @@ -1,4 +1,8 @@ -use crate::ir::{fingerprint_of, has_payload, serialize_artifact, validate, Field, Literal, Node, Plan}; +use crate::ir::{ + column_count, fingerprint_of, has_payload, serialize_artifact, validate, validate_profile, Field, + Literal, Node, Plan, Profile, +}; +use std::collections::HashMap; use crate::value::Value; use crate::wire::*; @@ -49,6 +53,42 @@ fn flatten<'a>(fields: &'a [Field], segs: &mut Vec<&'a str>, out: &mut Vec( + fields: &'a [Field], + segs: &mut Vec<&'a str>, + out: &mut Vec<&'static str>, +) -> bool { + if fields.is_empty() { + return false; + } + for f in fields { + match &f.ty { + Node::Struct(inner) => { + if f.optional || f.nullable { + return false; + } + segs.push(&f.name); + if !flatten_for_profile(inner, segs, out) { + return false; + } + segs.pop(); + } + t if LEAF_OK(t) => out.push(match t { + Node::Bool => "bool", + Node::Int { .. } => "int", + Node::Float64 => "float64", + Node::Str => "string", + Node::Bytes => "bytes", + Node::Enum(_) => "enum", + _ => "literal", + }), + _ => return false, + } + } + true +} + fn columnar_leaves(element: &Node) -> Option>> { let Node::Struct(fields) = element else { return None }; let mut out = Vec::new(); @@ -154,6 +194,9 @@ pub struct Codec { plan: Plan, limits: Limits, pack: bool, + profile: Option, + dicts: HashMap>, + codes: HashMap>, pub artifact: String, fp: [u8; 16], pub fingerprint: String, @@ -161,22 +204,64 @@ pub struct Codec { impl Codec { pub fn compile(ir: Node, plan: Plan, limits: Limits, pack: bool) -> Result { + Codec::compile_with_profile(ir, plan, limits, pack, None) + } + + pub fn compile_with_profile( + ir: Node, + plan: Plan, + limits: Limits, + pack: bool, + profile: Option, + ) -> Result { validate(&ir, "$")?; - let artifact = serialize_artifact(&ir, plan); + if let Some(p) = &profile { + if plan != Plan::Columnar { + return err(ErrorCode::Ir, "profiles apply to the columnar plan only"); + } + validate_profile(&ir, p)?; + } + let artifact = serialize_artifact(&ir, plan, profile.as_ref()); let fp = fingerprint_of(&artifact); let fingerprint = fp.iter().map(|b| format!("{b:02x}")).collect(); - Ok(Codec { ir, plan, limits, pack, artifact, fp, fingerprint }) + let mut dicts = HashMap::new(); + let mut codes = HashMap::new(); + if let Some(p) = &profile { + for column in &p.columns { + dicts.insert(column.leaf, column.dict.clone()); + let lookup: HashMap = column + .dict + .iter() + .enumerate() + .map(|(i, e)| (e.clone(), i as u64 + 1)) + .collect(); + codes.insert(column.leaf, lookup); + } + } + Ok(Codec { ir, plan, limits, pack, profile, dicts, codes, artifact, fp, fingerprint }) + } + + pub fn profile(&self) -> Option<&Profile> { + self.profile.as_ref() + } + + fn dict_of(&self, ordinal: usize) -> Option<&[String]> { + self.dicts.get(&ordinal).map(|d| d.as_slice()) + } + + fn codes_of(&self, ordinal: usize) -> Option<&HashMap> { + self.codes.get(&ordinal) } pub fn encode_body(&self, value: &Value) -> Result> { let mut out = Vec::new(); - self.enc(&mut out, &self.ir, value, "$", 0)?; + self.enc(&mut out, &self.ir, value, "$", 0, 0)?; Ok(out) } pub fn decode_body(&self, data: &[u8]) -> Result { let mut r = Reader::new(data, self.limits); - let value = self.dec(&mut r, &self.ir, "$", 0)?; + let value = self.dec(&mut r, &self.ir, "$", 0, 0)?; r.expect_end()?; Ok(value) } @@ -207,7 +292,7 @@ impl Codec { self.decode_body(&data[HEADER_SIZE..]) } - fn enc(&self, out: &mut Vec, node: &Node, value: &Value, path: &str, depth: u32) -> Result<()> { + fn enc(&self, out: &mut Vec, node: &Node, value: &Value, path: &str, depth: u32, column: usize) -> Result<()> { if depth > self.limits.max_depth { return err(ErrorCode::Depth, format!("{path}: nesting deeper than {}", self.limits.max_depth)); } @@ -277,13 +362,13 @@ impl Codec { Ok(()) } else { out.push(1); - self.enc(out, inner, value, path, depth + 1) + self.enc(out, inner, value, path, depth + 1, column) } } Node::Array { element, length } => { if self.plan == Plan::Columnar { if let Some(leaves) = columnar_leaves(element) { - return self.enc_columnar(out, element, &leaves, *length, value, path, depth); + return self.enc_columnar(out, column, &leaves, *length, value, path, depth); } } let Value::Array(items) = value else { @@ -301,7 +386,7 @@ impl Codec { None => write_uleb(out, items.len() as u64)?, } for (i, item) in items.iter().enumerate() { - self.enc(out, element, item, &format!("{path}[{i}]"), depth + 1)?; + self.enc(out, element, item, &format!("{path}[{i}]"), depth + 1, column)?; } Ok(()) } @@ -330,11 +415,14 @@ impl Codec { } write_bitmap(out, &presence); write_bitmap(out, &nulls); + let mut field_column = column; for f in fields { + let base = field_column; + field_column += column_count(&f.ty); match value.get(&f.name) { None => {} Some(Value::Null) if f.nullable => {} - Some(v) => self.enc(out, &f.ty, v, &format!("{path}.{}", f.name), depth + 1)?, + Some(v) => self.enc(out, &f.ty, v, &format!("{path}.{}", f.name), depth + 1, base)?, } } Ok(()) @@ -345,7 +433,7 @@ impl Codec { fn enc_columnar( &self, out: &mut Vec, - _element: &Node, + ordinal_base: usize, leaves: &[LeafCol], length: Option, value: &Value, @@ -369,7 +457,7 @@ impl Codec { } } - for leaf in leaves { + for (leaf_index, leaf) in leaves.iter().enumerate() { let f = leaf.field; let dotted = leaf.segs.join("."); let field_path = format!("{path}[].{dotted}"); @@ -457,11 +545,11 @@ impl Codec { _ => return err(ErrorCode::Type, format!("{field_path}[{i}]: expected string")), } } - enc_string_column(out, &strings, self.pack)?; + enc_string_column(out, &strings, self.pack, self.codes_of(ordinal_base + leaf_index))?; } t => { for (i, v) in participating.iter().enumerate() { - self.enc(out, t, v, &format!("{field_path}[{i}]"), depth + 2)?; + self.enc(out, t, v, &format!("{field_path}[{i}]"), depth + 2, ordinal_base + leaf_index)?; } } } @@ -469,7 +557,7 @@ impl Codec { Ok(()) } - fn dec(&self, r: &mut Reader, node: &Node, path: &str, depth: u32) -> Result { + fn dec(&self, r: &mut Reader, node: &Node, path: &str, depth: u32, column: usize) -> Result { if depth > self.limits.max_depth { return err(ErrorCode::Depth, format!("{path}: nesting deeper than {}", self.limits.max_depth)); } @@ -529,20 +617,20 @@ impl Codec { } Node::Nullable(inner) => match r.u8()? { 0 => Ok(Value::Null), - 1 => self.dec(r, inner, path, depth + 1), + 1 => self.dec(r, inner, path, depth + 1, column), m => err(ErrorCode::Marker, format!("{path}: invalid nullable marker {m:#x}")), }, Node::Array { element, length } => { if self.plan == Plan::Columnar { if let Some(leaves) = columnar_leaves(element) { - return self.dec_columnar(r, &leaves, *length, path, depth); + return self.dec_columnar(r, column, &leaves, *length, path, depth); } } let count = self.read_count(r, *length, path)?; self.bound_by_input(r, count, element, path)?; let mut out = Vec::with_capacity(count.min(4096)); for i in 0..count { - out.push(self.dec(r, element, &format!("{path}[{i}]"), depth + 1)?); + out.push(self.dec(r, element, &format!("{path}[{i}]"), depth + 1, column)?); } Ok(Value::Array(out)) } @@ -554,7 +642,10 @@ impl Codec { let mut pi = 0; let mut ni = 0; let mut out = Vec::new(); + let mut field_column = column; for f in fields { + let base = field_column; + field_column += column_count(&f.ty); let present = if f.optional { pi += 1; presence[pi - 1] @@ -577,7 +668,7 @@ impl Codec { out.push((f.name.clone(), Value::Null)); continue; } - out.push((f.name.clone(), self.dec(r, &f.ty, &format!("{path}.{}", f.name), depth + 1)?)); + out.push((f.name.clone(), self.dec(r, &f.ty, &format!("{path}.{}", f.name), depth + 1, base)?)); } Ok(Value::Object(out)) } @@ -611,7 +702,15 @@ impl Codec { Ok(n as usize) } - fn dec_columnar(&self, r: &mut Reader, leaves: &[LeafCol], length: Option, path: &str, depth: u32) -> Result { + fn dec_columnar( + &self, + r: &mut Reader, + ordinal_base: usize, + leaves: &[LeafCol], + length: Option, + path: &str, + depth: u32, + ) -> Result { let count = self.read_count(r, length, path)?; // the flattened leaves are the element's payload: any flag or non-literal leaf costs bits let element_has_payload = leaves @@ -683,7 +782,7 @@ impl Codec { } } - for leaf in leaves { + for (leaf_index, leaf) in leaves.iter().enumerate() { let f = leaf.field; let field_path = format!("{path}[].{}", leaf.segs.join(".")); // nested structs are required and non-nullable: materialize the container chain at @@ -730,14 +829,21 @@ impl Codec { .map(Value::Float) .collect(), Node::Bool => read_bitmap(r, slots.len(), &field_path)?.into_iter().map(Value::Bool).collect(), - Node::Str => dec_string_column(r, slots.len(), &field_path, &self.limits, self.pack)? + Node::Str => dec_string_column( + r, + slots.len(), + &field_path, + &self.limits, + self.pack, + self.dict_of(ordinal_base + leaf_index), + )? .into_iter() .map(Value::Str) .collect(), t => { let mut out = Vec::with_capacity(slots.len()); for row in &slots { - out.push(self.dec(r, t, &format!("{path}[{row}]"), depth + 2)?); + out.push(self.dec(r, t, &format!("{path}[{row}]"), depth + 2, ordinal_base + leaf_index)?); } out } @@ -939,33 +1045,69 @@ fn dec_float_column(r: &mut Reader, count: usize, path: &str) -> Result Ok(out) } -fn enc_string_column(out: &mut Vec, values: &[&str], pack: bool) -> Result<()> { +fn enc_string_column( + out: &mut Vec, + values: &[&str], + pack: bool, + codes_index: Option<&HashMap>, +) -> Result<()> { if values.is_empty() { out.push(0); return Ok(()); } let plain_cost: usize = values.iter().map(|s| uleb_len(s.len() as u64) + s.len()).sum(); + + let mut codes: Option> = None; + let mut dict_cost = usize::MAX; + if let Some(index) = codes_index { + let assigned: Vec = values.iter().map(|v| index.get(*v).copied().unwrap_or(0)).collect(); + dict_cost = assigned + .iter() + .zip(values.iter()) + .map(|(c, v)| uleb_len(*c) + if *c == 0 { uleb_len(v.len() as u64) + v.len() } else { 0 }) + .sum(); + codes = Some(assigned); + } + + let mut packed_cost = usize::MAX; + let mut packed_blob: Option> = None; if pack { let concat: Vec = values.iter().flat_map(|s| s.as_bytes().iter().copied()).collect(); let packed = miniz_oxide::deflate::compress_to_vec(&concat, 6); - let packed_cost: usize = values.iter().map(|s| uleb_len(s.len() as u64)).sum::() + packed_cost = values.iter().map(|s| uleb_len(s.len() as u64)).sum::() + uleb_len(packed.len() as u64) + packed.len(); - if packed_cost < plain_cost { - out.push(1); - for s in values { - write_uleb(out, s.len() as u64)?; + packed_blob = Some(packed); + } + + let best = plain_cost.min(dict_cost).min(packed_cost); + if best == plain_cost { + out.push(0x00); + for s in values { + write_uleb(out, s.len() as u64)?; + out.extend_from_slice(s.as_bytes()); + } + return Ok(()); + } + if best == dict_cost { + let assigned = codes.expect("dict cost implies codes"); + out.push(0x01); + for (c, v) in assigned.iter().zip(values.iter()) { + write_uleb(out, *c)?; + if *c == 0 { + write_uleb(out, v.len() as u64)?; + out.extend_from_slice(v.as_bytes()); } - write_uleb(out, packed.len() as u64)?; - out.extend_from_slice(&packed); - return Ok(()); } + return Ok(()); } - out.push(0); + let packed = packed_blob.expect("packed cost implies a blob"); + out.push(0x02); for s in values { write_uleb(out, s.len() as u64)?; - out.extend_from_slice(s.as_bytes()); } + write_uleb(out, packed.len() as u64)?; + out.extend_from_slice(&packed); Ok(()) } @@ -986,10 +1128,17 @@ fn inflate_exact(blob: &[u8], expected: usize) -> std::result::Result, ( Ok(out) } -fn dec_string_column(r: &mut Reader, count: usize, path: &str, limits: &Limits, pack: bool) -> Result> { +fn dec_string_column( + r: &mut Reader, + count: usize, + path: &str, + limits: &Limits, + pack: bool, + dict: Option<&[String]>, +) -> Result> { let mode = r.u8()?; - if mode > 1 { - return err(ErrorCode::Marker, format!("{path}: invalid string column mode {mode:#x}")); + if mode > 2 { + return err(ErrorCode::Marker, format!("{path}: invalid string column flags {mode:#x}")); } if count == 0 { if mode != 0 { @@ -1014,6 +1163,31 @@ fn dec_string_column(r: &mut Reader, count: usize, path: &str, limits: &Limits, } return Ok(out); } + if mode == 0x01 { + let entries = match dict { + Some(d) => d, + None => { + return err(ErrorCode::Unsupported, format!("{path}: dictionary column requires a profile for this leaf")) + } + }; + for i in 0..count { + let code = read_uleb(r)?; + if code == 0 { + let n = read_uleb(r)?; + if n > limits.max_byte_length { + return err(ErrorCode::Limit, format!("{path}[{i}]: string length exceeds limit")); + } + out.push(decode_slice(r.take(n as usize)?, i)?); + } else { + if code > entries.len() as u64 { + return err(ErrorCode::Range, format!("{path}[{i}]: dictionary code {code} out of range")); + } + out.push(entries[code as usize - 1].clone()); + } + } + return Ok(out); + } + let mut lengths = Vec::with_capacity(count); let mut total: u64 = 0; for i in 0..count { diff --git a/rust/src/ir.rs b/rust/src/ir.rs index 88ea424..69ceb86 100644 --- a/rust/src/ir.rs +++ b/rust/src/ir.rs @@ -227,12 +227,43 @@ impl Plan { fn version(self) -> u32 { match self { Plan::Row => 1, - Plan::Columnar => 2, + Plan::Columnar => 3, } } } -pub fn serialize_artifact(ir: &Node, plan: Plan) -> String { +pub const MAX_DICT_ENTRIES: usize = 16383; + +#[derive(Debug, Clone, PartialEq)] +pub struct ProfileColumn { + pub leaf: usize, + pub dict: Vec, +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Profile { + pub columns: Vec, +} + +pub fn serialize_shared(profile: &Profile, out: &mut String) { + out.push_str(r#"{"columns":["#); + for (i, c) in profile.columns.iter().enumerate() { + if i > 0 { + out.push(','); + } + out.push_str(&format!(r#"{{"leaf":{},"dict":["#, c.leaf)); + for (j, entry) in c.dict.iter().enumerate() { + if j > 0 { + out.push(','); + } + esc(entry, out); + } + out.push_str("]}"); + } + out.push_str("]}"); +} + +pub fn serialize_artifact(ir: &Node, plan: Plan, profile: Option<&Profile>) -> String { let mut out = String::new(); out.push_str(&format!( r#"{{"wire":1,"plan":{{"layout":"{}","version":{}}},"ir":"#, @@ -240,10 +271,85 @@ pub fn serialize_artifact(ir: &Node, plan: Plan) -> String { plan.version() )); serialize_node(ir, &mut out); + if let Some(p) = profile { + out.push_str(r#","profile":"#); + serialize_shared(p, &mut out); + } out.push('}'); out } +/// Spec 6.1: the kind of every columnar leaf in the schema, in ordinal order. +pub fn enumerate_columns(ir: &Node) -> Vec<&'static str> { + let mut out = Vec::new(); + walk_columns(ir, &mut out); + out +} + +/// Columnar leaves under this node. A pure function of the subtree, so two schema +/// positions sharing one node still count the same — which is why column bases are +/// threaded positionally rather than looked up by node identity. +pub fn column_count(node: &Node) -> usize { + let mut out = Vec::new(); + walk_columns(node, &mut out); + out.len() +} + +fn walk_columns(node: &Node, out: &mut Vec<&'static str>) { + match node { + Node::Array { element, .. } => { + if let Node::Struct(fields) = &**element { + let mut leaves = Vec::new(); + let mut segs = Vec::new(); + if crate::codec::flatten_for_profile(fields, &mut segs, &mut leaves) { + for kind in leaves { + out.push(kind); + } + return; + } + } + walk_columns(element, out); + } + Node::Nullable(inner) => walk_columns(inner, out), + Node::Struct(fields) => { + for f in fields { + walk_columns(&f.ty, out); + } + } + _ => {} + } +} + +pub fn validate_profile(ir: &Node, profile: &Profile) -> Result<()> { + let kinds = enumerate_columns(ir); + let mut previous: i64 = -1; + for column in &profile.columns { + if column.leaf >= kinds.len() { + return err(ErrorCode::Ir, format!("profile: leaf {} is not a column in this schema", column.leaf)); + } + if column.leaf as i64 <= previous { + return err(ErrorCode::Ir, "profile: columns must be sorted by ascending leaf and unique"); + } + previous = column.leaf as i64; + if kinds[column.leaf] != "string" { + return err(ErrorCode::Ir, format!("profile: leaf {} is not a string column", column.leaf)); + } + if column.dict.is_empty() || column.dict.len() > MAX_DICT_ENTRIES { + return err( + ErrorCode::Ir, + format!("profile: leaf {}: a dictionary holds 1 to {MAX_DICT_ENTRIES} entries", column.leaf), + ); + } + let mut seen = std::collections::HashSet::new(); + for entry in &column.dict { + if !seen.insert(entry) { + return err(ErrorCode::Ir, format!("profile: leaf {}: duplicate entry", column.leaf)); + } + } + } + Ok(()) +} + pub fn fingerprint_of(artifact: &str) -> [u8; 16] { let digest = Sha256::digest(artifact.as_bytes()); let mut fp = [0u8; 16]; diff --git a/rust/tests/packed_selftest.rs b/rust/tests/packed_selftest.rs index cf51cb7..9ed2361 100644 --- a/rust/tests/packed_selftest.rs +++ b/rust/tests/packed_selftest.rs @@ -30,3 +30,47 @@ fn packed_column_roundtrips_through_own_encoder() { let decoded = codec.decode_body(&encoded).expect("decode own output"); assert_eq!(decoded, value); } + +#[test] +fn codec_is_send_and_sync() { + fn assert_send_sync() {} + assert_send_sync::(); +} + +/// Two schema positions sharing one array node must still get distinct column ordinals. +#[test] +fn aliased_array_nodes_get_distinct_ordinals() { + use hyperfly_core::ir::{Profile, ProfileColumn}; + + let arr = Node::Array { + element: Box::new(Node::Struct(vec![Field { + name: "s".into(), + ty: Node::Str, + optional: false, + nullable: false, + }])), + length: None, + }; + let ir = Node::Struct(vec![ + Field { name: "a".into(), ty: arr.clone(), optional: false, nullable: false }, + Field { name: "b".into(), ty: arr, optional: false, nullable: false }, + ]); + let profile = Profile { + columns: vec![ + ProfileColumn { leaf: 0, dict: vec!["red".into(), "green".into()] }, + ProfileColumn { leaf: 1, dict: vec!["green".into(), "red".into()] }, + ], + }; + let codec = + Codec::compile_with_profile(ir, Plan::Columnar, Limits::default(), false, Some(profile)).unwrap(); + let row = |s: &str| Value::Array(vec![Value::Object(vec![("s".into(), Value::Str(s.into()))])]); + let value = Value::Object(vec![("a".into(), row("red")), ("b".into(), row("red"))]); + let encoded = codec.encode_body(&value).unwrap(); + // "red" is code 1 in leaf 0 and code 2 in leaf 1 + assert_eq!(to_hex(&encoded), "010101010102"); + assert_eq!(codec.decode_body(&encoded).unwrap(), value); +} + +fn to_hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} diff --git a/rust/tests/vectors.rs b/rust/tests/vectors.rs index b0521e2..1bc0df4 100644 --- a/rust/tests/vectors.rs +++ b/rust/tests/vectors.rs @@ -171,7 +171,8 @@ fn fingerprints() { "row" => Plan::Row, _ => Plan::Columnar, }; - let canonical = serialize_artifact(&node_of(&case["ir"]), plan); + let profile = case.get("profile").map(profile_of); + let canonical = serialize_artifact(&node_of(&case["ir"]), plan, profile.as_ref()); assert_eq!(canonical, case["canonical"].as_str().unwrap(), "{}", case["name"]); assert_eq!(to_hex(&fingerprint_of(&canonical)), case["fingerprint"].as_str().unwrap(), "{}", case["name"]); } @@ -189,3 +190,51 @@ fn envelope_roundtrip() { let other = compile(&file["valid"][3]["ir"], Plan::Row); assert_eq!(other.decode(&wire).unwrap_err().code.as_str(), "fingerprint"); } + +fn profile_of(json: &Json) -> hyperfly_core::ir::Profile { + hyperfly_core::ir::Profile { + columns: json["shared"]["columns"] + .as_array() + .unwrap() + .iter() + .map(|c| hyperfly_core::ir::ProfileColumn { + leaf: c["leaf"].as_u64().unwrap() as usize, + dict: c["dict"].as_array().unwrap().iter().map(|e| e.as_str().unwrap().to_owned()).collect(), + }) + .collect(), + } +} + +fn profiled_codec(v: &Json, with_profile: bool) -> Codec { + let profile = if with_profile { Some(profile_of(&v["profile"])) } else { None }; + Codec::compile_with_profile(node_of(&v["ir"]), Plan::Columnar, Limits::default(), false, profile).unwrap() +} + +#[test] +fn profiled_vectors() { + let file = load("columnar.json"); + let profiled = &file["profiled"]; + + for v in profiled["valid"].as_array().unwrap() { + let name = v["name"].as_str().unwrap(); + let codec = profiled_codec(v, true); + let value = value_of(&v["value"]); + let encoded = codec.encode_body(&value).unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(to_hex(&encoded), v["hex"].as_str().unwrap(), "{name}: bytes"); + assert!(deep_eq(&codec.decode_body(&from_hex(v["hex"].as_str().unwrap())).unwrap(), &value), "{name}"); + } + + for v in profiled["invalidDecode"].as_array().unwrap() { + let name = v["name"].as_str().unwrap(); + let codec = profiled_codec(v, true); + let e = codec.decode_body(&from_hex(v["hex"].as_str().unwrap())).expect_err(name); + assert_eq!(e.code.as_str(), v["error"].as_str().unwrap(), "{name}"); + } + + for v in profiled["requiresProfile"].as_array().unwrap() { + let name = v["name"].as_str().unwrap(); + let codec = profiled_codec(v, false); + let e = codec.decode_body(&from_hex(v["hex"].as_str().unwrap())).expect_err(name); + assert_eq!(e.code.as_str(), v["error"].as_str().unwrap(), "{name}"); + } +} diff --git a/spec/plan-columnar-v2.md b/spec/plan-columnar-v2.md deleted file mode 100644 index bc8c321..0000000 --- a/spec/plan-columnar-v2.md +++ /dev/null @@ -1,115 +0,0 @@ -# Hyperfly plan `columnar` — v2 - -Status: draft. Extends `spec/wire-v0.md`; everything there (envelope, varints, -bitmaps, scalar encodings, limits, canonical serialization) applies unchanged. -The artifact is `{"wire":1,"plan":{"layout":"columnar","version":2},"ir":…}` — -a different fingerprint than the row plan for the same IR, so the two never -mix on the wire. (v1 lacked string column modes and was never released; no v1 -artifact exists in the wild.) - -## 1. Scope - -Under this plan, every **eligible** `array` node is encoded column-wise. -Eligible: the element is a `struct` whose fields are primitives (`bool`, -`int`, `float64`, `string`, `bytes`, `enum`, `literal`) or required, -non-nullable structs of the same shape, recursively, with at least one leaf. -Nested structs flatten into leaf columns, depth-first in declared order, so a -leaf's participation is exactly its row's; leaf-level `optional`/`nullable` -flags keep their bitmaps. Anything else — arrays or nullable/optional structs -inside the element — sends the whole array down the row encoding from -wire-v0 §4.8/§4.10. Eligibility is a pure function of the IR, so both sides -always agree. - -## 2. Layout of an eligible array - -1. `uvarint` row count `n` (omitted for fixed-length arrays). -2. For each field, in declared order: - a. If `optional`: presence bitmap, `n` bits. - b. If `nullable`: null bitmap, `n` bits — one bit per row regardless of - presence; the bit for an absent row MUST be zero. - c. The column payload over **participating** rows (present and not - bitmap-null), in row order. - -## 3. Column payloads - -`k` = participating row count. - -- **literal** — zero bytes. -- **enum** — `uvarint` index per value. -- **bytes** — `uvarint` length + bytes per value (as wire-v0). -- **string** — one mode byte, then: - - `0x00` plain: `uvarint` length + strict UTF-8 bytes per value. - - `0x01` packed: `uvarint` byte length per value in row order, then a - `uvarint` blob length and a raw-deflate (RFC 1951) stream of the - concatenated UTF-8 bytes. The inflated size MUST equal the sum of the - declared lengths; each slice MUST be strict UTF-8; per-value and total - lengths obey the decoder byte limits. A decoder without an inflate - capability MUST reject packed columns as unsupported (and the protocol - layer falls back to JSON) rather than guess. -- **bool** — one bitmap of `k` bits (padding rules from wire-v0 §3.5). -- **int** — one mode byte, then: - - `0x00` raw: each value in its wire-v0 form (`uvarint(v - min)` when `min` - is declared, else `svarint(v)`). - - `0x01` delta: the first value in its wire-v0 form, then - `svarint(v[i] - v[i-1])` for each subsequent value. Differences stay - within 55 bits for domain-valid values, so the 8-byte uvarint cap holds. - - Other mode bytes MUST be rejected. Declared bounds are validated per - decoded value, after delta accumulation. -- **float64** — one mode byte, then: - - `0x00` raw: each value as 8 bytes LE (wire-v0 §4.5 rules per value). - - `0x01` xor: first value as 8 bytes LE, then for each subsequent value a - significance length byte `s` (0–8) followed by the `s` low-order bytes of - `bits(v[i]) XOR bits(v[i-1])`, little-endian. `s` MUST be minimal: for - `s > 0` the highest emitted byte MUST be non-zero; `s = 0` means the - value repeats exactly. `s > 8` MUST be rejected. Every reconstructed - value MUST be finite and MUST NOT be the negative-zero bit pattern. - - `0x02` scaled-delta / `0x03` scaled-raw: a scale byte `d` (0–8, larger - MUST be rejected), then mantissas `m[i] = v[i] · 10^d` — `svarint(m[0])` - followed by `svarint(m[i] - m[i-1])` for `0x02`, or `svarint(m[i])` per - value for `0x03`. The mantissa is pinned to pure IEEE 754 operations so - every language derives the same bytes: - `m = sign(v) · floor(|v| · 10^d + 0.5)`, where `|v| · 10^d` and the - subsequent `+ 0.5` are two separately rounded IEEE 754 binary64 - operations — implementations MUST NOT contract them into a fused - multiply-add, which would round once and can differ in the last place. - Encoders may choose these modes - only when `d` is the smallest scale for which every value satisfies - `m / 10^d == v` exactly with `m` in the integer domain; - decoding computes `Number(m) / 10^d`, which reproduces the encoder's - doubles exactly because both sides perform one correctly-rounded IEEE 754 - division of the same integers. Mantissas outside the v0 integer domain - MUST be rejected. - -When `k = 0`, int, float, and string columns still emit their mode byte -(`0x00`); other columns emit nothing. - -## 4. Encoder mode choice - -Encoders MUST pick the mode with the smaller encoded size; on a tie they MUST -pick the lowest mode byte (`0x00` < `0x01` < `0x02` < `0x03`), so a -decode → encode round trip is byte-identical and two conforming encoders agree. -Decoders accept any valid mode — canonicality is an encoder obligation, checked -by the re-encode property, not a decode-time recomputation. - -Two capabilities scope that obligation: - -- **Packing.** Deflate output is not canonical across libraries, so - byte-identical re-encode holds within an implementation+version; any - spec-valid stream decodes everywhere. An implementation without a deflate - capability emits string columns in plain mode (`0x00`) and rejects packed - input as unsupported — its output is canonical *for that capability*, and a - peer that packs still decodes it. The fingerprint identifies the schema and - plan, not the packing capability; peers with different packing capabilities - interoperate because every decoder accepts both string modes. -- **Inflater strictness.** A decoder's inflater MUST require the declared blob - to be exactly one complete DEFLATE stream: reject truncation, output longer - than the declared total, and any trailing bytes after the final block. - -## 5. Rationale (non-normative) - -Column layout groups same-typed bytes, which helps both this plan's own -transforms and any generic compressor stacked on top. Delta turns monotonic -series (timestamps, counters) into small varints. XOR captures the high-bit -locality of slowly-moving float series while staying byte-aligned — the -bit-granular Gorilla-style windows belong to a future plan, alongside -profile-trained dictionaries and entropy coding. diff --git a/spec/plan-columnar-v3.md b/spec/plan-columnar-v3.md new file mode 100644 index 0000000..ee05d07 --- /dev/null +++ b/spec/plan-columnar-v3.md @@ -0,0 +1,241 @@ +# Hyperfly plan `columnar` — v3 + +Status: draft. Extends `spec/wire-v0.md`; everything there (envelope, varints, +bitmaps, scalar encodings, limits, canonical serialization) applies unchanged. +The artifact is +`{"wire":1,"plan":{"layout":"columnar","version":3},"ir":…,"profile":…}` — a +different fingerprint than the row plan for the same IR, so the two never mix +on the wire. (v1 and v2 were never released; no artifact for either exists in +the wild.) + +## 1. Scope + +Under this plan, every **eligible** `array` node is encoded column-wise. +Eligible: the element is a `struct` whose fields are primitives (`bool`, +`int`, `float64`, `string`, `bytes`, `enum`, `literal`) or required, +non-nullable structs of the same shape, recursively, with at least one leaf. +Nested structs flatten into leaf columns, depth-first in declared order, so a +leaf's participation is exactly its row's; leaf-level `optional`/`nullable` +flags keep their bitmaps. Anything else — arrays or nullable/optional structs +inside the element — sends the whole array down the row encoding from +wire-v0 §4.8/§4.10. Eligibility is a pure function of the IR, so both sides +always agree. + +## 2. Layout of an eligible array + +1. `uvarint` row count `n` (omitted for fixed-length arrays). +2. For each field, in declared order: + a. If `optional`: presence bitmap, `n` bits. + b. If `nullable`: null bitmap, `n` bits — one bit per row regardless of + presence; the bit for an absent row MUST be zero. + c. The column payload over **participating** rows (present and not + bitmap-null), in row order. + +## 3. Column payloads + +`k` = participating row count. + +- **literal** — zero bytes. +- **enum** — `uvarint` index per value. +- **bytes** — `uvarint` length + bytes per value (as wire-v0). +- **string** — one **flags** byte, then the payload it selects. Bit 0 selects + dictionary coding, bit 1 selects deflate. Bits 2–7 are reserved and MUST be + zero. v3 defines three values; `0x03` (dictionary + deflate) is reserved and + MUST be rejected until a later version defines it. + - `0x00` plain: `uvarint` length + strict UTF-8 bytes per value. + - `0x01` dictionary: requires a dictionary for this column in the artifact's + profile (§6); a decoder without one MUST reject the column. Per value a + `uvarint` code: `0` is a literal escape followed by `uvarint` length + + UTF-8 bytes, and `n > 0` selects `entries[n − 1]`. A code beyond the + dictionary MUST be rejected. An encoder MUST emit the + code for any value present in the dictionary and MUST NOT escape it, so one + value never has two encodings. + - `0x02` deflate: `uvarint` byte length per value in row order, then a + `uvarint` blob length and a raw-deflate (RFC 1951) stream of the + concatenated UTF-8 bytes. The inflated size MUST equal the sum of the + declared lengths; each slice MUST be strict UTF-8; per-value and total + lengths obey the decoder byte limits. A decoder without an inflate + capability MUST reject deflate columns as unsupported (and the protocol + layer falls back to JSON) rather than guess. + + Bit 0 is the low bit deliberately: dictionary coding is fully deterministic + while deflate output is library-dependent, so "smallest, ties to the lowest + flags byte" (§4) also means "prefer the reproducible encoding". +- **bool** — one bitmap of `k` bits (padding rules from wire-v0 §3.5). +- **int** — one mode byte, then: + - `0x00` raw: each value in its wire-v0 form (`uvarint(v - min)` when `min` + is declared, else `svarint(v)`). + - `0x01` delta: the first value in its wire-v0 form, then + `svarint(v[i] - v[i-1])` for each subsequent value. Differences stay + within 55 bits for domain-valid values, so the 8-byte uvarint cap holds. + - Other mode bytes MUST be rejected. Declared bounds are validated per + decoded value, after delta accumulation. +- **float64** — one mode byte, then: + - `0x00` raw: each value as 8 bytes LE (wire-v0 §4.5 rules per value). + - `0x01` xor: first value as 8 bytes LE, then for each subsequent value a + significance length byte `s` (0–8) followed by the `s` low-order bytes of + `bits(v[i]) XOR bits(v[i-1])`, little-endian. `s` MUST be minimal: for + `s > 0` the highest emitted byte MUST be non-zero; `s = 0` means the + value repeats exactly. `s > 8` MUST be rejected. Every reconstructed + value MUST be finite and MUST NOT be the negative-zero bit pattern. + - `0x02` scaled-delta / `0x03` scaled-raw: a scale byte `d` (0–8, larger + MUST be rejected), then mantissas `m[i] = v[i] · 10^d` — `svarint(m[0])` + followed by `svarint(m[i] - m[i-1])` for `0x02`, or `svarint(m[i])` per + value for `0x03`. The mantissa is pinned to pure IEEE 754 operations so + every language derives the same bytes: + `m = sign(v) · floor(|v| · 10^d + 0.5)`, where `|v| · 10^d` and the + subsequent `+ 0.5` are two separately rounded IEEE 754 binary64 + operations — implementations MUST NOT contract them into a fused + multiply-add, which would round once and can differ in the last place. + Encoders may choose these modes + only when `d` is the smallest scale for which every value satisfies + `m / 10^d == v` exactly with `m` in the integer domain; + decoding computes `Number(m) / 10^d`, which reproduces the encoder's + doubles exactly because both sides perform one correctly-rounded IEEE 754 + division of the same integers. Mantissas outside the v0 integer domain + MUST be rejected. + +When `k = 0`, int, float, and string columns still emit their mode/flags byte, +which MUST be `0x00`; other columns emit nothing. + +## 4. Encoder mode choice + +Encoders MUST pick the mode with the smaller encoded size; on a tie they MUST +pick the lowest mode byte (`0x00` < `0x01` < `0x02` < `0x03`), so a +decode → encode round trip is byte-identical and two conforming encoders agree. +Decoders accept any valid mode — canonicality is an encoder obligation, checked +by the re-encode property, not a decode-time recomputation. + +Two capabilities scope that obligation: + +- **Packing.** Deflate output is not canonical across libraries, so + byte-identical re-encode holds within an implementation+version; any + spec-valid stream decodes everywhere. An implementation without a deflate + capability emits string columns in plain mode (`0x00`) and rejects packed + input as unsupported — its output is canonical *for that capability*, and a + peer that packs still decodes it. The fingerprint identifies the schema and + plan, not the packing capability; peers with different packing capabilities + interoperate because every decoder accepts both string modes. +- **Inflater strictness.** A decoder's inflater MUST require the declared blob + to be exactly one complete DEFLATE stream: reject truncation, output longer + than the declared total, and any trailing bytes after the final block. + +## 5. Rationale (non-normative) + +Column layout groups same-typed bytes, which helps both this plan's own +transforms and any generic compressor stacked on top. Delta turns monotonic +series (timestamps, counters) into small varints. XOR captures the high-bit +locality of slowly-moving float series while staying byte-aligned — the +bit-granular Gorilla-style windows belong to a future plan, alongside +profile-trained dictionaries and entropy coding. + +## 6. Profiles + +A **profile** carries knowledge learned from a route's traffic. v3 defines one +kind: per-column string dictionaries. + +### 6.1 Column ordinals + +A profile names columns by **ordinal**, never by a textual path. Field names may +legally contain `.`, `[`, `]`, and `$`, so a dotted path is ambiguous: +`struct{"a.b": struct{"c": string}}` and `struct{"a": struct{"b.c": string}}` +would produce the same key while binding to different leaves — a mismatch the +fingerprint cannot catch, because both peers compute the same artifact text. + +Ordinals come from one total enumeration of the IR: + +1. Walk the IR depth-first in declared order: struct fields in declared order, + `nullable` into its inner node, ineligible arrays into their element. +2. On reaching an **eligible** array (§1), emit its flattened leaves in + `flattenLeaves` order as consecutive ordinals, then do not descend further + into that array — an eligible element contains no nested arrays by + definition. + +The result numbers every columnar leaf in the whole schema `0 … N−1`. + +### 6.2 Profile document + +``` +{"version":1,"shared":{"columns":[{"leaf":N,"dict":["…","…"]}]},"hints":{…}} +``` + +- `shared` is decode-critical: without the identical bytes a peer cannot read + the payload. It is embedded in the artifact (§6.3). +- `hints` is advisory encoder guidance that does not affect decodability. It is + **not** part of the artifact and never changes the fingerprint, so an encoder + can adopt new hints without a fleet-wide cutover. v3 defines no hints. + +Constraints, all validated at compile time: + +- `leaf` MUST identify a `string` leaf under §6.1 and MUST be unique across + `columns`; `columns` MUST be sorted by ascending `leaf`. +- A dictionary holds 1–16383 entries, the ceiling at which a code still fits two + `uvarint` bytes. Entries are ordered most-valuable-first so the shortest codes + land on the most frequent values: code length then depends only on position, + which keeps an encoder's cost model linear rather than self-referential. +- Entries MUST be unique and well-formed Unicode (§4.10 of wire-v0). Duplicate + entries would give one value two codes and break canonicality. + +### 6.3 Artifact embedding + +The canonical artifact gains one key, after `ir`, present only when a profile +exists: + +``` +{"wire":1,"plan":{"layout":"columnar","version":3},"ir":,"profile":} +``` + +serialized with the §5 rules and these fixed key orders: + +``` +shared {"columns":[,…]} +column {"leaf":N,"dict":[,…]} +``` + +The **whole dictionary content** is embedded, not a hash or a name. A hash would +still require a canonical serializer to compute, and a name (`"prod-2026-08"`) +is expressly forbidden: two peers could then agree on a fingerprint while +holding different dictionary bytes, which is exactly the failure the +fingerprint exists to prevent. + +Artifact text is never trusted from a peer. An implementation MUST derive the +artifact from its own parsed IR and profile, and MUST NOT accept artifact text +and hash it — a decoder could otherwise match a fingerprint for a plan or +profile it cannot actually read. + +### 6.4 Rotation + +Retraining produces different dictionary bytes, therefore a different +fingerprint, therefore a hard cutover: during a rolling deploy every request +between mismatched peers falls back to JSON. A decoder SHOULD keep a registry +of codecs keyed by fingerprint and select per request, so old and new profiles +are readable simultaneously and rotation is not a cliff. + +### 6.5 Training is non-normative + +How a profile is produced is out of scope. Any document satisfying §6.2 is +valid, and the artifact pins the exact bytes, so implementations need not agree +on a training algorithm — only on canonicalization and the wire. Reference +trainers are conveniences, not part of the contract. + +If an implementation does order values (in a trainer, or anywhere else), it +MUST compare their UTF-8 byte sequences. JavaScript's default string comparison +is UTF-16 code-unit order, which disagrees with UTF-8 byte order above the BMP: +`U+FFFD` (`EF BF BD`) sorts before `U+10000` (`F0 90 80 80`) by bytes and by +code point, but after it in JavaScript. + +### 6.6 Scope and cautions + +Dictionaries apply only to **string columns of eligible arrays**. A string +outside an array, or inside an array that falls back to the row encoding, gets +nothing from a profile in v3. + +Two operational cautions: + +- A dictionary contains verbatim values from production traffic. It is a + build artifact that gets logged, committed, and shared, so a dictionary + trained on one tenant's data MUST NOT be used to serve another's. +- Whether a value is dictionary-coded is observable in the response length, so + a profiled route leaks a coarse membership signal about its own dictionary. + This is a persistent, cross-request variant of the compression-oracle + problem, and it is why dictionary coding is opt-in per route. diff --git a/spec/vectors/columnar.json b/spec/vectors/columnar.json index a2d839c..c8d4457 100644 --- a/spec/vectors/columnar.json +++ b/spec/vectors/columnar.json @@ -1,5 +1,5 @@ { - "description": "Golden vectors for plan columnar@2. Hex is body bytes only. All vectors compile with plan: columnar.", + "description": "Golden vectors for plan columnar@3. Hex is body bytes only. All vectors compile with plan: columnar.", "valid": [ { "name": "col-int-raw-on-tie", @@ -711,7 +711,7 @@ ] } }, - "hex": "0102", + "hex": "0103", "error": "marker" }, { @@ -730,7 +730,7 @@ ] } }, - "hex": "0201020208010300fcff686979", + "hex": "0202020208010300fcff686979", "error": "packed" }, { @@ -749,7 +749,7 @@ ] } }, - "hex": "010100040300dead", + "hex": "010200040300dead", "error": "packed" } ], @@ -852,7 +852,7 @@ "s": "yo" } ], - "hex": "0201020209010400fbff6869796f" + "hex": "0202020209010400fbff6869796f" }, { "name": "col-string-packed-empty-total", @@ -876,7 +876,675 @@ "s": "" } ], - "hex": "010100020300" + "hex": "010200020300" } - ] + ], + "profiled": { + "description": "Vectors compiled with plan columnar@3 AND the given profile. Packing is disabled so the encoding is fully deterministic.", + "valid": [ + { + "name": "dict-hit-miss-hit", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "profile": { + "version": 1, + "shared": { + "columns": [ + { + "leaf": 0, + "dict": [ + "online", + "offline" + ] + } + ] + } + }, + "value": [ + { + "s": "online" + }, + { + "s": "novel" + }, + { + "s": "offline" + } + ], + "hex": "03010100056e6f76656c02" + }, + { + "name": "dict-all-hits", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "profile": { + "version": 1, + "shared": { + "columns": [ + { + "leaf": 0, + "dict": [ + "online", + "offline" + ] + } + ] + } + }, + "value": [ + { + "s": "offline" + }, + { + "s": "online" + } + ], + "hex": "02010201" + }, + { + "name": "dict-two-arrays-distinct-ordinals", + "description": "each array is its own column: 'red' is code 1 under leaf 0 and code 2 under leaf 1", + "ir": { + "kind": "struct", + "fields": [ + { + "name": "a", + "type": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + } + }, + { + "name": "b", + "type": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + } + } + ] + }, + "profile": { + "version": 1, + "shared": { + "columns": [ + { + "leaf": 0, + "dict": [ + "red", + "green" + ] + }, + { + "leaf": 1, + "dict": [ + "green", + "red" + ] + } + ] + } + }, + "value": { + "a": [ + { + "s": "red" + } + ], + "b": [ + { + "s": "red" + } + ] + }, + "hex": "010101010102" + }, + { + "name": "dict-two-byte-codes", + "description": "codes past 127 take two uvarint bytes, which the cost model must charge", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "profile": { + "version": 1, + "shared": { + "columns": [ + { + "leaf": 0, + "dict": [ + "entry-value-0", + "entry-value-1", + "entry-value-2", + "entry-value-3", + "entry-value-4", + "entry-value-5", + "entry-value-6", + "entry-value-7", + "entry-value-8", + "entry-value-9", + "entry-value-10", + "entry-value-11", + "entry-value-12", + "entry-value-13", + "entry-value-14", + "entry-value-15", + "entry-value-16", + "entry-value-17", + "entry-value-18", + "entry-value-19", + "entry-value-20", + "entry-value-21", + "entry-value-22", + "entry-value-23", + "entry-value-24", + "entry-value-25", + "entry-value-26", + "entry-value-27", + "entry-value-28", + "entry-value-29", + "entry-value-30", + "entry-value-31", + "entry-value-32", + "entry-value-33", + "entry-value-34", + "entry-value-35", + "entry-value-36", + "entry-value-37", + "entry-value-38", + "entry-value-39", + "entry-value-40", + "entry-value-41", + "entry-value-42", + "entry-value-43", + "entry-value-44", + "entry-value-45", + "entry-value-46", + "entry-value-47", + "entry-value-48", + "entry-value-49", + "entry-value-50", + "entry-value-51", + "entry-value-52", + "entry-value-53", + "entry-value-54", + "entry-value-55", + "entry-value-56", + "entry-value-57", + "entry-value-58", + "entry-value-59", + "entry-value-60", + "entry-value-61", + "entry-value-62", + "entry-value-63", + "entry-value-64", + "entry-value-65", + "entry-value-66", + "entry-value-67", + "entry-value-68", + "entry-value-69", + "entry-value-70", + "entry-value-71", + "entry-value-72", + "entry-value-73", + "entry-value-74", + "entry-value-75", + "entry-value-76", + "entry-value-77", + "entry-value-78", + "entry-value-79", + "entry-value-80", + "entry-value-81", + "entry-value-82", + "entry-value-83", + "entry-value-84", + "entry-value-85", + "entry-value-86", + "entry-value-87", + "entry-value-88", + "entry-value-89", + "entry-value-90", + "entry-value-91", + "entry-value-92", + "entry-value-93", + "entry-value-94", + "entry-value-95", + "entry-value-96", + "entry-value-97", + "entry-value-98", + "entry-value-99", + "entry-value-100", + "entry-value-101", + "entry-value-102", + "entry-value-103", + "entry-value-104", + "entry-value-105", + "entry-value-106", + "entry-value-107", + "entry-value-108", + "entry-value-109", + "entry-value-110", + "entry-value-111", + "entry-value-112", + "entry-value-113", + "entry-value-114", + "entry-value-115", + "entry-value-116", + "entry-value-117", + "entry-value-118", + "entry-value-119", + "entry-value-120", + "entry-value-121", + "entry-value-122", + "entry-value-123", + "entry-value-124", + "entry-value-125", + "entry-value-126", + "entry-value-127", + "entry-value-128", + "entry-value-129", + "entry-value-130", + "entry-value-131", + "entry-value-132", + "entry-value-133", + "entry-value-134", + "entry-value-135", + "entry-value-136", + "entry-value-137", + "entry-value-138", + "entry-value-139", + "entry-value-140", + "entry-value-141", + "entry-value-142", + "entry-value-143", + "entry-value-144", + "entry-value-145", + "entry-value-146", + "entry-value-147", + "entry-value-148", + "entry-value-149", + "entry-value-150", + "entry-value-151", + "entry-value-152", + "entry-value-153", + "entry-value-154", + "entry-value-155", + "entry-value-156", + "entry-value-157", + "entry-value-158", + "entry-value-159", + "entry-value-160", + "entry-value-161", + "entry-value-162", + "entry-value-163", + "entry-value-164", + "entry-value-165", + "entry-value-166", + "entry-value-167", + "entry-value-168", + "entry-value-169", + "entry-value-170", + "entry-value-171", + "entry-value-172", + "entry-value-173", + "entry-value-174", + "entry-value-175", + "entry-value-176", + "entry-value-177", + "entry-value-178", + "entry-value-179", + "entry-value-180", + "entry-value-181", + "entry-value-182", + "entry-value-183", + "entry-value-184", + "entry-value-185", + "entry-value-186", + "entry-value-187", + "entry-value-188", + "entry-value-189", + "entry-value-190", + "entry-value-191", + "entry-value-192", + "entry-value-193", + "entry-value-194", + "entry-value-195", + "entry-value-196", + "entry-value-197", + "entry-value-198", + "entry-value-199" + ] + } + ] + } + }, + "value": [ + { + "s": "entry-value-0" + }, + { + "s": "entry-value-150" + }, + { + "s": "nope" + } + ], + "hex": "030101970100046e6f7065" + }, + { + "name": "dict-two-byte-codes-decide-the-mode", + "description": "Entries past index 126 take two-byte codes. Charging them as one byte would make dictionary mode look cheaper than plain and flip the chosen mode, so this vector fails for any encoder that mis-costs multi-byte codes.", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "profile": { + "version": 1, + "shared": { + "columns": [ + { + "leaf": 0, + "dict": [ + "fill0", + "fill1", + "fill2", + "fill3", + "fill4", + "fill5", + "fill6", + "fill7", + "fill8", + "fill9", + "fill10", + "fill11", + "fill12", + "fill13", + "fill14", + "fill15", + "fill16", + "fill17", + "fill18", + "fill19", + "fill20", + "fill21", + "fill22", + "fill23", + "fill24", + "fill25", + "fill26", + "fill27", + "fill28", + "fill29", + "fill30", + "fill31", + "fill32", + "fill33", + "fill34", + "fill35", + "fill36", + "fill37", + "fill38", + "fill39", + "fill40", + "fill41", + "fill42", + "fill43", + "fill44", + "fill45", + "fill46", + "fill47", + "fill48", + "fill49", + "fill50", + "fill51", + "fill52", + "fill53", + "fill54", + "fill55", + "fill56", + "fill57", + "fill58", + "fill59", + "fill60", + "fill61", + "fill62", + "fill63", + "fill64", + "fill65", + "fill66", + "fill67", + "fill68", + "fill69", + "fill70", + "fill71", + "fill72", + "fill73", + "fill74", + "fill75", + "fill76", + "fill77", + "fill78", + "fill79", + "fill80", + "fill81", + "fill82", + "fill83", + "fill84", + "fill85", + "fill86", + "fill87", + "fill88", + "fill89", + "fill90", + "fill91", + "fill92", + "fill93", + "fill94", + "fill95", + "fill96", + "fill97", + "fill98", + "fill99", + "fill100", + "fill101", + "fill102", + "fill103", + "fill104", + "fill105", + "fill106", + "fill107", + "fill108", + "fill109", + "fill110", + "fill111", + "fill112", + "fill113", + "fill114", + "fill115", + "fill116", + "fill117", + "fill118", + "fill119", + "fill120", + "fill121", + "fill122", + "fill123", + "fill124", + "fill125", + "fill126", + "a", + "b", + "c" + ] + } + ] + } + }, + "value": [ + { + "s": "a" + }, + { + "s": "b" + }, + { + "s": "c" + } + ], + "hex": "0300016101620163" + } + ], + "invalidDecode": [ + { + "name": "dict-code-out-of-range", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "profile": { + "version": 1, + "shared": { + "columns": [ + { + "leaf": 0, + "dict": [ + "online", + "offline" + ] + } + ] + } + }, + "hex": "010109", + "error": "range" + }, + { + "name": "dict-reserved-flags", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "profile": { + "version": 1, + "shared": { + "columns": [ + { + "leaf": 0, + "dict": [ + "online", + "offline" + ] + } + ] + } + }, + "hex": "0103", + "error": "marker" + } + ], + "requiresProfile": [ + { + "name": "dict-without-profile", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "hex": "010101", + "error": "unsupported" + } + ] + } } diff --git a/spec/vectors/fingerprints.json b/spec/vectors/fingerprints.json index 78d54b8..5d90040 100644 --- a/spec/vectors/fingerprints.json +++ b/spec/vectors/fingerprints.json @@ -125,8 +125,8 @@ "ir": { "kind": "bool" }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":2},\"ir\":{\"kind\":\"bool\"}}", - "fingerprint": "96f5b1aed1b1d4a4dee316623c08acc7" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"bool\"}}", + "fingerprint": "6a7373611b1a316a06e3f056229e4fbb" }, { "name": "bounded-int@columnar", @@ -136,8 +136,8 @@ "min": 0, "max": 100 }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":2},\"ir\":{\"kind\":\"int\",\"min\":0,\"max\":100}}", - "fingerprint": "6be791f84b580ee7dd78db739b143595" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"int\",\"min\":0,\"max\":100}}", + "fingerprint": "e0ff19388faced1d70ac4d8bb1d86b8d" }, { "name": "enum@columnar", @@ -151,8 +151,8 @@ "1d" ] }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":2},\"ir\":{\"kind\":\"enum\",\"members\":[\"1m\",\"5m\",\"1h\",\"1d\"]}}", - "fingerprint": "31ac340d52c50c2d65e23c1d031fede4" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"enum\",\"members\":[\"1m\",\"5m\",\"1h\",\"1d\"]}}", + "fingerprint": "ec9d5aac5141cc277b13c3775bef013b" }, { "name": "candles-response@columnar", @@ -224,8 +224,8 @@ } ] }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":2},\"ir\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"route\",\"type\":{\"kind\":\"literal\",\"value\":\"candles\"}},{\"name\":\"candles\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"t\",\"type\":{\"kind\":\"int\",\"min\":0}},{\"name\":\"o\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"h\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"l\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"c\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"v\",\"type\":{\"kind\":\"float64\"}}]}}},{\"name\":\"cursor\",\"type\":{\"kind\":\"string\"},\"optional\":true}]}}", - "fingerprint": "b7c09d113bd1134bdf0f0910d86996db" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"route\",\"type\":{\"kind\":\"literal\",\"value\":\"candles\"}},{\"name\":\"candles\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"t\",\"type\":{\"kind\":\"int\",\"min\":0}},{\"name\":\"o\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"h\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"l\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"c\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"v\",\"type\":{\"kind\":\"float64\"}}]}}},{\"name\":\"cursor\",\"type\":{\"kind\":\"string\"},\"optional\":true}]}}", + "fingerprint": "f000d175875a31831d1093b00a355708" }, { "name": "escaping@columnar", @@ -234,8 +234,102 @@ "kind": "literal", "value": "a\"b\\c" }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":2},\"ir\":{\"kind\":\"literal\",\"value\":\"a\\\"b\\\\c\"}}", - "fingerprint": "94f508736ee8b0eee4562a5b3d4eda07" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"literal\",\"value\":\"a\\\"b\\\\c\"}}", + "fingerprint": "cae5981b91f66217992b7652c780baa6" + }, + { + "name": "profiled-single", + "plan": "columnar", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + }, + "profile": { + "version": 1, + "shared": { + "columns": [ + { + "leaf": 0, + "dict": [ + "a\"q", + "b\\s", + "c\u0001", + "🚀" + ] + } + ] + } + }, + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"s\",\"type\":{\"kind\":\"string\"}}]}},\"profile\":{\"columns\":[{\"leaf\":0,\"dict\":[\"a\\\"q\",\"b\\\\s\",\"c\\u0001\",\"🚀\"]}]}}", + "fingerprint": "a65108e20d19c19ff525ddf4789d5ba1" + }, + { + "name": "profiled-two-arrays", + "plan": "columnar", + "ir": { + "kind": "struct", + "fields": [ + { + "name": "a", + "type": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "s", + "type": { + "kind": "string" + } + } + ] + } + } + }, + { + "name": "b", + "type": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "t", + "type": { + "kind": "string" + } + } + ] + } + } + } + ] + }, + "profile": { + "version": 1, + "shared": { + "columns": [ + { + "leaf": 1, + "dict": [ + "only-second" + ] + } + ] + } + }, + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"s\",\"type\":{\"kind\":\"string\"}}]}}},{\"name\":\"b\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"t\",\"type\":{\"kind\":\"string\"}}]}}}]},\"profile\":{\"columns\":[{\"leaf\":1,\"dict\":[\"only-second\"]}]}}", + "fingerprint": "123c921f0dccfe8d25f976757766c4e1" } ] } diff --git a/spec/vectors/generate-fingerprints.ts b/spec/vectors/generate-fingerprints.ts index 1214bbc..10771e6 100644 --- a/spec/vectors/generate-fingerprints.ts +++ b/spec/vectors/generate-fingerprints.ts @@ -34,6 +34,25 @@ const CASES: { name: string; ir: IRNode }[] = [ { name: "escaping", ir: { kind: "literal", value: 'a"b\\c' } }, ]; +const PROFILED: { name: string; ir: IRNode; profile: { version: 1; shared: { columns: { leaf: number; dict: string[] }[] } } }[] = [ + { + name: "profiled-single", + ir: { kind: "array", element: { kind: "struct", fields: [{ name: "s", type: { kind: "string" } }] } }, + profile: { version: 1, shared: { columns: [{ leaf: 0, dict: ['a"q', "b\\s", "c\u0001", "🚀"] }] } }, + }, + { + name: "profiled-two-arrays", + ir: { + kind: "struct", + fields: [ + { name: "a", type: { kind: "array", element: { kind: "struct", fields: [{ name: "s", type: { kind: "string" } }] } } }, + { name: "b", type: { kind: "array", element: { kind: "struct", fields: [{ name: "t", type: { kind: "string" } }] } } }, + ], + }, + profile: { version: 1, shared: { columns: [{ leaf: 1, dict: ["only-second"] }] } }, + }, +]; + const LAYOUTS: PlanLayout[] = ["row", "columnar"]; const out = LAYOUTS.flatMap((layout) => CASES.map(({ name, ir }) => { @@ -42,6 +61,11 @@ const out = LAYOUTS.flatMap((layout) => }), ); +for (const c of PROFILED) { + const canonical = serializeArtifact(c.ir, "columnar", c.profile); + out.push({ name: c.name, plan: "columnar", ir: c.ir, profile: c.profile, canonical, fingerprint: toHex(fingerprintOf(canonical)) } as never); +} + await Bun.write( new URL("./fingerprints.json", import.meta.url).pathname, JSON.stringify({ description: "IR → canonical artifact text → fingerprint (first 16 bytes of SHA-256, hex). Locks spec §5.", cases: out }, null, 2) + "\n",