Profiles v1: trained string dictionaries (columnar@3)

apps/bench/src/profiles.ts

@@ -0,0 +1,65 @@
+import { brotliCompressSync, constants } from "node:zlib";
+import { compile, toIR } from "hyperfly/zod";
+import { train } from "hyperfly";
+import { DeviceResponse } from "./corpora/devices.js";
+import { FeedResponse } from "./corpora/feed.js";
+import { devicesTraffic, feedTraffic } from "./traffic.js";
+
+const enc = new TextEncoder();
+const br = (bytes: Uint8Array, mode: number) =>
+  brotliCompressSync(bytes, {
+    params: {
+      [constants.BROTLI_PARAM_QUALITY]: 4,
+      [constants.BROTLI_PARAM_MODE]: mode,
+      [constants.BROTLI_PARAM_SIZE_HINT]: bytes.length,
+    },
+  }).length;
+
+/**
+ * Profiles are trained on one slice of sampled traffic and measured on a held-out
+ * slice, so what is reported is generalization rather than memorization.
+ */
+export function runProfileSuite(): void {
+  const suites = [
+    { name: "devices", schema: DeviceResponse, traffic: devicesTraffic(60, 50, 0xd7) },
+    { name: "feed", schema: FeedResponse, traffic: feedTraffic(60, 12, 0xf7) },
+  ] as const;
+
+  console.log("\nprofiles — trained on 80% of sampled responses, measured on the held-out 20%");
+  console.log("  corpus    json      json+br4  columnar  col+br4   profiled  prof+br4  dict");
+
+  for (const suite of suites) {
+    const ir = toIR(suite.schema as never);
+    const profile = train(ir, suite.traffic.train);
+    const columnar = compile(suite.schema as never, { plan: "columnar" });
+    const profiled = compile(suite.schema as never, { plan: "columnar", profile });
+
+    let json = 0;
+    let jsonBr = 0;
+    let col = 0;
+    let colBr = 0;
+    let prof = 0;
+    let profBr = 0;
+
+    for (const response of suite.traffic.holdout) {
+      const j = enc.encode(JSON.stringify(response));
+      json += j.length;
+      jsonBr += br(j, constants.BROTLI_MODE_TEXT);
+      const c = columnar.encode(response as never);
+      col += c.length;
+      colBr += br(c, constants.BROTLI_MODE_GENERIC);
+      const p = profiled.encode(response as never);
+      prof += p.length;
+      profBr += br(p, constants.BROTLI_MODE_GENERIC);
+      if (!Bun.deepEquals(profiled.decode(p), response, false)) {
+        throw new Error(`${suite.name}: profiled round-trip mismatch`);
+      }
+    }
+
+    const entries = profile?.shared.columns.reduce((n, c) => n + c.dict.length, 0) ?? 0;
+    const cell = (v: number) => `${String(v).padStart(7)} `;
+    console.log(
+      `  ${suite.name.padEnd(9)} ${cell(json)} ${cell(jsonBr)} ${cell(col)} ${cell(colBr)} ${cell(prof)} ${cell(profBr)} ${entries}`,
+    );
+  }
+}

apps/bench/src/run.ts

@@ -2,6 +2,7 @@ import { brotliCompressSync, brotliDecompressSync, constants, gzipSync, gunzipSy
 import { decode as cborDecode, encode as cborEncode } from "cbor-x";
 import { compile } from "hyperfly/zod";
 import { pack, unpack } from "msgpackr";
+import { runProfileSuite } from "./profiles.js";
 import { candlesProto, devicesProto, feedProto, type ProtoCodec } from "./proto.js";
 import { CandleResponse, candlesPayload } from "./corpora/candles.js";
 import { DeviceResponse, devicesPayload } from "./corpora/devices.js";
@@ -207,6 +208,8 @@ for (const suite of SUITES) {
   all.push(...runCorpus(suite.name, suite.schema, suite.payload, suite.proto, suite.warmup, suite.samples));
 }
 
+runProfileSuite();
+
 await Bun.write(
   new URL("../results/results.json", import.meta.url).pathname,
   JSON.stringify(

apps/bench/src/traffic.ts

@@ -0,0 +1,111 @@
+import { intIn, mulberry32, pick } from "./prng.js";
+import { DeviceResponse } from "./corpora/devices.js";
+import { FeedResponse } from "./corpora/feed.js";
+import type { z } from "zod";
+
+/**
+ * Sampled traffic for one route: many responses drawn from a stable universe, the
+ * way a real endpoint behaves. Profiles are trained on one slice and measured on a
+ * held-out slice, so the numbers reflect generalization rather than memorization.
+ */
+export interface Traffic<T> {
+  train: T[];
+  holdout: T[];
+}
+
+const REGIONS = ["us-east", "us-west", "eu-central", "eu-west", "ap-south", "ap-northeast", "sa-east", "af-south"] as const;
+const STATUSES = ["online", "online", "online", "online", "offline", "degraded", "provisioning", "unknown"] as const;
+const TAGS = ["fleet-a", "fleet-b", "fleet-a", "pilot", "lab", null, null, null];
+
+/** A fixed device fleet: the same ids and tags recur across every response. */
+export function devicesTraffic(
+  responses: number,
+  perResponse: number,
+  seed: number,
+): Traffic<z.output<typeof DeviceResponse>> {
+  const rng = mulberry32(seed);
+  const fleet = Array.from({ length: 400 }, (_, i) => ({
+    id: `dev-${String(intIn(rng, 0, 99)).padStart(2, "0")}-${String(i).padStart(5, "0")}`,
+    region: pick(rng, REGIONS),
+    tag: pick(rng, TAGS),
+    firmwareMajor: intIn(rng, 1, 4),
+    firmwareMinor: intIn(rng, 0, 27),
+  }));
+
+  const base = 1754000000000;
+  const all = Array.from({ length: responses }, (_, r) => ({
+    route: "devices" as const,
+    page: 0,
+    devices: Array.from({ length: perResponse }, () => {
+      const unit = fleet[intIn(rng, 0, fleet.length - 1)]!;
+      return {
+        ...unit,
+        status: pick(rng, STATUSES),
+        battery: intIn(rng, 0, 100),
+        rssi: intIn(rng, -120, 0),
+        uptimeSec: intIn(rng, 0, 40000000),
+        tempC: Math.round((15 + rng() * 45) * 10) / 10,
+        alarms: rng() < 0.85 ? 0 : intIn(rng, 1, 12),
+        shadowSynced: rng() < 0.93,
+        lastSeen: base + r * 60000 - intIn(rng, 0, 86400000),
+      };
+    }),
+  }));
+
+  const split = Math.floor(all.length * 0.8);
+  return { train: all.slice(0, split), holdout: all.slice(split) };
+}
+
+const HANDLES = Array.from({ length: 120 }, (_, i) => {
+  const first = ["ada", "linus", "grace", "alan", "edsger", "barbara", "donald", "radia", "ken", "margaret"][i % 10]!;
+  const last = ["hopper", "torvalds", "lovelace", "turing", "dijkstra", "liskov", "knuth", "perlman", "thompson", "hamilton"][
+    Math.floor(i / 10) % 10
+  ]!;
+  return { handle: `@${first}${last}${i}`, name: `${first[0]!.toUpperCase()}${first.slice(1)} ${last[0]!.toUpperCase()}${last.slice(1)}` };
+});
+
+const LEXICON = (
+  "the of and to in is that it was for on are as with his they at be this have from or had by hot word " +
+  "but what some we can out other were all there when up use your how said an each she which do their time " +
+  "deploy latency cluster rollout incident postmortem throughput regression release migration schema payload"
+).split(" ");
+
+/** A recurring cast of authors posting about a recurring vocabulary. */
+export function feedTraffic(
+  responses: number,
+  perResponse: number,
+  seed: number,
+): Traffic<z.output<typeof FeedResponse>> {
+  const rng = mulberry32(seed);
+  const base = 1754500000000;
+  const hex = (n: number) => Array.from({ length: n }, () => Math.floor(rng() * 16).toString(16)).join("");
+  const authors = HANDLES.map((h) => ({ ...h, id: hex(12), verified: rng() < 0.2 }));
+
+  const all = Array.from({ length: responses }, () => ({
+    route: "feed" as const,
+    posts: Array.from({ length: perResponse }, () => {
+      const author = authors[intIn(rng, 0, authors.length - 1)]!;
+      const sentences = Array.from({ length: intIn(rng, 1, 3) }, () => {
+        const words = Array.from({ length: intIn(rng, 8, 24) }, () => pick(rng, LEXICON));
+        const s = words.join(" ");
+        return s.charAt(0).toUpperCase() + s.slice(1) + ".";
+      });
+      return {
+        id: hex(16),
+        author,
+        body: sentences.join(" "),
+        lang: pick(rng, ["en", "en", "en", "de", "fr", "es", "ja"] as const),
+        likes: intIn(rng, 0, 50000),
+        replies: intIn(rng, 0, 2000),
+        reposts: intIn(rng, 0, 8000),
+        createdAt: base - intIn(rng, 0, 604800000),
+        inReplyTo: null,
+      };
+    }),
+  }));
+
+  const split = Math.floor(all.length * 0.8);
+  return { train: all.slice(0, split), holdout: all.slice(split) };
+}
+
+export { DeviceResponse, FeedResponse };

packages/hyperfly/src/canonical.ts

@@ -1,4 +1,5 @@
 import type { IRField, IRNode, LiteralValue } from "./ir.js";
+import type { SharedProfile } from "./profile.js";
 import { sha256 } from "./sha256.js";
 
 /** Spec §5: not generic JSON canonicalization — key order and escaping are fixed here. */
@@ -59,10 +60,22 @@ export function serializeNode(node: IRNode): string {
 
 export type PlanLayout = "row" | "columnar";
 
-const PLAN_VERSION: Record<PlanLayout, number> = { row: 1, columnar: 2 };
+const PLAN_VERSION: Record<PlanLayout, number> = { row: 1, columnar: 3 };
 
-export function serializeArtifact(ir: IRNode, layout: PlanLayout = "row"): string {
-  return `{"wire":1,"plan":{"layout":"${layout}","version":${PLAN_VERSION[layout]}},"ir":${serializeNode(ir)}}`;
+export function serializeShared(shared: SharedProfile): string {
+  const columns = shared.columns.map(
+    (c) => `{"leaf":${c.leaf},"dict":[${c.dict.map(escapeString).join(",")}]}`,
+  );
+  return `{"columns":[${columns.join(",")}]}`;
+}
+
+export function serializeArtifact(
+  ir: IRNode,
+  layout: PlanLayout = "row",
+  profile?: { shared: SharedProfile },
+): string {
+  const head = `{"wire":1,"plan":{"layout":"${layout}","version":${PLAN_VERSION[layout]}},"ir":${serializeNode(ir)}`;
+  return profile ? `${head},"profile":${serializeShared(profile.shared)}}` : `${head}}`;
 }
 
 export function fingerprintOf(artifact: string): Uint8Array {

packages/hyperfly/src/codec.ts

@@ -1,8 +1,9 @@
 import { fingerprintOf, serializeArtifact, toHex, type PlanLayout } from "./canonical.js";
 import { defaultPackHooks } from "./pack.js";
+import { indexProfile, validateProfile, type Profile } from "./profile.js";
 import { decodeNode } from "./decode.js";
 import { encodeNode } from "./encode.js";
-import { DecodeError, FingerprintMismatchError } from "./errors.js";
+import { DecodeError, FingerprintMismatchError, HyperflyError } from "./errors.js";
 import { validateIR, type IRNode } from "./ir.js";
 import { DEFAULT_LIMITS, Reader, type DecodeLimits } from "./reader.js";
 import { Writer } from "./writer.js";
@@ -20,13 +21,15 @@ export interface CompileOptions {
   limits?: Partial<DecodeLimits>;
   plan?: PlanLayout;
   pack?: PackHooks | false;
+  profile?: Profile;
 }
 
 export interface Codec<T = unknown> {
   readonly ir: IRNode;
   readonly artifact: string;
   readonly fingerprint: string;
   readonly plan: PlanLayout;
+  readonly profile?: Profile;
   encode(value: T): Uint8Array;
   decode(bytes: Uint8Array): T;
   encodeBody(value: T): Uint8Array;
@@ -47,7 +50,17 @@ export function compileIR<T = unknown>(ir: IRNode, options: CompileOptions = {})
   // fixed at compile time and the schema behind it must not drift
   ir = deepFreeze(structuredClone(ir));
   const plan: PlanLayout = options.plan ?? "row";
-  const artifact = serializeArtifact(ir, plan);
+  const profile = options.profile;
+  if (profile) {
+    if (plan !== "columnar") {
+      throw new HyperflyError("ir", "profiles apply to the columnar plan only");
+    }
+    validateProfile(ir, profile);
+  }
+  // the profile is fixed by the fingerprint exactly as the IR is
+  const frozenProfile = profile ? deepFreeze(structuredClone(profile)) : undefined;
+  const profileIndex = indexProfile(frozenProfile);
+  const artifact = serializeArtifact(ir, plan, frozenProfile);
   const fingerprintBytes = fingerprintOf(artifact);
   const fingerprint = toHex(fingerprintBytes);
   const limits: DecodeLimits = { ...DEFAULT_LIMITS, ...options.limits };
@@ -63,13 +76,14 @@ export function compileIR<T = unknown>(ir: IRNode, options: CompileOptions = {})
       columnar,
       deflate: pack.deflate,
       canInflate: pack.inflate !== undefined,
+      profile: profileIndex,
     });
     return w.finish();
   };
 
   const decodeBody = (bytes: Uint8Array): T => {
     const r = new Reader(bytes, limits);
-    const value = decodeNode(r, ir, "$", 0, columnar, pack.inflate);
+    const value = decodeNode(r, ir, "$", 0, columnar, pack.inflate, profileIndex);
     r.expectEnd();
     return value as T;
   };
@@ -79,6 +93,7 @@ export function compileIR<T = unknown>(ir: IRNode, options: CompileOptions = {})
     artifact,
     fingerprint,
     plan,
+    profile: frozenProfile,
     encodeBody,
     decodeBody,
     encode(value: T): Uint8Array {