Packed string columns + nested flattening (columnar@2)

apps/bench/README.md

@@ -14,7 +14,7 @@ cd apps/bench && bun run bench
 |---|---|---|
 | candles | numeric OHLCV rows, f64-heavy | weak under plan `row`; the flagship under plan `columnar` (delta timestamps, scaled-decimal prices) |
 | devices | enums, bounded ints, booleans | favorable for schema-only encoding under either plan |
-| feed | prose bodies, ids, names | honest loss case — Brotli eats text, schemas don't; the nested author struct keeps the posts array on the row path even under `columnar` |
+| feed | prose bodies, ids, names | thinnest margin — text columns deflate inside the codec (packed string mode), nested authors flatten into leaf columns |
 
 ## Fairness rules
 

apps/web/app/benchmark.tsx

@@ -26,7 +26,7 @@ const PAYLOADS: Payload[] = [
       { label: "JSON + Brotli — edge q4", bytes: 21315, kind: "generic" },
       { label: "Protobuf", bytes: 56996, kind: "binary" },
       { label: "Hyperfly · columnar", bytes: 12628, kind: "hyperfly" },
-      { label: "Hyperfly + Brotli", bytes: 7917, kind: "profile" },
+      { label: "Hyperfly + Brotli", bytes: 7915, kind: "profile" },
     ],
     note: "Columns ride separately: timestamps become deltas, and prices that are exact decimals travel as integer mantissas instead of eight raw bytes. No entropy coder is involved yet — layout alone, uncompressed, undercuts what the edge actually serves. It clears Brotli's offline q11 ceiling too; the harness in the repo has the receipts.",
   },
@@ -38,10 +38,10 @@ const PAYLOADS: Payload[] = [
       { label: "JSON + gzip", bytes: 17026, kind: "generic" },
       { label: "JSON + Brotli — edge q4", bytes: 16823, kind: "generic" },
       { label: "Protobuf", bytes: 29056, kind: "binary" },
-      { label: "Hyperfly · columnar", bytes: 17981, kind: "hyperfly" },
-      { label: "Hyperfly + Brotli", bytes: 9446, kind: "profile" },
+      { label: "Hyperfly · columnar", bytes: 11945, kind: "hyperfly" },
+      { label: "Hyperfly + Brotli", bytes: 9726, kind: "profile" },
     ],
-    note: "An enum with six members is an index, not a string. Bounded integers ship as offsets from their declared minimum, booleans pack into bitmaps. Standalone this sits at parity with edge Brotli — the win here comes from stacking, because columnar bytes compress far better than JSON does.",
+    note: "An enum with six members is an index, not a string. Bounded integers ship as offsets from their declared minimum, booleans pack into bitmaps, and repetitive id columns deflate inside the codec — so the uncompressed wire already undercuts what the edge serves.",
   },
   {
     route: "GET /v1/feed",
@@ -51,10 +51,10 @@ const PAYLOADS: Payload[] = [
       { label: "JSON + gzip", bytes: 7775, kind: "generic" },
       { label: "JSON + Brotli — edge q4", bytes: 7691, kind: "generic" },
       { label: "Protobuf", bytes: 15232, kind: "binary" },
-      { label: "Hyperfly", bytes: 14560, kind: "hyperfly" },
-      { label: "Hyperfly + Brotli", bytes: 7225, kind: "profile" },
+      { label: "Hyperfly · columnar", bytes: 6513, kind: "hyperfly" },
+      { label: "Hyperfly + Brotli", bytes: 6443, kind: "profile" },
     ],
-    note: "Structure compresses; prose does not. On mostly-text payloads Hyperfly lands at a wash with edge Brotli, and Brotli's offline ceiling still wins outright. Route-trained dictionaries are what change this picture, and they do not exist yet.",
+    note: "Prose does not vanish under a schema — so text columns pack through deflate inside the codec, with shared context across every row, and unpack bit-exactly. Structure travels as columns around them. The all-text route now lands ahead of Brotli's offline ceiling instead of behind it.",
   },
 ];
 

apps/web/app/page.tsx

@@ -166,9 +166,10 @@ export default function Home() {
               Measured with the TypeScript reference implementation on deterministic synthetic
               corpora — reproduce with `bun run bench` in the repo. The Brotli row is q4, the
               level edges actually run on dynamic responses; q6 (nginx's default) and the q11
-              offline ceiling are in the harness, and columnar clears even q11 on the candles
-              corpus. Protobuf gets proper enums and int64. Results depend entirely on the
-              payload: structure compresses, prose does not, and the feed tab is the honest wash.
+              offline ceiling are in the harness — and uncompressed columnar output clears even
+              q11 on all three corpora. Protobuf gets proper enums and int64. Results still
+              depend on the payload; the feed tab is where the margin is thinnest, and it says
+              why.
             </p>
           </Reveal>
         </section>

packages/hyperfly/src/canonical.ts

@@ -59,8 +59,10 @@ export function serializeNode(node: IRNode): string {
 
 export type PlanLayout = "row" | "columnar";
 
+const PLAN_VERSION: Record<PlanLayout, number> = { row: 1, columnar: 2 };
+
 export function serializeArtifact(ir: IRNode, layout: PlanLayout = "row"): string {
-  return `{"wire":1,"plan":{"layout":"${layout}","version":1},"ir":${serializeNode(ir)}}`;
+  return `{"wire":1,"plan":{"layout":"${layout}","version":${PLAN_VERSION[layout]}},"ir":${serializeNode(ir)}}`;
 }
 
 export function fingerprintOf(artifact: string): Uint8Array {

packages/hyperfly/src/codec.ts

@@ -1,4 +1,5 @@
 import { fingerprintOf, serializeArtifact, toHex, type PlanLayout } from "./canonical.js";
+import { defaultPackHooks } from "./pack.js";
 import { decodeNode } from "./decode.js";
 import { encodeNode } from "./encode.js";
 import { DecodeError, FingerprintMismatchError } from "./errors.js";
@@ -10,9 +11,15 @@ export const MAGIC = new Uint8Array([0x68, 0x66]);
 export const WIRE_VERSION = 1;
 export const HEADER_SIZE = 19;
 
+export interface PackHooks {
+  deflate?: (data: Uint8Array) => Uint8Array;
+  inflate?: (data: Uint8Array, maxOutputLength: number) => Uint8Array;
+}
+
 export interface CompileOptions {
   limits?: Partial<DecodeLimits>;
   plan?: PlanLayout;
+  pack?: PackHooks | false;
 }
 
 export interface Codec<T = unknown> {
@@ -34,16 +41,17 @@ export function compileIR<T = unknown>(ir: IRNode, options: CompileOptions = {})
   const fingerprint = toHex(fingerprintBytes);
   const limits: DecodeLimits = { ...DEFAULT_LIMITS, ...options.limits };
   const columnar = plan === "columnar";
+  const pack = options.pack === false ? {} : (options.pack ?? defaultPackHooks());
 
   const encodeBody = (value: T): Uint8Array => {
     const w = new Writer();
-    encodeNode(w, ir, value, "$", 0, { maxDepth: limits.maxDepth, columnar });
+    encodeNode(w, ir, value, "$", 0, { maxDepth: limits.maxDepth, columnar, deflate: pack.deflate });
     return w.finish();
   };
 
   const decodeBody = (bytes: Uint8Array): T => {
     const r = new Reader(bytes, limits);
-    const value = decodeNode(r, ir, "$", 0, columnar);
+    const value = decodeNode(r, ir, "$", 0, columnar, pack.inflate);
     r.expectEnd();
     return value as T;
   };