Python implementation + pydantic adapter

.github/workflows/ci.yml

@@ -27,3 +27,30 @@ jobs:
       - run: bun run check-types
 
       - run: bun run test
+
+  python:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - run: pip install ./python[test]
+
+      - run: pytest python/tests -q
+
+  rust:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - run: cargo test --manifest-path rust/Cargo.toml

.gitignore

@@ -23,3 +23,12 @@ next-env.d.ts
 .DS_Store
 *.pem
 npm-debug.log*
+
+# python
+__pycache__/
+*.pyc
+*.egg-info/
+.pytest_cache/
+
+# rust
+rust/target/

apps/bench/README.md

@@ -44,6 +44,13 @@ cd apps/bench && bun run bench
   the wire), which is the honest comparison: schema-aware is precisely the
   thing being measured against.
 - Every binary contender gets the same +br4 stacking option hyperfly gets.
+- Two known minor biases, disclosed rather than hidden: the route discriminator
+  is a schema literal Hyperfly erases to zero bytes but Protobuf sends as a
+  field (≈6–9 B/message in Hyperfly's favour), and the reported "codec compile"
+  time covers both Hyperfly plan compilations while Protobuf's `Root.fromJSON`
+  runs during suite construction outside the timer. Neither moves the byte
+  ranking; both are why these numbers stay private until a fresh-process,
+  matched-setup harness replaces them.
 
 ## What these numbers are NOT
 

packages/hyperfly/src/codec.ts

@@ -33,8 +33,19 @@ export interface Codec<T = unknown> {
   decodeBody(bytes: Uint8Array): T;
 }
 
+function deepFreeze<T>(value: T): T {
+  if (value && typeof value === "object") {
+    for (const inner of Object.values(value)) deepFreeze(inner);
+    Object.freeze(value);
+  }
+  return value;
+}
+
 export function compileIR<T = unknown>(ir: IRNode, options: CompileOptions = {}): Codec<T> {
   validateIR(ir);
+  // isolate from later mutation, caller-side or through codec.ir: the fingerprint is
+  // fixed at compile time and the schema behind it must not drift
+  ir = deepFreeze(structuredClone(ir));
   const plan: PlanLayout = options.plan ?? "row";
   const artifact = serializeArtifact(ir, plan);
   const fingerprintBytes = fingerprintOf(artifact);
@@ -45,7 +56,14 @@ export function compileIR<T = unknown>(ir: IRNode, options: CompileOptions = {})
 
   const encodeBody = (value: T): Uint8Array => {
     const w = new Writer();
-    encodeNode(w, ir, value, "$", 0, { maxDepth: limits.maxDepth, columnar, deflate: pack.deflate });
+    encodeNode(w, ir, value, "$", 0, {
+      maxDepth: limits.maxDepth,
+      maxItems: limits.maxItems,
+      maxByteLength: limits.maxByteLength,
+      columnar,
+      deflate: pack.deflate,
+      canInflate: pack.inflate !== undefined,
+    });
     return w.finish();
   };
 
@@ -56,7 +74,7 @@ export function compileIR<T = unknown>(ir: IRNode, options: CompileOptions = {})
     return value as T;
   };
 
-  return {
+  return Object.freeze({
     ir,
     artifact,
     fingerprint,
@@ -66,19 +84,20 @@ export function compileIR<T = unknown>(ir: IRNode, options: CompileOptions = {})
     encode(value: T): Uint8Array {
       const body = encodeBody(value);
       const out = new Uint8Array(HEADER_SIZE + body.length);
-      out.set(MAGIC, 0);
+      out[0] = 0x68;
+      out[1] = 0x66;
       out[2] = WIRE_VERSION;
       out.set(fingerprintBytes, 3);
       out.set(body, HEADER_SIZE);
       return out;
     },
     decode(bytes: Uint8Array): T {
       if (bytes.length < HEADER_SIZE) throw new DecodeError("header", "shorter than envelope header");
-      if (bytes[0] !== MAGIC[0] || bytes[1] !== MAGIC[1]) throw new DecodeError("header", "bad magic");
+      if (bytes[0] !== 0x68 || bytes[1] !== 0x66) throw new DecodeError("header", "bad magic");
       if (bytes[2] !== WIRE_VERSION) throw new DecodeError("header", `unsupported wire major ${bytes[2]}`);
       const actual = toHex(bytes.subarray(3, HEADER_SIZE));
       if (actual !== fingerprint) throw new FingerprintMismatchError(fingerprint, actual);
       return decodeBody(bytes.subarray(HEADER_SIZE));
     },
-  };
+  });
 }

packages/hyperfly/src/columnar.ts

@@ -1,8 +1,7 @@
-import { decodeNode, type Inflate } from "./decode.js";
+import { boundByInput, decodeNode, readBitmap, type Inflate } from "./decode.js";
 import { encodeNode, typeAcceptsNull, utf8Bytes, writeBitmap, type EncodeCtx } from "./encode.js";
 import { DecodeError, EncodeError } from "./errors.js";
 import type { IRField, IRNode } from "./ir.js";
-import { readBitmap } from "./decode.js";
 import type { Reader } from "./reader.js";
 import { INT_MAX, INT_MIN, readUleb, ulebLen, unzigzag, writeUleb, zigzag } from "./varint.js";
 import type { Writer } from "./writer.js";
@@ -102,7 +101,10 @@ function decodeIntColumn(r: Reader, node: IntNode, count: number, path: string):
   const mode = r.u8();
   if (mode > 1) throw new DecodeError("marker", `${path}: invalid int column mode 0x${mode.toString(16)}`);
   const out: number[] = new Array<number>(count);
-  if (count === 0) return out;
+  if (count === 0) {
+    if (mode !== 0) throw new DecodeError("marker", `${path}: empty column must use mode 0x00`);
+    return out;
+  }
 
   const fromForm = (form: bigint): bigint =>
     node.min !== undefined ? form + BigInt(node.min) : unzigzag(form);
@@ -147,13 +149,20 @@ function sigBytes(x: bigint): number {
 const POW10 = [1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000];
 const MAX_SCALE = POW10.length - 1;
 
+/** Spec-pinned mantissa recovery: sign(v) * floor(|v|*10^s + 0.5), pure IEEE ops. */
+function decimalMantissa(v: number, pow: number): number {
+  if (v > 0) return Math.floor(v * pow + 0.5);
+  if (v < 0) return -Math.floor(-v * pow + 0.5);
+  return 0;
+}
+
 /** Smallest s with every value exactly m/10^s for a safe integer m, or null. */
 function decimalScale(values: number[]): number | null {
   for (let s = 0; s <= MAX_SCALE; s++) {
     const pow = POW10[s]!;
     let ok = true;
     for (const v of values) {
-      const m = Math.round(v * pow);
+      const m = decimalMantissa(v, pow);
       if (!Number.isSafeInteger(m) || m / pow !== v) {
         ok = false;
         break;
@@ -190,7 +199,7 @@ function encodeFloatColumn(w: Writer, values: number[], path: string): void {
   let scaledRawCost = Infinity;
   let mantissas: bigint[] = [];
   if (scale !== null) {
-    mantissas = canon.map((v) => BigInt(Math.round(v * POW10[scale]!)));
+    mantissas = canon.map((v) => BigInt(decimalMantissa(v, POW10[scale]!)));
     scaledRawCost = 1 + mantissas.reduce((n, m) => n + ulebLen(zigzag(m)), 0);
     scaledDeltaCost = 1 + ulebLen(zigzag(mantissas[0]!));
     for (let i = 1; i < mantissas.length; i++) {
@@ -232,7 +241,10 @@ function decodeFloatColumn(r: Reader, count: number, path: string): number[] {
   const mode = r.u8();
   if (mode > 3) throw new DecodeError("marker", `${path}: invalid float column mode 0x${mode.toString(16)}`);
   const out: number[] = new Array<number>(count);
-  if (count === 0) return out;
+  if (count === 0) {
+    if (mode !== 0) throw new DecodeError("marker", `${path}: empty column must use mode 0x00`);
+    return out;
+  }
 
   if (mode >= 2) {
     const scale = r.u8();
@@ -294,7 +306,7 @@ function encodeStringColumn(w: Writer, values: unknown[], path: string, ctx: Enc
 
   let packed: Uint8Array | null = null;
   let packedCost = Infinity;
-  if (ctx.deflate) {
+  if (ctx.deflate && ctx.canInflate) {
     const total = bytes.reduce((n, b) => n + b.length, 0);
     const concat = new Uint8Array(total);
     let offset = 0;
@@ -323,11 +335,12 @@ function encodeStringColumn(w: Writer, values: unknown[], path: string, ctx: Enc
   }
 }
 
-const utf8Strict = new TextDecoder("utf-8", { fatal: true });
+const utf8Strict = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
 
 function decodeStringColumn(r: Reader, count: number, path: string, inflate?: Inflate): string[] {
   const mode = r.u8();
   if (mode > 1) throw new DecodeError("marker", `${path}: invalid string column mode 0x${mode.toString(16)}`);
+  if (count === 0 && mode !== 0) throw new DecodeError("marker", `${path}: empty column must use mode 0x00`);
   const out: string[] = new Array<string>(count);
   if (count === 0) return out;
 
@@ -431,7 +444,8 @@ export function encodeColumnarArray(
   const containerOf = (row: Record<string, unknown>, segs: readonly string[], i: number): Record<string, unknown> => {
     let obj: Record<string, unknown> = row;
     for (let d = 0; d < segs.length - 1; d++) {
-      const v = obj[segs[d]!];
+      const key = segs[d]!;
+      const v = Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : undefined;
       if (typeof v !== "object" || v === null || Array.isArray(v)) {
         throw new EncodeError(
           v === undefined ? "required" : "type",
@@ -448,7 +462,10 @@ export function encodeColumnarArray(
     const leafName = leaf.segs[leaf.segs.length - 1]!;
     const dotted = leaf.segs.join(".");
     const fieldPath = `${path}[].${dotted}`;
-    const values: unknown[] = rows.map((row, i) => containerOf(row, leaf.segs, i)[leafName]);
+    const values: unknown[] = rows.map((row, i) => {
+      const holder = containerOf(row, leaf.segs, i);
+      return Object.prototype.hasOwnProperty.call(holder, leafName) ? holder[leafName] : undefined;
+    });
     const states: RowState[] = values.map((v, i) => {
       const absent = v === undefined;
       if (absent && !field.optional) {
@@ -471,6 +488,15 @@ export function encodeColumnarArray(
       participating.push(values[i]);
     }
 
+    // row-equivalent depths: a nested container at chain position j sits at depth+2+j,
+    // the leaf value at depth+1+segs.length. Containers always exist; the leaf only when present.
+    if (rows.length > 0 && depth + leaf.segs.length > ctx.maxDepth) {
+      throw new EncodeError("depth", `${fieldPath}: nesting deeper than ${ctx.maxDepth}`);
+    }
+    if (participating.length > 0 && depth + 1 + leaf.segs.length > ctx.maxDepth) {
+      throw new EncodeError("depth", `${fieldPath}: nesting deeper than ${ctx.maxDepth}`);
+    }
+
     switch (field.type.kind) {
       case "int":
         encodeIntColumn(
@@ -514,6 +540,10 @@ export function decodeColumnarArray(
     }
     count = Number(raw);
   }
+  if (count > r.limits.maxItems) {
+    throw new DecodeError("limit", `${path}: array count ${count} exceeds limit ${r.limits.maxItems}`);
+  }
+  boundByInput(r, count, element, path);
 
   const out: Record<string, unknown>[] = Array.from({ length: count }, () => ({}));
   const leaves = flattenLeaves(element)!;
@@ -522,7 +552,8 @@ export function decodeColumnarArray(
     let obj = row;
     for (let d = 0; d < segs.length - 1; d++) {
       const seg = segs[d]!;
-      obj = (obj[seg] ??= {}) as Record<string, unknown>;
+      if (!Object.prototype.hasOwnProperty.call(obj, seg)) obj[seg] = {};
+      obj = obj[seg] as Record<string, unknown>;
     }
     return obj;
   };
@@ -531,6 +562,12 @@ export function decodeColumnarArray(
     const field = leaf.field;
     const leafName = leaf.segs[leaf.segs.length - 1]!;
     const fieldPath = `${path}[].${leaf.segs.join(".")}`;
+    // nested structs are required and non-nullable: materialize the container chain at
+    // this leaf's declared position for every row, so an all-absent nested struct still
+    // round-trips and keys stay in declared order across implementations
+    if (leaf.segs.length > 1) {
+      for (let i = 0; i < count; i++) containerOf(out[i]!, leaf.segs);
+    }
     const presence = field.optional ? readBitmap(r, count, fieldPath) : null;
     const nulls = field.nullable ? readBitmap(r, count, fieldPath) : null;
 
@@ -549,6 +586,13 @@ export function decodeColumnarArray(
       slots.push(i);
     }
 
+    if (count > 0 && depth + leaf.segs.length > r.limits.maxDepth) {
+      throw new DecodeError("depth", `${fieldPath}: nesting deeper than ${r.limits.maxDepth}`);
+    }
+    if (slots.length > 0 && depth + 1 + leaf.segs.length > r.limits.maxDepth) {
+      throw new DecodeError("depth", `${fieldPath}: nesting deeper than ${r.limits.maxDepth}`);
+    }
+
     switch (field.type.kind) {
       case "int": {
         const values = decodeIntColumn(r, field.type as IntNode, slots.length, fieldPath);

packages/hyperfly/src/decode.ts

@@ -1,10 +1,10 @@
 import { columnarEligible, decodeColumnarArray } from "./columnar.js";
 import { DecodeError } from "./errors.js";
-import type { IRNode } from "./ir.js";
+import { hasPayload, type IRNode } from "./ir.js";
 import type { Reader } from "./reader.js";
 import { INT_MAX, INT_MIN, readUleb, unzigzag } from "./varint.js";
 
-const utf8 = new TextDecoder("utf-8", { fatal: true });
+const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
 
 function fail(code: "type" | "range" | "utf8" | "float" | "marker" | "bitmap" | "depth" | "limit", path: string, message: string): never {
   throw new DecodeError(code, `${path}: ${message}`);
@@ -45,6 +45,19 @@ export function readBitmap(r: Reader, count: number, path: string): boolean[] {
 
 export type Inflate = (data: Uint8Array, maxOutputLength: number) => Uint8Array;
 
+/**
+ * A declared count must be payable by the bytes still on the wire: every element that
+ * carries any payload costs at least one bit, so a truncated body can never make a
+ * decoder allocate for millions of rows it will never read.
+ */
+export function boundByInput(r: Reader, count: number, element: IRNode, path: string): void {
+  if (count === 0 || !hasPayload(element)) return;
+  const affordable = r.remaining() * 8;
+  if (count > affordable) {
+    throw new DecodeError("limit", `${path}: declared ${count} items but only ${r.remaining()} byte(s) remain`);
+  }
+}
+
 export function decodeNode(
   r: Reader,
   node: IRNode,
@@ -103,6 +116,10 @@ export function decodeNode(
         return decodeColumnarArray(r, node, path, depth, inflate);
       }
       const count = node.length ?? readCount(r, r.limits.maxItems, "array count", path);
+      if (count > r.limits.maxItems) {
+        fail("limit", path, `array count ${count} exceeds limit ${r.limits.maxItems}`);
+      }
+      boundByInput(r, count, node.element, path);
       const out = new Array<unknown>(count);
       for (let i = 0; i < count; i++) out[i] = decodeNode(r, node.element, `${path}[${i}]`, depth + 1, columnar, inflate);
       return out;
@@ -114,6 +131,8 @@ export function decodeNode(
       const nulls = readBitmap(r, nullableCount, path);
       let pi = 0;
       let ni = 0;
+      // a field may legitimately be named constructor/toString/valueOf; assigning those on a
+      // prototypeful object would hit inherited accessors instead of creating own properties
       const out: Record<string, unknown> = {};
       for (const field of node.fields) {
         const present = field.optional ? presence[pi++]! : true;