From 53d07c21f13d4a0dd48653179f26f99a2145e9b7 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 2 Jul 2026 05:50:20 +0000 Subject: [PATCH 01/14] Replace wasm-bindgen wasm layer with napi-rs wasm (kimchi-napi) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'wasm' backend is now the wasm32-wasip1-threads build of the same kimchi-napi crate that powers the native backend, loaded via generated @napi-rs/wasm-runtime loaders. Native and wasm share one TS conversion layer and one JSOO FFI handoff (globalThis.__o1js_kimchi_ffi). Removed: worker-spec.js pointer marshalling, worker-helpers.js, fix-wasm-bindings-node.js, the WebAssembly.Memory Proxy hack, the wbg_rayon worker bootstrapping, and the wasm-bindgen conversion layer (conversion-{core,proof,oracles,verifier-index}.ts, srs.ts). - node-backend.js: thin loader for kimchi_napi.wasi.cjs; sets RAYON_NUM_THREADS before instantiation; noop thread pool - web-backend.js: Option A — napi-wasm browser loader on the main thread (single-threaded rayon until worker-hosted Option B lands, see PLAN.md/STATE.md) - crypto/bindings.ts: single napi conversion bundle, cached per module - types now sourced from the generated kimchi_napi.wasi.d.cts - build:wasm:* scripts rebuilt around napi build; JSOO postprocessing no longer rewrites requires - @napi-rs/wasm-runtime added as a runtime dependency - deleted bindings.unit-test.ts (its Rust reference fns were wasm-bindgen-only exports); gate-vector-napi.unit-test now also runs against the wasm build Validated: wasm+native builds from one crate produce byte-identical circuit digests, poseidon output and SRS points; rayon threading works under Node; tsc clean. Requires a JSOO artifact rebuild in CI — previously compiled artifacts reference the removed kimchi_wasm.cjs. See PLAN.md (design + implementation report), STATE.md (current status + web options), AGENT_LOG.md (migration gotchas). Co-Authored-By: Claude Fable 5 --- AGENT.md | 12 +- AGENT_LOG.md | 143 ++++++++ PLAN.md | 331 ++++++++++++++++++ STATE.md | 94 +++++ package-lock.json | 20 +- package.json | 1 + scripts/build/jsoo/build-node.sh | 21 +- scripts/build/wasm/build-kimchi-napi-wasm.sh | 53 +++ scripts/build/wasm/build-node.sh | 50 +-- scripts/build/wasm/build-web.sh | 76 ++-- src/bindings.d.ts | 4 +- src/bindings/README.md | 21 +- src/bindings/crypto/bindings.ts | 75 ++-- .../crypto/bindings/bindings.unit-test.ts | 226 ------------ .../crypto/bindings/conversion-base.ts | 16 +- .../crypto/bindings/conversion-core.ts | 218 ------------ .../crypto/bindings/conversion-oracles.ts | 121 ------- .../crypto/bindings/conversion-proof.ts | 271 -------------- .../bindings/conversion-verifier-index.ts | 291 --------------- .../bindings/gate-vector-napi.unit-test.ts | 4 +- src/bindings/crypto/bindings/kimchi-types.ts | 2 +- src/bindings/crypto/bindings/srs.ts | 284 --------------- .../native/napi-conversion-verifier-index.ts | 42 +-- src/bindings/crypto/native/napi-srs.ts | 6 +- src/bindings/crypto/native/napi-wrappers.ts | 13 +- src/bindings/js/node/native-backend.js | 3 + src/bindings/js/node/node-backend.js | 221 ++---------- src/bindings/js/web/web-backend.js | 242 ++----------- src/bindings/js/web/worker-helpers.js | 136 ------- src/bindings/js/web/worker-spec.js | 169 --------- src/build/build-example.js | 2 +- src/build/build-node.js | 2 +- src/build/build-web.js | 117 +++---- src/build/copy-to-dist.js | 2 +- src/build/fix-wasm-bindings-node.js | 50 --- src/lib/proof-system/prover-keys.ts | 11 +- src/mina | 2 +- 37 files changed, 866 insertions(+), 2486 deletions(-) create mode 100644 PLAN.md create mode 100644 STATE.md create mode 100755 scripts/build/wasm/build-kimchi-napi-wasm.sh delete mode 100644 src/bindings/crypto/bindings/bindings.unit-test.ts delete mode 100644 src/bindings/crypto/bindings/conversion-core.ts delete mode 100644 src/bindings/crypto/bindings/conversion-oracles.ts delete mode 100644 src/bindings/crypto/bindings/conversion-proof.ts delete mode 100644 src/bindings/crypto/bindings/conversion-verifier-index.ts delete mode 100644 src/bindings/crypto/bindings/srs.ts delete mode 100644 src/bindings/js/web/worker-helpers.js delete mode 100644 src/bindings/js/web/worker-spec.js delete mode 100644 src/build/fix-wasm-bindings-node.js diff --git a/AGENT.md b/AGENT.md index 1aadc1c140..ab6596ea5f 100644 --- a/AGENT.md +++ b/AGENT.md @@ -34,7 +34,7 @@ and are proven/verified using the underlying proof system. │ Pasta curves (Pallas / Vesta cycle) │ ├─────────────────────────────────────────────────┤ │ Runtime Target │ -│ WASM (browser/Node via wasm-bindgen) │ +│ native .node or WASM via napi-rs (kimchi-napi) │ └─────────────────────────────────────────────────┘ ``` @@ -75,10 +75,12 @@ critical to understand before making changes: ### WASM boundary -The Rust backend is compiled to WASM via `wasm-bindgen` for use in both browser -and Node.js environments. This boundary is a major source of subtle bugs — see -`AGENT_LOG.md` for historical context on panics, threading issues, and memory -problems. +The Rust backend is the `kimchi-napi` crate (napi-rs), built two ways from the +same source: native `.node` prebuilds per platform, and a +`wasm32-wasip1-threads` WASM build (loaded via `@napi-rs/wasm-runtime`) used as +the portable fallback in Node and in browsers. The old `wasm-bindgen` layer was +removed — see `PLAN.md` for the migration and `AGENT_LOG.md` for historical +context on panics, threading issues, and memory problems. ## Essential Commands diff --git a/AGENT_LOG.md b/AGENT_LOG.md index afd780b7b3..0d320efb84 100644 --- a/AGENT_LOG.md +++ b/AGENT_LOG.md @@ -187,4 +187,147 @@ does not guarantee WASM safety. **Relevant files:** `src/bindings/compiled/`, `src/bindings/native/` +--- + +date: 2026-07-02 agent: claude-fable-5 session: wasm-bindgen→napi-rs-wasm-migration +category: rust-wasm-boundary severity: high tags: [napi-rs, wasm, +wasm32-wasip1-threads, migration, build-system] + +--- + +### napi-rs CLI only emits wasi JS loaders when the napi config declares a wasi target + +**Context:** Migrating the wasm backend from wasm-bindgen (`kimchi_wasm`) to the +napi-rs `kimchi-napi` crate compiled for `wasm32-wasip1-threads`. + +**What happened:** `napi build --target wasm32-wasip1-threads` produced only the +`.wasm` binary and `index.d.ts` — no `kimchi_napi.wasi.cjs`, browser loader, or +worker files. Also, raw `cargo check --target wasm32-wasip1-threads` fails with +`EMNAPI_LINK_DIR must be set` (napi-build's wasi branch) — the env var is set by +the napi CLI, so always build through `napi build`, not raw cargo. + +**Root cause:** The CLI's `writeWasiBinding` only runs when the napi config +(`package.json` → `napi.targets`) includes a wasi target, and `--platform` must +be passed. Without `napi.packageName`, the generated loader falls back to +requiring the literal package `undefined-wasm32-wasi`. + +**Resolution/Workaround:** In +`src/mina/src/lib/crypto/kimchi_bindings/js/native/package.json`, set `name`, +`napi.packageName`, and `napi.targets` (including `wasm32-wasip1-threads`), and +build with `napi build --platform`. Memory limits are configured via +`napi.wasm.initialMemory/maximumMemory` (in 64KiB pages). + +**Key takeaway:** For napi-rs wasm builds, the JS loaders are driven by the napi +config, not by the build target alone — declare the wasi target in +`napi.targets` and pass `--platform`. + +**Relevant files:** +`src/mina/src/lib/crypto/kimchi_bindings/js/native/package.json`, +`scripts/build/wasm/build-kimchi-napi-wasm.sh` + +--- + +date: 2026-07-02 agent: claude-fable-5 session: wasm-bindgen→napi-rs-wasm-migration +category: build-system severity: medium tags: [rust, stable, cfg_attr, +nightly-feature, wasm32] + +--- + +### Stale `#![cfg_attr(target_arch = "wasm32", feature(...))]` breaks stable wasm builds + +**Context:** First `wasm32-wasip1-threads` build of the proof-systems workspace +on stable Rust 1.92. + +**What happened:** `o1-utils` and `poly-commitment` failed with E0554 +(`#![feature]` may not be used on the stable release channel). Both had +`#![cfg_attr(target_arch = "wasm32", feature(unsigned_is_multiple_of))]` — added +when wasm builds required nightly; the feature has been stable since Rust 1.87. + +**Root cause:** The old wasm-bindgen pipeline built wasm32 on nightly, so the +cfg_attr was harmless. `wasm32-wasip1-threads` is a tier-2 target with prebuilt +std, so stable works — and then the vestigial feature attribute becomes a hard +error (only under `target_arch = "wasm32"`, so native CI never saw it). + +**Resolution/Workaround:** Deleted the cfg_attr lines (proof-systems submodule). + +**Key takeaway:** cfg_attr'd nightly features hide from native CI and only +explode when someone builds that target on stable — grep for +`cfg_attr(target_arch = "wasm32", feature` when moving wasm builds to stable. + +**Relevant files:** `src/mina/src/lib/crypto/proof-systems/utils/src/lib.rs`, +`src/mina/src/lib/crypto/proof-systems/poly-commitment/src/lib.rs` + +--- + +date: 2026-07-02 agent: claude-fable-5 session: wasm-bindgen→napi-rs-wasm-migration +category: native-ffi severity: medium tags: [napi-rs, type-defs, typescript, +header-d.ts] + +--- + +### napi-rs type-def generation is not valid TS out of the box + +**Context:** Making the generated `kimchi-napi` `index.d.ts` the canonical type +source for the TS conversion layer (replacing the wasm-bindgen `.d.ts`). + +**What happened:** Three classes of problems: (1) setters generated with +optional parameters (`set x(v?: T)`) are invalid TS (TS1051); (2) many alias +names used in signatures (`NapiVector`, `NapiPastaFp`, `NapiPlonkVerifierIndex`, +…) are never declared; (3) `#[napi(object)]` types generate `interface`s (plain +JS objects at runtime) while `#[napi]` structs generate classes — TS code that +did `new napi.WasmFpDomain(...)`-style access typechecked against the old +wasm-bindgen typings but was `undefined` at runtime (it was never called, only +passed around). + +**Resolution/Workaround:** (1) post-process the d.ts in +`build-kimchi-napi-wasm.sh` (regex-drop the `?`); (2) declare the missing +aliases in `header-d.ts` (which napi injects into the generated file); +(3) removed the vestigial "classes" plumbing for object-types from +`napi-conversion-verifier-index.ts` / `napi-wrappers.ts`. + +**Key takeaway:** Treat generated napi type-defs as a build input needing a fix +pass, and know the `#[napi(object)]` (plain object) vs `#[napi]` class (has +constructor) distinction — only the latter exist as runtime exports. + +**Relevant files:** +`src/mina/src/lib/crypto/kimchi_bindings/js/native/header-d.ts`, +`scripts/build/wasm/build-kimchi-napi-wasm.sh`, +`src/bindings/crypto/native/napi-wrappers.ts` + +--- + +date: 2026-07-02 agent: claude-fable-5 session: wasm-bindgen→napi-rs-wasm-migration +category: concurrency severity: high tags: [rayon, wasi-threads, browser, +main-thread, atomics] + +--- + +### Browser main threads cannot block — napi-wasm threading needs a worker host for parallel web proving + +**Context:** Replacing the wasm-bindgen web backend (worker-hosted wasm + +`worker-spec.js` pointer marshalling) with the napi-rs `wasm32-wasip1-threads` +build. + +**What happened:** On Node, the napi-wasm build runs rayon-parallel code fine on +the main thread (Node allows blocking; verified `caml_fp_srs_create_parallel` +spawns worker_threads and returns correctly, and `RAYON_NUM_THREADS` is honored +via the WASI env). In browsers, the main thread cannot execute +`memory.atomic.wait32`, so rayon join points called synchronously from the main +thread will trap once a multi-threaded pool exists. + +**Root cause:** JS embedder rule ([[CanBlock]] = false on the main thread); this +is the same constraint the old wasm-bindgen backend solved by hosting the whole +wasm instance in a dedicated worker and spin-waiting via `wait_until_non_zero`. + +**Resolution/Workaround:** Web backend (Option A, this migration) instantiates +the napi-wasm module on the main thread; parallelism on web is limited until +Option B (worker-hosted module + explicit handle-table RPC, designed in PLAN.md) +lands. Node is unaffected. + +**Key takeaway:** Sync FFI + rayon + browser main thread is fundamentally +impossible without a worker host — plan web parallelism as a separate, +deliberate step, and always benchmark web proving before/after backend changes. + +**Relevant files:** `src/bindings/js/web/web-backend.js`, `PLAN.md` + diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000000..8e20e73da4 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,331 @@ +# PLAN: Replace the wasm-bindgen WASM layer with napi-rs WASM + +**Status:** implemented (Node fully validated; web Option A scaffolded — see §6) +· **Branch:** `florian/wasm-napi` · **Author:** Claude (agent session, +2026-07-02) + +## TL;DR — Feasibility verdict + +**Feasible, and most of the hard work already exists in this repo.** o1js +already ships a complete napi-rs backend (`kimchi-napi` crate + +`src/bindings/crypto/native/napi-*.ts` conversion layer + the `kimchi_ffi` JSOO +stub) that mirrors the entire wasm-bindgen API surface. napi-rs v3 compiles the +_same_ `#[napi]` crate to `wasm32-wasip1-threads`, with Node and browser loaders +provided by `@napi-rs/wasm-runtime` (emnapi). So "replace wasm-bindgen with +napi-rs wasm" reduces to: **build `kimchi-napi` for `wasm32-wasip1-threads`, +point the loaders at it, and delete the wasm-bindgen layer.** + +Recommended rollout is staged: **Node first (high confidence, done in this +branch), web second (needs one design decision about main-thread blocking, +scaffolded here).** + +--- + +## 1. Where we are today + +Two parallel backends implement the same `kimchi_ffi` API consumed by the +js_of_ocaml (JSOO) layer: + +``` + OCaml (Pickles/Snarky) ──jsoo──► o1js_node.bc.cjs / o1js_web.bc.js + │ calls `kimchi_ffi.*` + ┌───────────────────────────────────────────────┴──────────────────────────┐ + │ wasm backend (wasm-bindgen) │ native backend (napi-rs) │ + │ kimchi_wasm crate → kimchi_wasm.cjs/_bg.wasm │ kimchi-napi crate │ + │ conversion-{core,proof,oracles,verifier- │ → kimchi_napi.node │ + │ index}.ts + srs.ts │ napi-conversion-*.ts │ + │ node-backend.js: worker_threads + shared │ + napi-srs.ts │ + │ memory Proxy hack + wbg_rayon glue │ native-backend.js: │ + │ web-backend.js: eval'd JSOO + worker-spec.js │ plain require(), noop │ + │ pointer marshalling + u32 spin-wait sync │ thread pool │ + │ fix-wasm-bindings-node.js: patches wasm- │ │ + │ bindgen output post-build │ │ + └────────────────────────────────────────────────┴──────────────────────────┘ +``` + +### What makes the wasm-bindgen side "a mess" + +1. **Post-build patching** — `src/build/fix-wasm-bindings-node.js` rewrites + wasm-bindgen's generated JS (memory injection, thread stack size, two + different code paths depending on wasm-bindgen version). Fragile on every + wasm-bindgen upgrade. +2. **Hand-rolled rayon threading** — `node-backend.js` monkey-patches + `WebAssembly.Memory` with a `Proxy` to share memory with `worker_threads`, + implements the `wbg_rayon` startup dance, worker-ready timeouts, clone-error + diagnostics, etc. +3. **Web worker marshalling** — `web-backend.js` + `worker-spec.js` maintain a + manual registry of every function's signature so wasm-bindgen objects can + cross the worker boundary as raw `__wbg_ptr` u32s, with + `create_zero_u32_ptr`/`wait_until_non_zero` spin-wait synchronization. It + reaches into wasm-bindgen internals (`__destroy_into_raw`, `__wrap`) that are + not a stable API. +4. **Duplicate everything** — two Rust crates (`kimchi_wasm`, `kimchi-napi`), + two TS conversion layers, two JSOO stubs, two build pipelines, + `__kimchi_backend` runtime switching, and a `worker-spec.js` that must stay + in sync with the Rust API by hand. +5. **Unrecoverable rayon panics** in the wasm-bindgen threading model (see + `AGENT_LOG.md`, `rust-wasm-boundary`), while the napi boundary can catch + them. + +### What already exists on the napi side (verified in this session) + +- `proof-systems/kimchi-napi` (in the `src/mina` submodule) exports the **full** + kimchi FFI surface — gate vectors, prover/verifier indexes, proofs + (create/verify/batch), oracles, poseidon, SRS incl. `create_parallel` and + Lagrange-basis APIs, vectors, lookup tables — with `Napi*` classes aliased to + the `Wasm*` names the TS layer expects. +- `src/bindings/crypto/native/napi-conversion-{core,proof,oracles,verifier-index}.ts` + and `napi-srs.ts` form a complete conversion bundle, selected in + `src/bindings/crypto/bindings.ts` via `__kimchi_backend`. +- The JSOO artifact picks its backend at _runtime_: the `kimchi_ffi` stub + (`kimchi_bindings/js/node_js/node_backend.js`) requires either + `@o1js/native-{platform}-{arch}` or `./kimchi_wasm.js`. +- Platform prebuilds ship as `@o1js/native-*` npm packages (napi-rs CLI + conventions, `@napi-rs/cli` ^3.4.1 already a devDependency). +- Note: the Rust `napi`/`napi-derive` deps come from an **o1-labs fork of + napi-rs 3.3.0** (`o1-labs/napi-rs@023d1d4f`). The fork delta needs review + before relying on upstream wasm behavior (see Risks). + +### What napi-rs WASM support gives us (from napi-rs v3 docs) + +- Single supported target: **`wasm32-wasip1-threads`** — threads/Atomics over + SharedArrayBuffer work out of the box; Rust code (incl. rayon) compiles + unmodified. +- `napi build --target wasm32-wasip1-threads` emits + `kimchi_napi.wasm32-wasi.wasm` plus generated loaders: `.wasi.cjs` (Node), + `.wasi-browser.js` (browser), and `wasi-worker(-browser).mjs` worker files. + Runtime is `@napi-rs/wasm-runtime` (emnapi + WASI shim + memfs). +- Standard packaging convention: a `{name}-wasm32-wasi` npm package with + `cpu: ["wasm32"]`, used automatically as fallback by generated index.js + loaders. +- Browser use requires COOP/COEP headers for SharedArrayBuffer (same requirement + the current wasm-bindgen web backend already has). +- C/C++ deps would need `WASI_SDK_PATH`; the kimchi dependency tree is pure + Rust, so this should not be needed. + +--- + +## 2. Target architecture + +``` + OCaml (Pickles/Snarky) ──jsoo──► o1js_node.bc.cjs / o1js_web.bc.js + │ calls `kimchi_ffi.*` + ┌───────────────────────────────────────────────┴──────────────────────────┐ + │ ONE Rust crate: kimchi-napi │ + │ napi build ──► kimchi_napi.node (@o1js/native-{platform}-{arch}) │ + │ napi build --target wasm32-wasip1-threads │ + │ ──► kimchi_napi.wasm (@o1js/native-wasm32-wasi) │ + │ loaded via @napi-rs/wasm-runtime (.wasi.cjs / │ + │ .wasi-browser.js — threading handled by the runtime) │ + │ │ + │ ONE TS conversion layer: napi-conversion-*.ts + napi-srs.ts │ + │ ONE loader per platform: thin `require`/`import`, no patching │ + └──────────────────────────────────────────────────────────────────────────┘ +``` + +Deleted: `kimchi_wasm` consumption in o1js, +`conversion-{core,proof,oracles,verifier-index}.ts`, `srs.ts` (wasm variant), +`worker-spec.js`, `worker-helpers.js` marshalling, `fix-wasm-bindings-node.js`, +the wasm-bindgen halves of `node-backend.js`/`web-backend.js`, +`scripts/build/wasm/*`, and the `__kimchi_backend` dual-conversion switch. + +`setBackend('wasm' | 'native')` keeps its public semantics — it now selects +_napi-wasm_ vs _napi-native_ builds of the same crate. + +## 3. Migration steps + +### Phase 0 — Groundwork (verify before committing to the cutover) + +- [x] Confirm `kimchi-napi` API parity with `kimchi_wasm` (done — `lib.rs` + exports the full `caml_*` surface with `Wasm*` aliases; JSOO + `bindings/*.js` only references `kimchi_ffi` + `tsBindings`). +- [x] Confirm napi-rs v3 wasm target + toolchain availability (Rust 1.92 + + `wasm32-wasip1-threads` installed in this environment). +- [ ] `cargo check -p kimchi-napi --target wasm32-wasip1-threads` — surface any + target-specific compile issues (e.g. `libc` usage, `getrandom` config, fs + access in `srs.rs` read/write paths). +- [ ] Review the o1-labs napi-rs fork delta vs upstream 3.3.0/3.9+; wasm-runtime + fixes have landed steadily upstream, so pin the CLI + runtime versions + that match. +- [ ] Verify rayon thread-pool sizing under WASI (native uses OS core count; + under wasi the loader must pass thread count — set via `RAYON_NUM_THREADS` + in the WASI env or an explicit `ThreadPoolBuilder` init call exposed over + napi). + +### Phase 1 — Node cutover (this branch) + +1. **Rust/submodule:** add a wasm build entry point next to the existing native + one — `kimchi_bindings/js/native/build-wasm.sh` running + `napi build --package kimchi-napi --target wasm32-wasip1-threads --release --esm`. + Drop the vestigial `kimchi_wasm` dependency from `kimchi-napi/Cargo.toml` + (only a comment references it). +2. **Packaging:** new `scripts/build/native/build-wasm.sh` in o1js producing + `native/wasm32-wasi/` → `@o1js/native-wasm32-wasi` (napi conventions: + `cpu: ["wasm32"]`, ships `.wasm` + generated `.wasi.cjs`/browser loaders; + depends on `@napi-rs/wasm-runtime`). +3. **Loader:** rewrite `src/bindings/js/node/node-backend.js` as a thin loader + for the napi-wasm artifact (require `@o1js/native-wasm32-wasi`'s `.wasi.cjs`, + fall back to locally-built artifacts for dev). No memory Proxy, no + `wbg_rayon` dance — thread spawning is `@napi-rs/wasm-runtime`'s job. + `withThreadPool` becomes the same no-op state machine the native backend + uses. +4. **JSOO stub:** update `kimchi_bindings/js/node_js/node_backend.js` default + branch to require the napi-wasm loader instead of `./kimchi_wasm.js`, and set + `__kimchi_backend = 'native'`-style routing so the **napi conversion layer is + used for both backends** (the flag effectively becomes "napi object model + everywhere"). +5. **TS layer:** collapse `getRustConversion()` to the napi bundle; delete the + wasm conversion files once web is migrated (Phase 2) — until then they stay + but are unreachable from Node. +6. **Build scripts:** `build:wasm:node` now builds the napi wasm artifact; + `fix-wasm-bindings-node.js` is deleted; `download-bindings.sh` / + `update-o1js-bindings.sh` updated for the new artifact names. +7. **Validation:** smoke-test the wasm artifact from Node (gate vector + round-trip, SRS create, poseidon), then `npm run build` + jest suites with + `O1JS_BACKEND=wasm` (napi-wasm) and `O1JS_BACKEND=native`; perf-regression + baselines re-dumped. + +### Phase 2 — Web cutover (design decision required) + +The one genuinely new problem: **browser main threads cannot block** (no +`Atomics.wait`/`memory.atomic.wait32`), and JSOO calls `kimchi_ffi` +synchronously from the main thread. Kimchi's rayon calls block the calling +thread at join points. The current wasm-bindgen web backend solves this by +hosting the whole wasm instance in a dedicated worker and spin-waiting on the +main thread (`worker-spec.js` pointer marshalling). Options for napi-wasm: + +- **Option A — single-threaded on main thread (simplest, ship first):** + instantiate the `.wasi-browser.js` module on the main thread; rayon falls back + to inline sequential execution when it has no pool. Correctness preserved, + zero marshalling code, but web proving loses multi-core parallelism vs today. +- **Option B — worker-hosted module + handle-registry RPC (target state):** host + the napi-wasm instance in one worker (blocking allowed there, full rayon + parallelism); main thread proxies `kimchi_ffi` calls with an explicit + object-handle table (integer IDs) + SharedArrayBuffer spin-wait — same shape + as today's `worker-spec.js`, but against _our own_ stable handle table instead + of wasm-bindgen pointer internals. +- **Option C — move whole proving pipeline (JSOO included) into a worker:** + cleanest long-term (no sync-over-async hacks at all), but changes o1js's + public initialization story; out of scope here. + +Recommendation: implement **A** as part of this migration (it deletes the entire +marshalling layer and gets web onto napi immediately), benchmark; follow up with +**B** behind the same loader interface if web proving perf matters before **C** +lands. + +### Phase 3 — Cleanup + +- Delete `kimchi_wasm`-consuming code from o1js: `conversion-*.ts` wasm bundle, + `srs.ts` (wasm), `worker-spec.js`, web marshalling, `scripts/build/wasm/*`, + wasm branches in `bindings.ts`. +- (Upstream, later) retire the `kimchi_wasm` crate's JS packaging (`node_js/`, + `web/` dune targets) in the mina repo once no consumer remains. +- Update `bindings.d.ts` to source types from napi-generated `index.d.ts`. +- Update `AGENT.md`/`AGENT_LOG.md` + `src/bindings/README.md` to describe the + new layer. + +## 4. Risks & mitigations + +| Risk | Impact | Mitigation | +| ----------------------------------------------------------------------------------------------------- | ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Browser main-thread blocking with threads (Phase 2) | Web proving perf or crashes | Option A first (inline rayon fallback, no blocking), Option B for parallelism; verify rayon's spawn-failure fallback actually engages under wasip1-threads on the main thread | +| o1-labs napi-rs fork may lag upstream wasm fixes | wasm target bugs | Diff fork vs upstream; prefer upstreaming the fork's patches or rebasing onto ≥3.9 before web rollout | +| Rayon pool sizing under WASI defaults to 1 thread | Silent perf loss on Node-wasm | Loader passes explicit thread count (env `RAYON_NUM_THREADS` via WASI env vars or an exported pool-init fn); assert via `build_info` | +| Perf delta napi-wasm vs wasm-bindgen (`wasm-opt -O4`, externref glue, emnapi overhead, no SIMD flags) | Slower proving in wasm fallback | Run `tests/perf-regression` against both before deleting wasm-bindgen; apply `wasm-opt` to the napi artifact too | +| `srs.rs` `caml_*_srs_read/write` use real fs paths | Web/wasi fs mismatch | `@napi-rs/wasm-runtime` provides memfs; these entry points are unused by o1js's JS-side SRS cache — keep them compiled but untested on wasm | +| Artifact size (wasip1-threads links wasi-libc; no `wasm-opt` by default) | Bigger downloads | Measure vs current `_bg.wasm` (already multi-MB); `wasm-opt -O4` in the packaging step | +| JSOO artifact regeneration requires OCaml/dune toolchain not present in this sandbox | Can't fully e2e-test here | JSOO stub change is 5 lines of plain JS; validate via CI (`build:bindings-remote`) / `download-bindings.sh`; smoke-test the napi-wasm module directly from Node in the meantime | +| npm skips `cpu: wasm32` optional deps by default | Fallback not installed for some users | Same convention napi-rs ecosystem uses; document `npm install --cpu=wasm32`, and keep the wasm package as a regular (non-optional) dependency of `o1js` if we want a guaranteed fallback | + +## 5. Validation plan + +1. `cargo check`/`cargo build -p kimchi-napi` for host and + `wasm32-wasip1-threads`. +2. Node smoke test of the wasm artifact: load `.wasi.cjs`, exercise gate vector + round-trip, poseidon block cipher, `caml_fp_srs_create_parallel` (exercises + rayon threads), proof of a trivial circuit if JSOO artifacts are available. +3. Full jest + perf-regression with `O1JS_BACKEND=wasm` (now napi-wasm) and + `native`, locally or in CI once bindings artifacts are rebuilt. +4. Web: playwright e2e (`npm run test:e2e`) against the Option-A loader. + +## 6. Implementation report (what actually landed on this branch) + +**Rust / mina submodule** (`src/mina`, incl. `proof-systems` sub-submodule): + +- `kimchi-napi` builds for `wasm32-wasip1-threads` on **stable** Rust 1.92 via + the napi CLI. Two blockers fixed: vestigial + `#![cfg_attr(target_arch="wasm32", feature(unsigned_is_multiple_of))]` in + `o1-utils` and `poly-commitment` (E0554 on stable), and missing wasm cfg arms + in `kimchi-napi/src/build_info.rs` (`OS_NAME`/`ARCH_NAME`; `BACKING` now + reports `"wasm"` on wasm targets). +- napi config (`kimchi_bindings/js/native/package.json`) now declares + `packageName: "@o1js/native"`, all build targets incl. + `wasm32-wasip1-threads`, and wasm memory limits (initial 1024 / max 65536 + pages) — required for the CLI to emit the `.wasi.cjs` / `.wasi-browser.js` / + worker loaders. +- New `kimchi_bindings/js/native/build-wasm.sh` (mirrors `build.sh`). +- `header-d.ts` extended so the generated `index.d.ts` is valid, self-contained + TS (exports opaque pointer types + missing `Napi*` aliases). +- JSOO `kimchi_ffi` stubs (`node_js/node_backend.js`, `web/web_backend.js`) + rewritten: they now read the FFI module from `globalThis.__o1js_kimchi_ffi`, + installed by the o1js loaders. OCaml-land no longer does platform-specific + `require`s — this needs a JSOO artifact rebuild in CI before the branch is + testable end-to-end. + +**o1js:** + +- `node-backend.js`: thin loader for + `compiled/node_bindings/kimchi_napi.wasi.cjs`; sets `RAYON_NUM_THREADS` from + `workers.numWorkers`/CPU count before instantiation; no-op thread pool. + Deleted: memory Proxy hack, `wbg_rayon` worker dance. +- `native-backend.js`: unchanged flow, now also installs `__o1js_kimchi_ffi`. +- `web-backend.js`: Option A — imports the bundled `.wasi-browser.js`, installs + the global, evals the JSOO artifact. Deleted: `worker-spec.js`, + `worker-helpers.js`, the u32 spin-wait marshalling. +- `crypto/bindings.ts`: single napi conversion bundle (wasm variants deleted: + `conversion-{core,proof,oracles,verifier-index}.ts`, `srs.ts`); added + per-module bundle caching. Cleaned vestigial "constructor" plumbing for + `#[napi(object)]` types out of + `napi-conversion-verifier-index.ts`/`napi-wrappers.ts`. +- Types: all imports retargeted from `kimchi_wasm.cjs` to `kimchi_napi.wasi.cjs` + (d.cts generated from the napi build + post-processed). `prover-keys.ts` now + types prover indexes as `ExternalObject<...>` matching reality. +- Build: `scripts/build/wasm/build-{node,web}.sh` rewritten around a shared + `build-kimchi-napi-wasm.sh`; `fix-wasm-bindings-node.js` deleted; + `build-web.js` pre-bundles the browser loader (resolving + `@napi-rs/wasm-runtime`) and ships the `.wasm` + worker as plain files; JSOO + postprocessing dropped the `kimchi_wasm.js→.cjs` sed and the + webpack-native-require perl hack. +- `@napi-rs/wasm-runtime` added as a runtime dependency. Distribution decision: + the wasm artifacts ship **inside the o1js package** (as before), not as a + separate `@o1js/native-wasm32-wasi` npm package — fewer moving parts; the + packaging convention can be revisited when publishing infra wants it. +- `bindings.unit-test.ts` (TS-vs-Rust equivalence for bigint256/field/projective + ops) was deleted: its Rust reference implementations were wasm-bindgen-only + exports that kimchi-napi intentionally does not expose. + `gate-vector-napi.unit-test.ts` now also runs against the wasm build. + +**Validated in this environment** (no OCaml toolchain, so no JSOO rebuild here): + +- `kimchi-napi` wasm build end-to-end via the new scripts (6.3 MB wasm binary). +- Node smoke tests: module load, gate-vector round-trip + digest via the shared + napi conversion layer, poseidon block cipher, `caml_fp_srs_create_parallel` + with real rayon worker threads (`RAYON_NUM_THREADS` honored), serial SRS, + `withThreadPool`. +- `tsc -p tsconfig.node.json` and full-repo `tsc` clean (modulo pre-existing + examples-vs-dist errors). + +**Still needed (CI / follow-up):** + +1. Rebuild JSOO artifacts (`npm run build:jsoo` in CI) — old compiled artifacts + reference the removed `kimchi_wasm.cjs` and won't work with this branch. +2. Run the full jest + vk/perf-regression suites for `O1JS_BACKEND=wasm|native`; + re-dump wasm perf baselines (napi-wasm ≠ wasm-bindgen perf profile). +3. Web: benchmark Option A; implement Option B (worker-hosted RPC) if web + proving parallelism is required; validate SharedArrayBuffer/COOP-COEP e2e via + playwright. +4. Decide on iOS memory ceiling (generated loader hardcodes max 4 GiB; old code + used 1 GiB on iOS) — post-process or napi config per-target if needed. +5. Review the o1-labs napi-rs fork delta vs upstream before web rollout; the + `kimchi_wasm` crate and its dune targets in the mina repo can be retired once + no consumer remains. diff --git a/STATE.md b/STATE.md new file mode 100644 index 0000000000..bc307f18d0 --- /dev/null +++ b/STATE.md @@ -0,0 +1,94 @@ +# STATE: wasm-bindgen → napi-rs wasm migration + +**Branch:** `florian/wasm-napi` · **Last updated:** 2026-07-02 · Companion docs: +`PLAN.md` (design + implementation report), `AGENT_LOG.md` (gotchas learned +during the migration). + +## Summary + +The wasm backend of o1js is no longer wasm-bindgen (`kimchi_wasm`). It is now +the `wasm32-wasip1-threads` build of the **`kimchi-napi`** crate — the same +napi-rs crate that powers the native (`.node`) backend — loaded through +generated `@napi-rs/wasm-runtime` loaders. Native and wasm are two build +targets of one crate, sharing one TS conversion layer +(`src/bindings/crypto/native/`) and one JSOO FFI stub protocol +(`globalThis.__o1js_kimchi_ffi`, installed by the o1js backend loaders before +the compiled OCaml artifact is evaluated). + +Deleted: `worker-spec.js` pointer marshalling, `worker-helpers.js`, +`fix-wasm-bindings-node.js`, the `WebAssembly.Memory` Proxy hack, the +`wbg_rayon` worker bootstrapping, and the wasm-bindgen TS conversion layer +(`conversion-{core,proof,oracles,verifier-index}.ts`, `srs.ts`). + +Changes span three repos (all committed locally, none pushed): + +- **o1js** — loaders, conversion-layer collapse, build scripts, type + retargeting, docs. +- **`src/mina` submodule** — JSOO `kimchi_ffi` stubs, napi config + (`kimchi_bindings/js/native/package.json`), `header-d.ts`, `build-wasm.sh`. +- **`proof-systems` sub-submodule** — wasm cfg arms in + `kimchi-napi/src/build_info.rs`, removed vestigial nightly-feature attrs in + `o1-utils`/`poly-commitment`, dropped unused `kimchi_wasm` dep. + +## Status by platform + +### Node — implemented and validated + +- `kimchi-napi` builds for `wasm32-wasip1-threads` on **stable Rust 1.92** via + `napi build` (never raw cargo — it needs `EMNAPI_LINK_DIR` set by the CLI). +- Verified empirically in this environment: module loads, rayon spawns real + worker threads (`RAYON_NUM_THREADS` honored via WASI env), `withThreadPool` + works, and **native vs wasm backends produce byte-identical results** + (circuit digest, poseidon block cipher, SRS points). +- `tsc` clean; `build:wasm:node` / `build:dev` pipeline runs end-to-end; + `gate-vector-napi.unit-test` passes against both backends. + +### Web — implemented (Option A), NOT yet validated + +`web-backend.js` loads the generated `kimchi_napi.wasi-browser.js` on the main +thread; `build-web.js` pre-bundles the loader (resolving +`@napi-rs/wasm-runtime`) and ships the `.wasm` + worker file next to the +bundle. Untested: rebuilding `o1js_web.bc.js` requires the OCaml toolchain and +there is no browser here. Run `npm run test:e2e` after CI rebuilds bindings. + +**The web threading constraint:** browser main threads cannot block +(`memory.atomic.wait32` traps), and JSOO calls the FFI synchronously from the +main thread. Options, as discussed in PLAN.md §3 Phase 2: + +- **Option A (current):** instantiate the napi-wasm module on the main thread. + Correct but rayon-parallel sections cannot fan out — web proving is + effectively single-threaded. Simplest possible architecture; benchmark + before accepting. +- **Option B (designed, not built):** host the napi-wasm instance in one + dedicated Web Worker (blocking allowed there → full rayon parallelism) and + proxy `kimchi_ffi` calls from the main thread via an explicit object-handle + table + SharedArrayBuffer spin-wait. Same shape as the old `worker-spec.js` + machinery, but against our own stable handle registry instead of + wasm-bindgen pointer internals. Recommended follow-up if Option A benchmarks + poorly (likely for large circuits). +- **Option C (out of scope):** move the whole proving pipeline (JSOO included) + into a worker — cleanest long-term, but changes o1js's public initialization + story. + +## Blocking next steps (CI) + +1. **Rebuild JSOO artifacts** (`npm run build:jsoo` / `build:bindings-remote`). + Previously compiled `o1js_node.bc.cjs`/`o1js_web.bc.js` artifacts reference + the removed `kimchi_wasm.cjs` and are incompatible with this branch. +2. Full jest + vk/perf-regression suites for `O1JS_BACKEND=wasm|native`; + re-dump wasm perf baselines (napi-wasm ≠ wasm-bindgen perf profile). +3. Web e2e (playwright) against Option A; benchmark web proving. + +## Open items / risks + +- **iOS memory ceiling:** the generated browser loader hardcodes shared memory + max 4 GiB (napi config); the old backend used 1 GiB on iOS. May need + post-processing or a config split. +- **o1-labs napi-rs fork** (`o1-labs/napi-rs@023d1d4f`, v3.3.0): review delta + vs upstream before web rollout; upstream has ongoing wasm-runtime fixes. +- **Deleted test coverage:** `bindings.unit-test.ts` (TS-vs-Rust equivalence + for bigint256/field/projective arithmetic) had wasm-bindgen-only reference + functions; could be restored by exposing them from kimchi-napi behind a + test-only feature. +- **Upstream cleanup (later):** retire the `kimchi_wasm` crate and its + `node_js/`/`web/` dune packaging in the mina repo once no consumer remains. diff --git a/package-lock.json b/package-lock.json index 7852a859f2..7aaaeeb044 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "3.0.0-mesa.final.pr2888.0", "license": "Apache-2.0", "dependencies": { + "@napi-rs/wasm-runtime": "^1.0.7", "@noble/hashes": "^1.7.1", "blakejs": "1.2.1", "cachedir": "^2.4.0", @@ -101,7 +102,6 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.23.3.tgz", "integrity": "sha512-Jg+msLuNuCJDyBvFv5+OKOUjWMZgd85bKjbICd3zWrKAo+bJ49HJufi7CQE0q0uR8NGyO6xkCACScNqyjHSZew==", "dev": true, - "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.22.13", @@ -570,9 +570,7 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.7.0.tgz", "integrity": "sha512-pJdKGq/1iquWYtv1RRSljZklxHCOCAJFJrImO5ZLKPJVJlVUcs8yFwNQlqS0Lo8xT1VAXXTCZocF9n26FWEKsw==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "@emnapi/wasi-threads": "1.1.0", "tslib": "^2.4.0" @@ -582,9 +580,7 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.0.tgz", "integrity": "sha512-oAYoQnCYaQZKVS53Fq23ceWMRxq5EhQsE0x0RdQ55jT7wagMu5k+fS39v1fiSLrtrLQlXwVINenqhLMtTrV/1Q==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "tslib": "^2.4.0" } @@ -593,9 +589,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "tslib": "^2.4.0" } @@ -3067,9 +3061,7 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.0.7.tgz", "integrity": "sha512-SeDnOO0Tk7Okiq6DbXmmBODgOAb9dp9gjlphokTUxmt8U3liIP1ZsozBahH69j/RJv+Rfs6IwUKHTgQYJ/HBAw==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "@emnapi/core": "^1.5.0", "@emnapi/runtime": "^1.5.0", @@ -3498,7 +3490,6 @@ "integrity": "sha512-1LFfa/qnMQvEOAdzlQymH0ulepxbxnCYAKJZfMci/5XJyIHWgEYnDmgnKakbTh7CH2tFQ5O60oYDvns4i9RAIg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.1.0", @@ -3728,7 +3719,6 @@ "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -4091,9 +4081,7 @@ "version": "0.10.1", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "tslib": "^2.4.0" } @@ -4219,7 +4207,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.9.tgz", "integrity": "sha512-0f5klcuImLnG4Qreu9hPj/rEfFq6YRc5n2mAjSsH+ec/mJL+3voBH0+8T7o8RpFjH7ovc+TRsL/c7OYIQsPTfQ==", "dev": true, - "peer": true, "dependencies": { "undici-types": "~5.26.4" } @@ -4499,7 +4486,6 @@ "url": "https://github.com/sponsors/ai" } ], - "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001541", "electron-to-chromium": "^1.4.535", @@ -5644,7 +5630,6 @@ "resolved": "https://registry.npmjs.org/jest/-/jest-28.1.3.tgz", "integrity": "sha512-N4GT5on8UkZgH0O5LUavMRV1EDEhNTL0KEfRmDIeZHSV7p2XgLoY9t9VDUgL6o+yfdgYHVxuz81G8oB9VG5uyA==", "dev": true, - "peer": true, "dependencies": { "@jest/core": "^28.1.3", "@jest/types": "^28.1.3", @@ -7656,7 +7641,6 @@ "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -8409,7 +8393,6 @@ "integrity": "sha512-5PzUddaA9FbaarUzIsEc4wNXCiO4Ot3bJNeMF2qKpYlTmM9TTaSHQ7162w756ERCkXER/+o2purRG6YOAv6EMA==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "@gerrit0/mini-shiki": "^3.2.2", "lunr": "^2.3.9", @@ -8482,7 +8465,6 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz", "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==", "dev": true, - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/package.json b/package.json index 60ef99d9de..43f6d542d1 100644 --- a/package.json +++ b/package.json @@ -137,6 +137,7 @@ "typescript": "^5.4.5" }, "dependencies": { + "@napi-rs/wasm-runtime": "^1.0.7", "@noble/hashes": "^1.7.1", "blakejs": "1.2.1", "cachedir": "^2.4.0", diff --git a/scripts/build/jsoo/build-node.sh b/scripts/build/jsoo/build-node.sh index c127815a1f..a9953388e6 100755 --- a/scripts/build/jsoo/build-node.sh +++ b/scripts/build/jsoo/build-node.sh @@ -52,23 +52,10 @@ info "moving some files to CommonJS format..." run_cmd mv -f $BINDINGS_PATH/o1js_node.bc.js $BINDINGS_PATH/o1js_node.bc.cjs ok "Node.js bindings copied" -info "Updating WASM references in bindings..." -run_cmd sed -i 's/kimchi_wasm.js/kimchi_wasm.cjs/' $BINDINGS_PATH/o1js_node.bc.cjs -ok "WASM references updated" - -info "making native require opaque to webpack..." -# webpack creates a context module from dynamic require("@o1js/native-" + ...) which -# greedily bundles all files in the @o1js/ scope. __non_webpack_require__ tells webpack -# to skip this require while keeping it functional at runtime. -run_cmd perl -e ' - local $/; - open(F, "<", $ARGV[0]) or die $!; - my $c = ; close(F); - $c =~ s/require\s*\("\@o1js\/native-"/(typeof __non_webpack_require__ !== "undefined" ? __non_webpack_require__ : require)("\@o1js\/native-"/g; - open(F, ">", $ARGV[0]) or die $!; - print F $c; close(F); -' $BINDINGS_PATH/o1js_node.bc.cjs -ok "native require made webpack-safe" +# note: the kimchi_ffi stub in the artifact reads the FFI module from +# globalThis.__o1js_kimchi_ffi (installed by the o1js backend loaders before the +# artifact is evaluated), so no require-rewriting for wasm or native packages is +# needed here anymore info "fixing JS bindings for better error handling..." run_cmd sed -i 's/function failwith(s){throw \[0,Failure,s\]/function failwith(s){throw globalThis.Error(s.c)/' "${BINDINGS_PATH}"/o1js_node.bc.cjs diff --git a/scripts/build/wasm/build-kimchi-napi-wasm.sh b/scripts/build/wasm/build-kimchi-napi-wasm.sh new file mode 100755 index 0000000000..b3fcde6889 --- /dev/null +++ b/scripts/build/wasm/build-kimchi-napi-wasm.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# Description: +# Builds the wasm32-wasip1-threads target of the kimchi-napi crate — the same +# crate that powers the native (.node) backend — using the napi-rs CLI. The +# resulting artifacts (wasm binary + generated Node/browser loaders backed by +# @napi-rs/wasm-runtime) are shared by the node and web wasm build scripts. +# +# Requirements: +# - Rust toolchain with the `wasm32-wasip1-threads` target installed +# - node_modules installed (uses @napi-rs/cli and emnapi) +# +# Usage: +# invoked by ./scripts/build/wasm/build-node.sh and ./build-web.sh + +source ./scripts/lib/ux.sh + +setup_script "kimchi-napi-wasm-build" "kimchi-napi wasm build" + +MINA_PATH=./src/mina +NATIVE_PATH=$MINA_PATH/src/lib/crypto/kimchi_bindings/js/native +PROOF_SYSTEMS_PATH=$MINA_PATH/src/lib/crypto/proof-systems +NAPI=$(pwd)/node_modules/.bin/napi +ARTIFACTS_PATH=$NATIVE_PATH/artifacts-wasm + +info "building kimchi-napi for wasm32-wasip1-threads..." + +( + cd $NATIVE_PATH + run_cmd "$NAPI" build \ + --manifest-path ../../../proof-systems/Cargo.toml \ + --package kimchi-napi \ + --target wasm32-wasip1-threads \ + --release \ + --platform \ + --output-dir ./artifacts-wasm +) + +info "fixing generated type definitions..." + +# The napi-rs type-def generator emits setters with optional parameters, which +# is invalid TypeScript (TS1051). Rewrite them to required parameters (the +# `| undefined | null` in the type already conveys optionality). +node -e ' + let fs = require("fs"); + let path = process.argv[1]; + let src = fs.readFileSync(path, "utf8"); + src = src.replace(/(set [A-Za-z_][A-Za-z0-9_]*\([A-Za-z_][A-Za-z0-9_]*)\?:/g, "$1:"); + fs.writeFileSync(path, src); +' $ARTIFACTS_PATH/index.d.ts + +success "kimchi-napi wasm build success!" diff --git a/scripts/build/wasm/build-node.sh b/scripts/build/wasm/build-node.sh index 93f9fb8602..bc30f2d7cf 100755 --- a/scripts/build/wasm/build-node.sh +++ b/scripts/build/wasm/build-node.sh @@ -2,16 +2,13 @@ set -Eeuo pipefail # Description: -# Builds the Kimchi WebAssembly (WASM) bindings for Node.js. This script: -# - Compiles the Kimchi proof system’s Node bindings using Dune, generating -# the WebAssembly and JavaScript interface files: -# - `plonk_wasm_bg.wasm` and its TypeScript definitions. -# - `plonk_wasm.js` (the JS interface) and its type declarations. -# - Copies all generated artifacts into `src/bindings/compiled/node_bindings/`. -# - Converts the output files to CommonJS format (`.cjs` / `.d.cts`) for -# compatibility with Node.js environments. -# - Applies automatic fixes to the generated bindings via -# `src/build/fix-wasm-bindings-node.js` to ensure correct runtime behavior. +# Builds the Kimchi WebAssembly bindings for Node.js. This script: +# - Builds the wasm32-wasip1-threads target of the kimchi-napi crate (the +# same crate that powers the native backend) via the napi-rs CLI. +# - Copies the wasm binary, the generated Node loader (`kimchi_napi.wasi.cjs`, +# backed by @napi-rs/wasm-runtime) and its worker file into +# `src/bindings/compiled/node_bindings/`. +# - Installs the generated type definitions as `kimchi_napi.wasi.d.cts`. # # Usage: # npm run build:wasm:node @@ -21,35 +18,20 @@ source ./scripts/lib/ux.sh setup_script "wasm-node-build" "wasm node build" MINA_PATH=./src/mina -KIMCHI_PATH=$MINA_PATH/src/lib/crypto/kimchi_bindings/js/node_js/ -BUILT_PATH=./_build/default/$KIMCHI_PATH +ARTIFACTS_PATH=$MINA_PATH/src/lib/crypto/kimchi_bindings/js/native/artifacts-wasm BINDINGS_PATH=./src/bindings/compiled/node_bindings/ -mkdir -p $BINDINGS_PATH - -info "building Kimchi bindings for node..." +./scripts/build/wasm/build-kimchi-napi-wasm.sh -TARGETS=(\ - kimchi_wasm_bg.wasm \ - kimchi_wasm_bg.wasm.d.ts \ - kimchi_wasm.js \ - kimchi_wasm.d.ts \ -) -dune build ${TARGETS[@]/#/$KIMCHI_PATH/} +mkdir -p $BINDINGS_PATH info "copying artifacts into the right place..." -for target in "${TARGETS[@]}"; do - cp $BUILT_PATH/$target $BINDINGS_PATH/$target - chmod 660 $BINDINGS_PATH/$target -done - -info "moving some files to CommonJS format..." - -mv $BINDINGS_PATH/kimchi_wasm.js $BINDINGS_PATH/kimchi_wasm.cjs -mv $BINDINGS_PATH/kimchi_wasm.d.ts $BINDINGS_PATH/kimchi_wasm.d.cts - -info "autofixing wasm bindings for Node.JS..." -run_cmd node src/build/fix-wasm-bindings-node.js $BINDINGS_PATH/kimchi_wasm.cjs +# note: the debug wasm is intentionally not copied — the generated loader +# prefers it over the release binary when both are present +cp $ARTIFACTS_PATH/kimchi_napi.wasm32-wasi.wasm $BINDINGS_PATH/ +cp $ARTIFACTS_PATH/kimchi_napi.wasi.cjs $BINDINGS_PATH/ +cp $ARTIFACTS_PATH/wasi-worker.mjs $BINDINGS_PATH/ +cp $ARTIFACTS_PATH/index.d.ts $BINDINGS_PATH/kimchi_napi.wasi.d.cts success "WASM node build success!" diff --git a/scripts/build/wasm/build-web.sh b/scripts/build/wasm/build-web.sh index ad41580cc0..9835600622 100755 --- a/scripts/build/wasm/build-web.sh +++ b/scripts/build/wasm/build-web.sh @@ -2,66 +2,50 @@ set -Eeuo pipefail # Description: -# Builds the Kimchi WebAssembly (WASM) bindings for Web (browser) usage. This script: -# - Compiles the Kimchi proof system’s Web bindings using Dune, generating: -# - `plonk_wasm_bg.wasm` (the main WebAssembly module) and its TypeScript definitions. -# - `plonk_wasm.js` (the JavaScript interface) and its type declarations. -# - Copies all generated artifacts into `src/bindings/compiled/web_bindings/`. -# - Optimizes the WebAssembly binary using `wasm-opt` with advanced flags -# (`--detect-features`, `--enable-mutable-globals`, `-O4`) for performance and size reduction. -# - Ensures proper file permissions after optimization. -# -# Note: -# - Requires `wasm-opt` (from Binaryen) to be installed and available in PATH. +# Builds the Kimchi WebAssembly bindings for Web (browser) usage. This script: +# - Builds the wasm32-wasip1-threads target of the kimchi-napi crate (the +# same crate that powers the native backend) via the napi-rs CLI. +# - Copies the wasm binary, the generated browser loader +# (`kimchi_napi.wasi-browser.js`, backed by @napi-rs/wasm-runtime) and its +# worker file into `src/bindings/compiled/web_bindings/`. +# - Optimizes the WebAssembly binary with `wasm-opt` when available. # # Usage: # npm run build:wasm:web - source ./scripts/lib/ux.sh setup_script "wasm-web-build" "wasm web build" -if ! command -v wasm-opt >/dev/null 2>&1; then - error "wasm-opt is required for web bindings optimization" - exit 1 -fi - MINA_PATH=./src/mina -KIMCHI_PATH=$MINA_PATH/src/lib/crypto/kimchi_bindings/js/web/ -BUILT_PATH=./_build/default/$KIMCHI_PATH +ARTIFACTS_PATH=$MINA_PATH/src/lib/crypto/kimchi_bindings/js/native/artifacts-wasm BINDINGS_PATH=./src/bindings/compiled/web_bindings/ -mkdir -p $BINDINGS_PATH - +./scripts/build/wasm/build-kimchi-napi-wasm.sh -info "building Kimchi bindings for web..." - -TARGETS=(\ - kimchi_wasm_bg.wasm \ - kimchi_wasm_bg.wasm.d.ts \ - kimchi_wasm.js \ - kimchi_wasm.d.ts\ -) -dune build ${TARGETS[@]/#/$KIMCHI_PATH/} +mkdir -p $BINDINGS_PATH info "copying artifacts into the right place..." -for target in "${TARGETS[@]}"; do - cp $BUILT_PATH/$target $BINDINGS_PATH/$target -done - -info "optimizing wasm with wasm-opt..." -run_cmd wasm-opt \ - --detect-features \ - --enable-mutable-globals \ - -O4 \ - -o $BINDINGS_PATH/kimchi_wasm_bg.wasm.opt \ - $BINDINGS_PATH/kimchi_wasm_bg.wasm -run_cmd mv $BINDINGS_PATH/kimchi_wasm_bg.wasm.opt $BINDINGS_PATH/kimchi_wasm_bg.wasm - -ok "wasm optimized" - -chmod 660 ${TARGETS[@]/#/$BINDINGS_PATH/} +cp $ARTIFACTS_PATH/kimchi_napi.wasm32-wasi.wasm $BINDINGS_PATH/ +cp $ARTIFACTS_PATH/kimchi_napi.wasi-browser.js $BINDINGS_PATH/ +cp $ARTIFACTS_PATH/wasi-worker-browser.mjs $BINDINGS_PATH/ +cp $ARTIFACTS_PATH/index.d.ts $BINDINGS_PATH/kimchi_napi.wasi-browser.d.ts + +if command -v wasm-opt >/dev/null 2>&1; then + info "optimizing wasm with wasm-opt..." + run_cmd wasm-opt \ + --detect-features \ + --enable-mutable-globals \ + --enable-threads \ + --enable-bulk-memory \ + -O4 \ + -o $BINDINGS_PATH/kimchi_napi.wasm32-wasi.wasm.opt \ + $BINDINGS_PATH/kimchi_napi.wasm32-wasi.wasm + run_cmd mv $BINDINGS_PATH/kimchi_napi.wasm32-wasi.wasm.opt $BINDINGS_PATH/kimchi_napi.wasm32-wasi.wasm + ok "wasm optimized" +else + warn "wasm-opt not found — skipping wasm optimization" +fi success "WASM web build success!" diff --git a/src/bindings.d.ts b/src/bindings.d.ts index d99101b1d0..6744314e2a 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -29,8 +29,8 @@ import type { import type { WasmFpSrs, WasmFqSrs, -} from './bindings/compiled/node_bindings/kimchi_wasm.cjs'; -import * as wasm from './bindings/compiled/node_bindings/kimchi_wasm.cjs'; +} from './bindings/compiled/node_bindings/kimchi_napi.wasi.cjs'; +import * as wasm from './bindings/compiled/node_bindings/kimchi_napi.wasi.cjs'; import type { KimchiGateType } from './lib/provable/gates.ts'; import type { MlConstraintSystem } from './lib/provable/core/provable-context.ts'; import type { FieldVector } from './bindings/crypto/bindings/vector.ts'; diff --git a/src/bindings/README.md b/src/bindings/README.md index eda670bbe5..16db74c1b3 100644 --- a/src/bindings/README.md +++ b/src/bindings/README.md @@ -6,16 +6,21 @@ OCaml. **Directory structure** -- `/compiled` - compiled JS and Wasm artifacts produced by `js_of_ocaml` and - `wasm-bindgen` from Rust and OCaml source code. We keep these artifacts in the - source tree so that developing on o1js can be done with standard JS tooling - and doesn't require setting up the full OCaml/Rust build pipeline. +- `/compiled` - compiled JS and Wasm artifacts produced by `js_of_ocaml` (from + OCaml source code) and by `napi-rs` (the `wasm32-wasip1-threads` build of the + `kimchi-napi` Rust crate — the same crate that powers the native backend). + We keep these artifacts in the source tree so that developing on o1js can be + done with standard JS tooling and doesn't require setting up the full + OCaml/Rust build pipeline. - `/crypto` - pure TS implementations of a subset of the crypto primitives we use, including finite field and elliptic curve arithmetic. This is used by - mina-signer (a pure TS package) to hash and sign transactions. -- `/js` - JS-side wrappers for the artifacts located in `/compiled`, which - differs between the Node.js and web versions of o1js. Includes code for - setting up workers to support using `rayon` in Rust. + mina-signer (a pure TS package) to hash and sign transactions. Also includes + the `native/` conversion layer between OCaml/ML data structures and the + kimchi-napi FFI (shared by the native and wasm backends). +- `/js` - JS-side backend loaders for the artifacts located in `/compiled`, + which differ between the Node.js and web versions of o1js. Threading (rayon) + is handled by `@napi-rs/wasm-runtime` inside the compiled artifact, so these + loaders are thin. - `/lib` - miscellaneous low-level TypeScript, which underpins o1js and provides generic ways to connect with a proof system and blockchain protocol. - `/mina-transaction` - TS types and modules that specialize the generic tooling diff --git a/src/bindings/crypto/bindings.ts b/src/bindings/crypto/bindings.ts index 9716a20d68..5d5192629b 100644 --- a/src/bindings/crypto/bindings.ts +++ b/src/bindings/crypto/bindings.ts @@ -3,28 +3,21 @@ * It is exposed to JSOO by populating a global variable with an object. * It gets imported as the first thing in ../../bindings.js so that the global variable is ready by the time JSOO code gets executed. */ -import type * as rustNamespace from '../compiled/node_bindings/kimchi_wasm.cjs'; +import type * as rustNamespace from '../compiled/node_bindings/kimchi_napi.wasi.cjs'; import { prefixHashes, prefixHashesLegacy } from '../crypto/constants.js'; import { Bigint256Bindings } from './bindings/bigint256.js'; import { fieldsFromRustFlat, fieldsToRustFlat } from './bindings/conversion-base.js'; -import { conversionCore as wasmConversionCore } from './bindings/conversion-core.js'; -import { oraclesConversion as wasmOraclesConversion } from './bindings/conversion-oracles.js'; -import { proofConversion as wasmProofConversion } from './bindings/conversion-proof.js'; -import { verifierIndexConversion as wasmVerifierIndexConversion } from './bindings/conversion-verifier-index.js'; import { PallasBindings, VestaBindings } from './bindings/curve.js'; import { jsEnvironment } from './bindings/env.js'; import { FpBindings, FqBindings } from './bindings/field.js'; import { FpVectorBindings, FqVectorBindings } from './bindings/vector.js'; -import { srs as wasmSrs } from './bindings/srs.js'; -import { srs as napiSrs } from './native/napi-srs.js'; import { napiConversionCore } from './native/napi-conversion-core.js'; +import { napiOraclesConversion } from './native/napi-conversion-oracles.js'; import { napiProofConversion } from './native/napi-conversion-proof.js'; import { napiVerifierIndexConversion } from './native/napi-conversion-verifier-index.js'; -import { napiOraclesConversion } from './native/napi-conversion-oracles.js'; - -export { Napi, Wasm, RustConversion, getRustConversion }; - +import { srs as napiSrs } from './native/napi-srs.js'; +export { Napi, RustConversion, Wasm, getRustConversion }; const tsBindings = { jsEnvironment, @@ -44,44 +37,17 @@ const tsBindings = { // this is put in a global variable so that mina/src/lib/crypto/kimchi_bindings/js/bindings.js finds it (globalThis as any).__snarkyTsBindings = tsBindings; +// Both backends (the native .node build and the wasm32-wasip1-threads build) +// are napi-rs builds of the same kimchi-napi crate, so a single conversion +// layer serves both. type Rust = typeof rustNamespace; type Wasm = Rust; type Napi = Rust; -type BackendKind = 'wasm' | 'native'; - -// Whether or not native backend is in use -function getKimchiBackend(rust: Rust): BackendKind { - const backend = (rust as any).__kimchi_backend ?? (globalThis as any)?.__kimchi_backend; - return backend === 'native' ? 'native' : 'wasm'; -} -type WasmConversion = ReturnType; -type NapiConversion = ReturnType; - -type RustConversion = B extends 'wasm' - ? WasmConversion - : NapiConversion; +type RustConversion = ReturnType; function getRustConversion(rust: Rust): RustConversion { - return getKimchiBackend(rust) === 'wasm' - ? buildWasmRustConversion(rust) - : buildNapiRustConversion(rust); -} - -function buildWasmRustConversion(wasm: Rust) { - let core = wasmConversionCore(wasm); - let proof = wasmProofConversion(wasm, core); - let oracles = wasmOraclesConversion(wasm); - let verifierIndex = wasmVerifierIndexConversion(wasm, core); - - return { - fp: { ...core.fp, ...verifierIndex.fp, ...oracles.fp, ...proof.fp }, - fq: { ...core.fq, ...verifierIndex.fq, ...oracles.fq, ...proof.fq }, - fieldsToRustFlat, - fieldsFromRustFlat, - wireToRust: core.wireToRust, - mapMlArrayToRustVector: core.mapMlArrayToRustVector, - }; + return getConversionBundle(rust).conversion; } function buildNapiRustConversion(napi: Rust) { @@ -100,18 +66,21 @@ function buildNapiRustConversion(napi: Rust) { }; } -type ConversionBundle = { - kind: B; +type ConversionBundle = { rust: Rust; - conversion: RustConversion; - srs: B extends 'wasm' ? ReturnType : ReturnType; + conversion: RustConversion; + srs: ReturnType; }; -function getConversionBundle(rust: Rust): ConversionBundle { - if (getKimchiBackend(rust) === 'wasm') { - const conversion = buildWasmRustConversion(rust); - return { kind: 'wasm', rust, conversion, srs: wasmSrs(rust, conversion) }; - } +// cache the bundle per FFI module, so that repeated calls from JSOO don't +// rebuild the conversion tables +let bundleCache = new WeakMap(); + +function getConversionBundle(rust: Rust): ConversionBundle { + let cached = bundleCache.get(rust as object); + if (cached !== undefined) return cached; const conversion = buildNapiRustConversion(rust); - return { kind: 'native', rust, conversion, srs: napiSrs(rust, conversion) }; + const bundle = { rust, conversion, srs: napiSrs(rust, conversion) }; + bundleCache.set(rust as object, bundle); + return bundle; } diff --git a/src/bindings/crypto/bindings/bindings.unit-test.ts b/src/bindings/crypto/bindings/bindings.unit-test.ts deleted file mode 100644 index 48da4934df..0000000000 --- a/src/bindings/crypto/bindings/bindings.unit-test.ts +++ /dev/null @@ -1,226 +0,0 @@ -/** - * This file exhaustively tests JS implementations of `pasta_bindings.ml` for consistency. - * The TS impl is tested to be equivalent to the Rust/Wasm impl. - * - * "Equivalent" is defined as follows: - * - They throw errors for the same inputs - * - If they don't throw an error, outputs must be the same - */ -import { - Bigint256, - Bigint256Bindings, - MlBytes, - fromMlString, - mlBytesFromUint8Array, - mlBytesToUint8Array, - toMlStringAscii, -} from './bigint256.js'; -import { wasm } from '../../js/node/node-backend.js'; -import { Spec, ToSpec, FromSpec, defaultAssertEqual, id } from '../../../lib/testing/equivalent.js'; -import { Random } from '../../../lib/testing/property.js'; -import { - WasmAffine, - WasmProjective, - affineFromRust, - affineToRust, - fieldFromRust, - fieldToRust, -} from './conversion-base.js'; -import { equivalentRecord } from './test-utils.js'; -import { Field, FpBindings, FqBindings } from './field.js'; -import { MlBool, MlOption } from '../../../lib/ml/base.js'; -import { OrInfinity, PallasBindings, VestaBindings, toMlOrInfinity } from './curve.js'; -import { GroupProjective, Pallas, ProjectiveCurve, Vesta } from '../elliptic-curve.js'; -import { - WasmGPallas, - WasmGVesta, - WasmPallasGProjective, - WasmVestaGProjective, -} from '../../compiled/node_bindings/kimchi_wasm.cjs'; -import { FiniteField, Fp, Fq } from '../finite-field.js'; - -let number: ToSpec = { back: id }; -let numberLessThan = (max: number): FromSpec => ({ - rng: Random.nat(max - 1), - there: id, -}); -let uint31: Spec = { - rng: Random.nat(0x7fffffff), - there: id, - back: id, -}; - -let bigint256: Spec = { - rng: Random.map(Random.biguint(256), (x) => [0, x]), - there: fieldToRust, - back: fieldFromRust, -}; -let fp: Spec = { - rng: Random.map(Random.field, (x) => [0, x]), - there: fieldToRust, - back: fieldFromRust, -}; -let fq: Spec = { - rng: Random.map(Random.scalar, (x) => [0, x]), - there: fieldToRust, - back: fieldFromRust, -}; - -let boolean: Spec = { - rng: Random.map(Random.boolean, MlBool), - there: Boolean, - back: MlBool, -}; -let decimalString: Spec = { - rng: Random.map(Random.json.field, toMlStringAscii), - there: fromMlString, - back: toMlStringAscii, -}; -let bytes: Spec = { - rng: Random.map(Random.bytes(32), mlBytesFromUint8Array), - there: mlBytesToUint8Array, - back: mlBytesFromUint8Array, -}; - -function option(spec: Spec): Spec, S | undefined> { - return { - rng: Random.map(Random.oneOf(spec.rng, undefined), (o) => MlOption(o)), - there: (x) => MlOption.mapFrom(x, spec.there), - back: (x) => MlOption.mapTo(x, spec.back), - }; -} - -equivalentRecord( - Bigint256Bindings as Omit< - typeof Bigint256Bindings, - 'caml_bigint_256_print' | 'caml_bigint_256_to_string' - >, - wasm, - { - caml_bigint_256_of_numeral: undefined, // TODO - caml_bigint_256_of_decimal_string: { from: [decimalString], to: bigint256 }, - caml_bigint_256_num_limbs: { from: [], to: number }, - caml_bigint_256_bytes_per_limb: { from: [], to: number }, - caml_bigint_256_div: { from: [bigint256, bigint256], to: bigint256 }, - caml_bigint_256_compare: { from: [bigint256, bigint256], to: number }, - caml_bigint_256_test_bit: { - from: [bigint256, numberLessThan(256)], - to: boolean, - }, - caml_bigint_256_to_bytes: { from: [bigint256], to: bytes }, - caml_bigint_256_of_bytes: { from: [bytes], to: bigint256 }, - caml_bigint_256_deep_copy: { from: [bigint256], to: bigint256 }, - } -); - -// elliptic curve - -let pallas = projective( - Pallas, - Fq, - wasm.caml_pallas_affine_one, - wasm.caml_pallas_of_affine, - wasm.caml_pallas_to_affine -); -let pallasAffine = affine(Pallas, Fq, wasm.caml_pallas_affine_one); - -let vesta = projective( - Vesta, - Fp, - wasm.caml_vesta_affine_one, - wasm.caml_vesta_of_affine, - wasm.caml_vesta_to_affine -); -let vestaAffine = affine(Vesta, Fp, wasm.caml_vesta_affine_one); - -equivalentRecord(PallasBindings, wasm, { - caml_pallas_one: { from: [], to: pallas }, - caml_pallas_add: { from: [pallas, pallas], to: pallas }, - caml_pallas_sub: { from: [pallas, pallas], to: pallas }, - caml_pallas_negate: { from: [pallas], to: pallas }, - caml_pallas_double: { from: [pallas], to: pallas }, - caml_pallas_scale: { from: [pallas, fq], to: pallas }, - caml_pallas_random: undefined, // random outputs won't match - caml_pallas_rng: undefined, // random outputs won't match - caml_pallas_endo_base: { from: [], to: fp }, - caml_pallas_endo_scalar: { from: [], to: fq }, - caml_pallas_to_affine: { from: [pallas], to: pallasAffine }, - caml_pallas_of_affine: { from: [pallasAffine], to: pallas }, - caml_pallas_of_affine_coordinates: { from: [fp, fp], to: pallas }, - caml_pallas_affine_deep_copy: { from: [pallasAffine], to: pallasAffine }, -}); - -equivalentRecord(VestaBindings, wasm, { - caml_vesta_one: { from: [], to: vesta }, - caml_vesta_add: { from: [vesta, vesta], to: vesta }, - caml_vesta_sub: { from: [vesta, vesta], to: vesta }, - caml_vesta_negate: { from: [vesta], to: vesta }, - caml_vesta_double: { from: [vesta], to: vesta }, - caml_vesta_scale: { from: [vesta, fp], to: vesta }, - caml_vesta_random: undefined, // random outputs won't match - caml_vesta_rng: undefined, // random outputs won't match - caml_vesta_endo_base: { from: [], to: fq }, - caml_vesta_endo_scalar: { from: [], to: fp }, - caml_vesta_to_affine: { from: [vesta], to: vestaAffine }, - caml_vesta_of_affine: { from: [vestaAffine], to: vesta }, - caml_vesta_of_affine_coordinates: { from: [fq, fq], to: vesta }, - caml_vesta_affine_deep_copy: { from: [vestaAffine], to: vestaAffine }, -}); - -function projective( - Curve: ProjectiveCurve, - Scalar: FiniteField, - affineOne: () => WasmA, - projOfAffine: (a: WasmA) => WasmP, - projToAffine: (p: WasmP) => WasmA -): Spec { - let randomScaled = Random(() => Curve.scale(Curve.one, Scalar.random())); - - return { - rng: Random.oneOf(Curve.zero, Curve.one, randomScaled, randomScaled, randomScaled), - // excessively expensive to work around limited Rust API - only use for tests - there(p: GroupProjective): WasmP { - let { x, y, infinity } = Curve.toAffine(p); - let pAffineRust = affineOne(); - if (infinity) { - pAffineRust.infinity = true; - } else { - pAffineRust.x = fieldToRust([0, x]); - pAffineRust.y = fieldToRust([0, y]); - } - return projOfAffine(pAffineRust); - }, - back(p: WasmP): GroupProjective { - let pAffineRust = projToAffine(p); - if (pAffineRust.infinity) { - pAffineRust.free(); - return Curve.zero; - } else { - let [, x] = fieldFromRust(pAffineRust.x); - let [, y] = fieldFromRust(pAffineRust.y); - return Curve.fromAffine({ x, y, infinity: false }); - } - }, - // we have to relax equality since we always normalize Rust points for conversion, - // but TS points are not normalized - assertEqual(g, h, message) { - defaultAssertEqual(Curve.equal(g, h), true, message); - }, - }; -} - -function affine( - Curve: ProjectiveCurve, - Scalar: FiniteField, - affineOne: () => WasmA -): Spec { - let randomScaled = Random(() => Curve.scale(Curve.one, Scalar.random())); - let rngProjective = Random.oneOf(Curve.zero, Curve.one, randomScaled, randomScaled, randomScaled); - let rng = Random.map(rngProjective, (p) => toMlOrInfinity(Curve.toAffine(p))); - - return { - rng, - there: (p: OrInfinity) => affineToRust(p, affineOne), - back: affineFromRust, - }; -} diff --git a/src/bindings/crypto/bindings/conversion-base.ts b/src/bindings/crypto/bindings/conversion-base.ts index 895cfe5678..29dcc72145 100644 --- a/src/bindings/crypto/bindings/conversion-base.ts +++ b/src/bindings/crypto/bindings/conversion-base.ts @@ -1,17 +1,11 @@ import type { MlArray } from '../../../lib/ml/base.js'; -import type { - WasmGPallas, - WasmGVesta, - WasmPallasGProjective, - WasmVestaGProjective, -} from '../../compiled/node_bindings/kimchi_wasm.cjs'; +import type { WasmGPallas, WasmGVesta } from '../../compiled/node_bindings/kimchi_napi.wasi.cjs'; import { bigintToBytes32, bytesToBigint32 } from '../bigint-helpers.js'; import { Infinity, OrInfinity } from './curve.js'; import { Field } from './field.js'; export { WasmAffine, - WasmProjective, affineFromRust, affineToRust, fieldFromRust, @@ -71,18 +65,16 @@ function maybeFieldToRust(x?: Field): Uint8Array | undefined { return x && fieldToRust(x); } -// affine +// affine — napi `#[napi(object)]` points, i.e. plain { x, y, infinity } objects type WasmAffine = WasmGVesta | WasmGPallas; function affineFromRust(pt: A): OrInfinity { if (pt.infinity) { - pt.free(); return 0; } else { let x = fieldFromRust(pt.x); let y = fieldFromRust(pt.y); - pt.free(); return [0, [0, x, y]]; } } @@ -98,7 +90,3 @@ function affineToRust(pt: OrInfinity, makeAffine: () => A) } return res; } - -// projective - -type WasmProjective = WasmVestaGProjective | WasmPallasGProjective; diff --git a/src/bindings/crypto/bindings/conversion-core.ts b/src/bindings/crypto/bindings/conversion-core.ts deleted file mode 100644 index aba3725d88..0000000000 --- a/src/bindings/crypto/bindings/conversion-core.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { MlArray } from '../../../lib/ml/base.js'; -import type * as wasmNamespace from '../../compiled/node_bindings/kimchi_wasm.cjs'; -import type { - WasmFpGate, - WasmFpPolyComm, - WasmFqGate, - WasmFqPolyComm, - WasmGPallas, - WasmGVesta, -} from '../../compiled/node_bindings/kimchi_wasm.cjs'; -import { - WasmAffine, - affineFromRust, - affineToRust, - fieldsFromRustFlat, - fieldsToRustFlat, -} from './conversion-base.js'; -import { Gate, OrInfinity, PolyComm, Wire } from './kimchi-types.js'; -import { mapTuple } from './util.js'; - -export { - ConversionCore, - ConversionCores, - conversionCore, - freeOnFinalize, - intoRaw, - mapFromUintArray, - mapToUint32Array, - unwrap, - wrap, -}; - -// basic conversion functions for each field - -type wasm = typeof wasmNamespace; - -type WasmPolyComm = WasmFpPolyComm | WasmFqPolyComm; - -type WasmClasses = { - CommitmentCurve: typeof WasmGVesta | typeof WasmGPallas; - makeAffine: () => WasmAffine; - Gate: typeof WasmFpGate | typeof WasmFqGate; - PolyComm: typeof WasmFpPolyComm | typeof WasmFqPolyComm; -}; - -type ConversionCore = ReturnType; -type ConversionCores = ReturnType; - -function conversionCore(wasm: wasm) { - const fp = conversionCorePerField(wasm, { - CommitmentCurve: wasm.WasmGVesta, - makeAffine: wasm.caml_vesta_affine_one, - Gate: wasm.WasmFpGate, - PolyComm: wasm.WasmFpPolyComm, - }); - const fq = conversionCorePerField(wasm, { - CommitmentCurve: wasm.WasmGPallas, - makeAffine: wasm.caml_pallas_affine_one, - Gate: wasm.WasmFqGate, - PolyComm: wasm.WasmFqPolyComm, - }); - - return { - fp, - fq, - wireToRust: fp.wireToRust, // doesn't depend on the field - mapMlArrayToRustVector( - [, ...array]: MlArray, - map: (x: TMl) => TRust - ): Uint32Array { - // Transfer ownership to Rust when the wasm-bindgen wrapper supports it. - return mapToUint32Array(array, (x) => intoRaw(map(x))); - }, - }; -} - -function conversionCorePerField( - wasm: wasm, - { CommitmentCurve, makeAffine, Gate, PolyComm }: WasmClasses -) { - let self = { - wireToRust([, row, col]: Wire) { - return wasm.Wire.create(row, col); - }, - - vectorToRust: fieldsToRustFlat, - vectorFromRust: fieldsFromRustFlat, - - gateToRust(gate: Gate) { - let [, typ, [, ...wires], coeffs] = gate; - let rustWires = new wasm.WasmGateWires(...mapTuple(wires, self.wireToRust)); - let rustCoeffs = fieldsToRustFlat(coeffs); - return new Gate(typ, rustWires, rustCoeffs); - }, - gateFromRust(wasmGate: WasmFpGate | WasmFqGate) { - // note: this was never used and the old implementation was wrong - // (accessed non-existent fields on wasmGate) - throw Error('gateFromRust not implemented'); - }, - - pointToRust(point: OrInfinity) { - return affineToRust(point, makeAffine); - }, - pointFromRust: affineFromRust, - - pointsToRust([, ...points]: MlArray): Uint32Array { - return mapToUint32Array(points, (point) => intoRaw(self.pointToRust(point))); - }, - pointsFromRust(points: Uint32Array): MlArray { - let arr = mapFromUintArray(points, (ptr) => affineFromRust(wrap(ptr, CommitmentCurve))); - return [0, ...arr]; - }, - - polyCommToRust(polyComm: PolyComm): WasmPolyComm { - let [, camlElems] = polyComm; - let rustShifted = undefined; - let rustUnshifted = self.pointsToRust(camlElems); - return new PolyComm(rustUnshifted, rustShifted); - }, - polyCommFromRust(polyComm: WasmPolyComm): PolyComm { - let rustUnshifted = polyComm.unshifted; - let mlUnshifted = mapFromUintArray(rustUnshifted, (ptr) => { - return affineFromRust(wrap(ptr, CommitmentCurve)); - }); - // Real wasm-bindgen wrappers own resources and should be released now. - // Synthetic wrappers created by `wrap()` only borrow raw pointers and - // must not call free(). - if (!(polyComm as any).__o1js_wrapped_ptr && typeof (polyComm as any).free === 'function') { - (polyComm as any).free(); - } - return [0, [0, ...mlUnshifted]]; - }, - - polyCommsToRust([, ...comms]: MlArray): Uint32Array { - return mapToUint32Array(comms, (c) => intoRaw(self.polyCommToRust(c))); - }, - polyCommsFromRust(rustComms: Uint32Array): MlArray { - let comms = mapFromUintArray(rustComms, (ptr) => self.polyCommFromRust(wrap(ptr, PolyComm))); - return [0, ...comms]; - }, - }; - - return self; -} - -// generic rust helpers - -type Freeable = { free(): void }; -type Constructor = { prototype: T }; - -function wrap(ptr: number, Class: Constructor): T { - const obj = Object.create(Class.prototype); - obj.__wbg_ptr = ptr; - obj.__o1js_wrapped_ptr = true; - return obj; -} -function unwrap(obj: T): number { - // Beware: caller may need to do finalizer things to avoid these - // pointers disappearing out from under us. - let ptr = (obj as any).__wbg_ptr; - if (ptr === undefined) throw Error('unwrap: missing ptr'); - return ptr; -} - -// Return a pointer suitable for passing to Rust FFI. -// - wasm-bindgen wrappers: call `__destroy_into_raw()` to transfer ownership -// to Rust and detach JS-side finalization. -// - plain pointer wrappers (no destroy hook): fall back to `unwrap()`. -// This prevents JS and Rust from both believing they own the same allocation. -function intoRaw(obj: T): number { - let destroyIntoRaw = (obj as any)?.__destroy_into_raw; - if (typeof destroyIntoRaw === 'function') return destroyIntoRaw.call(obj); - return unwrap(obj); -} - -const registry = new FinalizationRegistry((ptr: Freeable) => { - ptr.free(); -}); -function freeOnFinalize(instance: T) { - // wasm-bindgen wrappers already manage finalization/ownership internally. - // Adding another free path here can double-free or free borrowed values. - if (typeof (instance as any)?.__destroy_into_raw === 'function') { - return instance; - } - // We want `instance` to be garbage-collected naturally, but still release - // its Rust allocation when that happens. - // - // FinalizationRegistry cannot hold `instance` itself as the representative - // value, because that would keep it alive. Instead we create a tiny stand-in - // that only carries the prototype + raw pointer, which is enough to call - // `.free()` once `instance` is collected. - // - // We intentionally avoid `__wrap()` here, because that constructor path is - // for normal wasm-bindgen object creation and can interact with ownership - // bookkeeping we do not want in this finalizer surrogate. - let instanceRepresentative = Object.create((instance as any).constructor.prototype); - (instanceRepresentative as any).__wbg_ptr = (instance as any).__wbg_ptr; - registry.register(instance, instanceRepresentative, instance); - return instance; -} - -function mapFromUintArray(array: Uint32Array | Uint8Array, map: (i: number) => T) { - let n = array.length; - let result: T[] = Array(n); - for (let i = 0; i < n; i++) { - result[i] = map(array[i]); - } - return result; -} - -function mapToUint32Array(array: T[], map: (t: T) => number) { - let n = array.length; - let result = new Uint32Array(n); - for (let i = 0; i < n; i++) { - result[i] = map(array[i]); - } - return result; -} diff --git a/src/bindings/crypto/bindings/conversion-oracles.ts b/src/bindings/crypto/bindings/conversion-oracles.ts deleted file mode 100644 index 96fb53923b..0000000000 --- a/src/bindings/crypto/bindings/conversion-oracles.ts +++ /dev/null @@ -1,121 +0,0 @@ -import type { - WasmFpOracles, - WasmFpRandomOracles, - WasmFqOracles, - WasmFqRandomOracles, -} from '../../compiled/node_bindings/kimchi_wasm.cjs'; -import type * as wasmNamespace from '../../compiled/node_bindings/kimchi_wasm.cjs'; -import { MlOption } from '../../../lib/ml/base.js'; -import { Field, Oracles, RandomOracles, ScalarChallenge } from './kimchi-types.js'; -import { - fieldFromRust, - fieldToRust, - fieldsFromRustFlat, - fieldsToRustFlat, - maybeFieldToRust, -} from './conversion-base.js'; - -export { oraclesConversion }; - -type wasm = typeof wasmNamespace; - -type WasmRandomOracles = WasmFpRandomOracles | WasmFqRandomOracles; -type WasmOracles = WasmFpOracles | WasmFqOracles; - -type WasmClasses = { - RandomOracles: typeof WasmFpRandomOracles | typeof WasmFqRandomOracles; - Oracles: typeof WasmFpOracles | typeof WasmFqOracles; -}; - -function oraclesConversion(wasm: wasm) { - return { - fp: oraclesConversionPerField({ - RandomOracles: wasm.WasmFpRandomOracles, - Oracles: wasm.WasmFpOracles, - }), - fq: oraclesConversionPerField({ - RandomOracles: wasm.WasmFqRandomOracles, - Oracles: wasm.WasmFqOracles, - }), - }; -} - -function oraclesConversionPerField({ RandomOracles, Oracles }: WasmClasses) { - function randomOraclesToRust(ro: RandomOracles): WasmRandomOracles { - let jointCombinerMl = MlOption.from(ro[1]); - let jointCombinerChal = maybeFieldToRust(jointCombinerMl?.[1][1]); - let jointCombiner = maybeFieldToRust(jointCombinerMl?.[2]); - let beta = fieldToRust(ro[2]); - let gamma = fieldToRust(ro[3]); - let alphaChal = fieldToRust(ro[4][1]); - let alpha = fieldToRust(ro[5]); - let zeta = fieldToRust(ro[6]); - let v = fieldToRust(ro[7]); - let u = fieldToRust(ro[8]); - let zetaChal = fieldToRust(ro[9][1]); - let vChal = fieldToRust(ro[10][1]); - let uChal = fieldToRust(ro[11][1]); - return new RandomOracles( - jointCombinerChal, - jointCombiner, - beta, - gamma, - alphaChal, - alpha, - zeta, - v, - u, - zetaChal, - vChal, - uChal - ); - } - function randomOraclesFromRust(ro: WasmRandomOracles): RandomOracles { - let jointCombinerChal = ro.joint_combiner_chal; - let jointCombiner = ro.joint_combiner; - let jointCombinerOption = MlOption<[0, ScalarChallenge, Field]>( - jointCombinerChal && - jointCombiner && [0, [0, fieldFromRust(jointCombinerChal)], fieldFromRust(jointCombiner)] - ); - let mlRo: RandomOracles = [ - 0, - jointCombinerOption, - fieldFromRust(ro.beta), - fieldFromRust(ro.gamma), - [0, fieldFromRust(ro.alpha_chal)], - fieldFromRust(ro.alpha), - fieldFromRust(ro.zeta), - fieldFromRust(ro.v), - fieldFromRust(ro.u), - [0, fieldFromRust(ro.zeta_chal)], - [0, fieldFromRust(ro.v_chal)], - [0, fieldFromRust(ro.u_chal)], - ]; - ro.free(); - return mlRo; - } - - return { - oraclesToRust(oracles: Oracles): WasmOracles { - let [, o, pEval, openingPrechallenges, digestBeforeEvaluations] = oracles; - return new Oracles( - randomOraclesToRust(o), - fieldToRust(pEval[1]), - fieldToRust(pEval[2]), - fieldsToRustFlat(openingPrechallenges), - fieldToRust(digestBeforeEvaluations) - ); - }, - oraclesFromRust(oracles: WasmOracles): Oracles { - let mlOracles: Oracles = [ - 0, - randomOraclesFromRust(oracles.o), - [0, fieldFromRust(oracles.p_eval0), fieldFromRust(oracles.p_eval1)], - fieldsFromRustFlat(oracles.opening_prechallenges), - fieldFromRust(oracles.digest_before_evaluations), - ]; - oracles.free(); - return mlOracles; - }, - }; -} diff --git a/src/bindings/crypto/bindings/conversion-proof.ts b/src/bindings/crypto/bindings/conversion-proof.ts deleted file mode 100644 index f8580a6168..0000000000 --- a/src/bindings/crypto/bindings/conversion-proof.ts +++ /dev/null @@ -1,271 +0,0 @@ -import { MlArray, MlOption, MlTuple } from '../../../lib/ml/base.js'; -import type * as wasmNamespace from '../../compiled/node_bindings/kimchi_wasm.cjs'; -import type { - WasmFpLookupCommitments, - WasmFpOpeningProof, - WasmFpProverCommitments, - WasmFpProverProof, - WasmFpRuntimeTable, - WasmFqLookupCommitments, - WasmFqOpeningProof, - WasmFqProverCommitments, - WasmFqProverProof, - WasmFqRuntimeTable, - WasmPastaFpLookupTable, - WasmPastaFpRuntimeTableCfg, - WasmPastaFqLookupTable, - WasmPastaFqRuntimeTableCfg, - WasmVecVecFp, - WasmVecVecFq, -} from '../../compiled/node_bindings/kimchi_wasm.cjs'; -import { - fieldFromRust, - fieldToRust, - fieldsFromRustFlat, - fieldsToRustFlat, -} from './conversion-base.js'; -import { ConversionCore, ConversionCores, intoRaw, mapToUint32Array } from './conversion-core.js'; -import { - proofEvaluationsToRust, - proofEvaluationsFromRust, - pointEvalsOptionToRust, - pointEvalsOptionFromRust, -} from './conversion-proof-shared.js'; -import type { - LookupCommitments, - LookupTable, - OpeningProof, - OrInfinity, - PointEvaluations, - PolyComm, - ProofEvaluations, - ProofWithPublic, - ProverCommitments, - ProverProof, - RecursionChallenge, - RuntimeTable, - RuntimeTableCfg, -} from './kimchi-types.js'; - -export { proofConversion }; - -type WasmProofEvaluations = [ - 0, - MlOption>, - ...RemoveLeadingZero>, -]; - -type wasm = typeof wasmNamespace; - -type WasmProverCommitments = WasmFpProverCommitments | WasmFqProverCommitments; -type WasmOpeningProof = WasmFpOpeningProof | WasmFqOpeningProof; -type WasmProverProof = WasmFpProverProof | WasmFqProverProof; -type WasmLookupCommitments = WasmFpLookupCommitments | WasmFqLookupCommitments; -type WasmRuntimeTable = WasmFpRuntimeTable | WasmFqRuntimeTable; -type WasmRuntimeTableCfg = WasmPastaFpRuntimeTableCfg | WasmPastaFqRuntimeTableCfg; -type WasmLookupTable = WasmPastaFpLookupTable | WasmPastaFqLookupTable; - -type WasmClasses = { - ProverCommitments: typeof WasmFpProverCommitments | typeof WasmFqProverCommitments; - OpeningProof: typeof WasmFpOpeningProof | typeof WasmFqOpeningProof; - VecVec: typeof WasmVecVecFp | typeof WasmVecVecFq; - ProverProof: typeof WasmFpProverProof | typeof WasmFqProverProof; - LookupCommitments: typeof WasmFpLookupCommitments | typeof WasmFqLookupCommitments; - RuntimeTable: typeof WasmFpRuntimeTable | typeof WasmFqRuntimeTable; - RuntimeTableCfg: typeof WasmPastaFpRuntimeTableCfg | typeof WasmPastaFqRuntimeTableCfg; - LookupTable: typeof WasmPastaFpLookupTable | typeof WasmPastaFqLookupTable; -}; - -function proofConversion(wasm: wasm, core: ConversionCores) { - return { - fp: proofConversionPerField(core.fp, { - ProverCommitments: wasm.WasmFpProverCommitments, - OpeningProof: wasm.WasmFpOpeningProof, - VecVec: wasm.WasmVecVecFp, - ProverProof: wasm.WasmFpProverProof, - LookupCommitments: wasm.WasmFpLookupCommitments, - RuntimeTable: wasm.WasmFpRuntimeTable, - RuntimeTableCfg: wasm.WasmPastaFpRuntimeTableCfg, - LookupTable: wasm.WasmPastaFpLookupTable, - }), - fq: proofConversionPerField(core.fq, { - ProverCommitments: wasm.WasmFqProverCommitments, - OpeningProof: wasm.WasmFqOpeningProof, - VecVec: wasm.WasmVecVecFq, - ProverProof: wasm.WasmFqProverProof, - LookupCommitments: wasm.WasmFqLookupCommitments, - RuntimeTable: wasm.WasmFqRuntimeTable, - RuntimeTableCfg: wasm.WasmPastaFqRuntimeTableCfg, - LookupTable: wasm.WasmPastaFqLookupTable, - }), - }; -} - -function proofConversionPerField( - core: ConversionCore, - { - ProverCommitments, - OpeningProof, - VecVec, - ProverProof, - LookupCommitments, - RuntimeTable, - RuntimeTableCfg, - LookupTable, - }: WasmClasses -) { - function commitmentsToRust(commitments: ProverCommitments): WasmProverCommitments { - let wComm = core.polyCommsToRust(commitments[1]); - let zComm = core.polyCommToRust(commitments[2]); - let tComm = core.polyCommToRust(commitments[3]); - let lookup = MlOption.mapFrom(commitments[4], lookupCommitmentsToRust); - return new ProverCommitments(wComm, zComm, tComm, lookup); - } - function commitmentsFromRust(commitments: WasmProverCommitments): ProverCommitments { - let wComm = core.polyCommsFromRust(commitments.w_comm); - let zComm = core.polyCommFromRust(commitments.z_comm); - let tComm = core.polyCommFromRust(commitments.t_comm); - let lookup = MlOption.mapTo(commitments.lookup, lookupCommitmentsFromRust); - commitments.free(); - return [0, wComm as MlTuple, zComm, tComm, lookup]; - } - - function lookupCommitmentsToRust(lookup: LookupCommitments): WasmLookupCommitments { - let sorted = core.polyCommsToRust(lookup[1]); - let aggreg = core.polyCommToRust(lookup[2]); - let runtime = MlOption.mapFrom(lookup[3], core.polyCommToRust); - return new LookupCommitments(sorted, aggreg, runtime); - } - function lookupCommitmentsFromRust(lookup: WasmLookupCommitments): LookupCommitments { - let sorted = core.polyCommsFromRust(lookup.sorted); - let aggreg = core.polyCommFromRust(lookup.aggreg); - let runtime = MlOption.mapTo(lookup.runtime, core.polyCommFromRust); - lookup.free(); - return [0, sorted, aggreg, runtime]; - } - - function openingProofToRust(proof: OpeningProof): WasmOpeningProof { - let [_, [, ...lr], delta, z1, z2, sg] = proof; - // We pass l and r as separate vectors over the FFI - let l: MlArray = [0]; - let r: MlArray = [0]; - for (let [, li, ri] of lr) { - l.push(li); - r.push(ri); - } - return new OpeningProof( - core.pointsToRust(l), - core.pointsToRust(r), - core.pointToRust(delta), - fieldToRust(z1), - fieldToRust(z2), - core.pointToRust(sg) - ); - } - function openingProofFromRust(proof: WasmOpeningProof): OpeningProof { - let [, ...l] = core.pointsFromRust(proof.lr_0); - let [, ...r] = core.pointsFromRust(proof.lr_1); - let n = l.length; - if (n !== r.length) throw Error('openingProofFromRust: l and r length mismatch.'); - let lr = l.map<[0, OrInfinity, OrInfinity]>((li, i) => [0, li, r[i]]); - let delta = core.pointFromRust(proof.delta); - let z1 = fieldFromRust(proof.z1); - let z2 = fieldFromRust(proof.z2); - let sg = core.pointFromRust(proof.sg); - proof.free(); - return [0, [0, ...lr], delta, z1, z2, sg]; - } - - function runtimeTableToRust([, id, data]: RuntimeTable): WasmRuntimeTable { - return new RuntimeTable(id, core.vectorToRust(data)); - } - - function runtimeTableCfgToRust([, id, firstColumn]: RuntimeTableCfg): WasmRuntimeTableCfg { - return new RuntimeTableCfg(id, core.vectorToRust(firstColumn)); - } - - function lookupTableToRust([, id, [, ...data]]: LookupTable): WasmLookupTable { - let n = data.length; - let wasmData = new VecVec(n); - for (let i = 0; i < n; i++) { - wasmData.push(fieldsToRustFlat(data[i])); - } - return new LookupTable(id, wasmData); - } - - return { - proofToRust([, public_evals, proof]: ProofWithPublic): WasmProverProof { - let commitments = commitmentsToRust(proof[1]); - let openingProof = openingProofToRust(proof[2]); - let [, ...evals] = proofEvaluationsToRust(proof[3]); - let publicEvals = pointEvalsOptionToRust(public_evals); - // TODO typed as `any` in wasm-bindgen, this has the correct type - let evalsActual: WasmProofEvaluations = [0, publicEvals, ...evals]; - - let ftEval1 = fieldToRust(proof[4]); - let public_ = fieldsToRustFlat(proof[5]); - let [, ...prevChallenges] = proof[6]; - let n = prevChallenges.length; - let prevChallengeScalars = new VecVec(n); - let prevChallengeCommsMl: MlArray = [0]; - for (let [, scalars, comms] of prevChallenges) { - prevChallengeScalars.push(fieldsToRustFlat(scalars)); - prevChallengeCommsMl.push(comms); - } - let prevChallengeComms = core.polyCommsToRust(prevChallengeCommsMl); - return new ProverProof( - commitments, - openingProof, - evalsActual, - ftEval1, - public_, - prevChallengeScalars, - prevChallengeComms - ); - }, - proofFromRust(wasmProof: WasmProverProof): ProofWithPublic { - let commitments = commitmentsFromRust(wasmProof.commitments); - let openingProof = openingProofFromRust(wasmProof.proof); - // TODO typed as `any` in wasm-bindgen, this is the correct type - let [, wasmPublicEvals, ...wasmEvals]: WasmProofEvaluations = wasmProof.evals; - let publicEvals = pointEvalsOptionFromRust(wasmPublicEvals); - let evals = proofEvaluationsFromRust([0, ...wasmEvals]); - - let ftEval1 = fieldFromRust(wasmProof.ft_eval1); - let public_ = fieldsFromRustFlat(wasmProof.public_); - let prevChallengeScalars = wasmProof.prev_challenges_scalars; - let [, ...prevChallengeComms] = core.polyCommsFromRust(wasmProof.prev_challenges_comms); - let prevChallenges = prevChallengeComms.map((comms, i) => { - let scalars = fieldsFromRustFlat(prevChallengeScalars.get(i)); - return [0, scalars, comms]; - }); - wasmProof.free(); - let proof: ProverProof = [ - 0, - commitments, - openingProof, - evals, - ftEval1, - public_, - [0, ...prevChallenges], - ]; - return [0, publicEvals, proof]; - }, - - runtimeTablesToRust([, ...tables]: MlArray): Uint32Array { - return mapToUint32Array(tables, (table) => intoRaw(runtimeTableToRust(table))); - }, - - runtimeTableCfgsToRust([, ...tableCfgs]: MlArray): Uint32Array { - return mapToUint32Array(tableCfgs, (tableCfg) => intoRaw(runtimeTableCfgToRust(tableCfg))); - }, - - lookupTablesToRust([, ...tables]: MlArray) { - return mapToUint32Array(tables, (table) => intoRaw(lookupTableToRust(table))); - }, - }; -} - -// helper - -type RemoveLeadingZero = T extends [0, ...infer U] ? U : never; diff --git a/src/bindings/crypto/bindings/conversion-verifier-index.ts b/src/bindings/crypto/bindings/conversion-verifier-index.ts deleted file mode 100644 index 39b88f7d29..0000000000 --- a/src/bindings/crypto/bindings/conversion-verifier-index.ts +++ /dev/null @@ -1,291 +0,0 @@ -import { MlArray, MlBool, MlOption } from '../../../lib/ml/base.js'; -import type * as wasmNamespace from '../../compiled/node_bindings/kimchi_wasm.cjs'; -import type { - WasmFpDomain, - WasmFpLookupSelectors, - WasmFpLookupVerifierIndex, - WasmFpPlonkVerificationEvals, - WasmFpPlonkVerifierIndex, - WasmFpShifts, - WasmFqDomain, - WasmFqLookupSelectors, - WasmFqLookupVerifierIndex, - WasmFqPlonkVerificationEvals, - WasmFqPlonkVerifierIndex, - WasmFqShifts, - LookupInfo as WasmLookupInfo, -} from '../../compiled/node_bindings/kimchi_wasm.cjs'; -import { fieldFromRust, fieldToRust } from './conversion-base.js'; -import { ConversionCore, ConversionCores, freeOnFinalize } from './conversion-core.js'; -import { Domain, Field, PolyComm, VerificationEvals, VerifierIndex } from './kimchi-types.js'; -import { Lookup, LookupInfo, LookupSelectors } from './lookup.js'; - -export { verifierIndexConversion }; - -type wasm = typeof wasmNamespace; - -type WasmDomain = WasmFpDomain | WasmFqDomain; -type WasmVerificationEvals = WasmFpPlonkVerificationEvals | WasmFqPlonkVerificationEvals; -type WasmShifts = WasmFpShifts | WasmFqShifts; -type WasmVerifierIndex = WasmFpPlonkVerifierIndex | WasmFqPlonkVerifierIndex; - -type WasmLookupVerifierIndex = WasmFpLookupVerifierIndex | WasmFqLookupVerifierIndex; -type WasmLookupSelector = WasmFpLookupSelectors | WasmFqLookupSelectors; - -type WasmClasses = { - Domain: typeof WasmFpDomain | typeof WasmFqDomain; - VerificationEvals: typeof WasmFpPlonkVerificationEvals | typeof WasmFqPlonkVerificationEvals; - Shifts: typeof WasmFpShifts | typeof WasmFqShifts; - VerifierIndex: typeof WasmFpPlonkVerifierIndex | typeof WasmFqPlonkVerifierIndex; - LookupVerifierIndex: typeof WasmFpLookupVerifierIndex | typeof WasmFqLookupVerifierIndex; - LookupSelector: typeof WasmFpLookupSelectors | typeof WasmFqLookupSelectors; -}; - -function verifierIndexConversion(wasm: wasm, core: ConversionCores) { - return { - fp: verifierIndexConversionPerField(wasm, core.fp, { - Domain: wasm.WasmFpDomain, - VerificationEvals: wasm.WasmFpPlonkVerificationEvals, - Shifts: wasm.WasmFpShifts, - VerifierIndex: wasm.WasmFpPlonkVerifierIndex, - LookupVerifierIndex: wasm.WasmFpLookupVerifierIndex, - LookupSelector: wasm.WasmFpLookupSelectors, - }), - fq: verifierIndexConversionPerField(wasm, core.fq, { - Domain: wasm.WasmFqDomain, - VerificationEvals: wasm.WasmFqPlonkVerificationEvals, - Shifts: wasm.WasmFqShifts, - VerifierIndex: wasm.WasmFqPlonkVerifierIndex, - LookupVerifierIndex: wasm.WasmFqLookupVerifierIndex, - LookupSelector: wasm.WasmFqLookupSelectors, - }), - }; -} - -function verifierIndexConversionPerField( - wasm: wasm, - core: ConversionCore, - { - Domain, - VerificationEvals, - Shifts, - VerifierIndex, - LookupVerifierIndex, - LookupSelector, - }: WasmClasses -) { - function domainToRust([, logSizeOfGroup, groupGen]: Domain): WasmDomain { - return new Domain(logSizeOfGroup, fieldToRust(groupGen)); - } - function domainFromRust(domain: WasmDomain): Domain { - let logSizeOfGroup = domain.log_size_of_group; - let groupGen = fieldFromRust(domain.group_gen); - domain.free(); - return [0, logSizeOfGroup, groupGen]; - } - - function verificationEvalsToRust(evals: VerificationEvals): WasmVerificationEvals { - let sigmaComm = core.polyCommsToRust(evals[1]); - let coefficientsComm = core.polyCommsToRust(evals[2]); - let genericComm = core.polyCommToRust(evals[3]); - let psmComm = core.polyCommToRust(evals[4]); - let completeAddComm = core.polyCommToRust(evals[5]); - let mulComm = core.polyCommToRust(evals[6]); - let emulComm = core.polyCommToRust(evals[7]); - let endomulScalarComm = core.polyCommToRust(evals[8]); - let xorComm = MlOption.mapFrom(evals[9], core.polyCommToRust); - let rangeCheck0Comm = MlOption.mapFrom(evals[10], core.polyCommToRust); - let rangeCheck1Comm = MlOption.mapFrom(evals[11], core.polyCommToRust); - let foreignFieldAddComm = MlOption.mapFrom(evals[12], core.polyCommToRust); - let foreignFieldMulComm = MlOption.mapFrom(evals[13], core.polyCommToRust); - let rotComm = MlOption.mapFrom(evals[14], core.polyCommToRust); - return new VerificationEvals( - sigmaComm, - coefficientsComm, - genericComm, - psmComm, - completeAddComm, - mulComm, - emulComm, - endomulScalarComm, - xorComm, - rangeCheck0Comm, - rangeCheck1Comm, - foreignFieldAddComm, - foreignFieldMulComm, - rotComm - ); - } - function verificationEvalsFromRust(evals: WasmVerificationEvals): VerificationEvals { - let mlEvals: VerificationEvals = [ - 0, - core.polyCommsFromRust(evals.sigma_comm), - core.polyCommsFromRust(evals.coefficients_comm), - core.polyCommFromRust(evals.generic_comm), - core.polyCommFromRust(evals.psm_comm), - core.polyCommFromRust(evals.complete_add_comm), - core.polyCommFromRust(evals.mul_comm), - core.polyCommFromRust(evals.emul_comm), - core.polyCommFromRust(evals.endomul_scalar_comm), - MlOption.mapTo(evals.xor_comm, core.polyCommFromRust), - MlOption.mapTo(evals.range_check0_comm, core.polyCommFromRust), - MlOption.mapTo(evals.range_check1_comm, core.polyCommFromRust), - MlOption.mapTo(evals.foreign_field_add_comm, core.polyCommFromRust), - MlOption.mapTo(evals.foreign_field_mul_comm, core.polyCommFromRust), - MlOption.mapTo(evals.rot_comm, core.polyCommFromRust), - ]; - evals.free(); - return mlEvals; - } - - function lookupVerifierIndexToRust(lookup: Lookup): WasmLookupVerifierIndex { - let [ - , - joint_lookup_used, - lookup_table, - selectors, - table_ids, - lookup_info, - runtime_tables_selector, - ] = lookup; - return new LookupVerifierIndex( - MlBool.from(joint_lookup_used), - core.polyCommsToRust(lookup_table), - lookupSelectorsToRust(selectors), - MlOption.mapFrom(table_ids, core.polyCommToRust), - lookupInfoToRust(lookup_info), - MlOption.mapFrom(runtime_tables_selector, core.polyCommToRust) - ); - } - function lookupVerifierIndexFromRust(lookup: WasmLookupVerifierIndex): Lookup { - let mlLookup: Lookup = [ - 0, - MlBool(lookup.joint_lookup_used), - core.polyCommsFromRust(lookup.lookup_table), - lookupSelectorsFromRust(lookup.lookup_selectors), - MlOption.mapTo(lookup.table_ids, core.polyCommFromRust), - lookupInfoFromRust(lookup.lookup_info), - MlOption.mapTo(lookup.runtime_tables_selector, core.polyCommFromRust), - ]; - lookup.free(); - return mlLookup; - } - - function lookupSelectorsToRust([ - , - lookup, - xor, - range_check, - ffmul, - ]: LookupSelectors): WasmLookupSelector { - return new LookupSelector( - MlOption.mapFrom(xor, core.polyCommToRust), - MlOption.mapFrom(lookup, core.polyCommToRust), - MlOption.mapFrom(range_check, core.polyCommToRust), - MlOption.mapFrom(ffmul, core.polyCommToRust) - ); - } - function lookupSelectorsFromRust(selector: WasmLookupSelector): LookupSelectors { - let lookup = MlOption.mapTo(selector.lookup, core.polyCommFromRust); - let xor = MlOption.mapTo(selector.xor, core.polyCommFromRust); - let range_check = MlOption.mapTo(selector.range_check, core.polyCommFromRust); - let ffmul = MlOption.mapTo(selector.ffmul, core.polyCommFromRust); - selector.free(); - return [0, lookup, xor, range_check, ffmul]; - } - - function lookupInfoToRust([, maxPerRow, maxJointSize, features]: LookupInfo): WasmLookupInfo { - let [, patterns, joint_lookup_used, uses_runtime_tables] = features; - let [, xor, lookup, range_check, foreign_field_mul] = patterns; - let wasmPatterns = new wasm.LookupPatterns( - MlBool.from(xor), - MlBool.from(lookup), - MlBool.from(range_check), - MlBool.from(foreign_field_mul) - ); - let wasmFeatures = new wasm.LookupFeatures( - wasmPatterns, - MlBool.from(joint_lookup_used), - MlBool.from(uses_runtime_tables) - ); - return new wasm.LookupInfo(maxPerRow, maxJointSize, wasmFeatures); - } - function lookupInfoFromRust(info: WasmLookupInfo): LookupInfo { - let features = info.features; - let patterns = features.patterns; - let mlInfo: LookupInfo = [ - 0, - info.max_per_row, - info.max_joint_size, - [ - 0, - [ - 0, - MlBool(patterns.xor), - MlBool(patterns.lookup), - MlBool(patterns.range_check), - MlBool(patterns.foreign_field_mul), - ], - MlBool(features.joint_lookup_used), - MlBool(features.uses_runtime_tables), - ], - ]; - info.free(); - return mlInfo; - } - - let self = { - shiftsToRust([, ...shifts]: MlArray): WasmShifts { - let s = shifts.map((s) => fieldToRust(s)); - return new Shifts(s[0], s[1], s[2], s[3], s[4], s[5], s[6]); - }, - shiftsFromRust(s: WasmShifts): MlArray { - let shifts = [s.s0, s.s1, s.s2, s.s3, s.s4, s.s5, s.s6]; - if (typeof (s as { free?: () => void }).free === 'function') { - s.free(); - } - return [0, ...shifts.map(fieldFromRust)]; - }, - - verifierIndexToRust(vk: VerifierIndex): WasmVerifierIndex { - let domain = domainToRust(vk[1]); - let maxPolySize = vk[2]; - let nPublic = vk[3]; - let prevChallenges = vk[4]; - let srs = vk[5]; - let evals = verificationEvalsToRust(vk[6]); - let shifts = self.shiftsToRust(vk[7]); - let lookupIndex = MlOption.mapFrom(vk[8], lookupVerifierIndexToRust); - let zkRows = vk[9]; - return new VerifierIndex( - domain, - maxPolySize, - nPublic, - prevChallenges, - srs, - evals, - shifts, - lookupIndex, - zkRows - ); - }, - verifierIndexFromRust(vk: WasmVerifierIndex): VerifierIndex { - let mlVk: VerifierIndex = [ - 0, - domainFromRust(vk.domain), - vk.max_poly_size, - vk.public_, - vk.prev_challenges, - freeOnFinalize(vk.srs), - verificationEvalsFromRust(vk.evals), - self.shiftsFromRust(vk.shifts), - MlOption.mapTo(vk.lookup_index, lookupVerifierIndexFromRust), - vk.zk_rows, - ]; - vk.free(); - return mlVk; - }, - }; - - return self; -} diff --git a/src/bindings/crypto/bindings/gate-vector-napi.unit-test.ts b/src/bindings/crypto/bindings/gate-vector-napi.unit-test.ts index 5197840bb8..d8776c328f 100644 --- a/src/bindings/crypto/bindings/gate-vector-napi.unit-test.ts +++ b/src/bindings/crypto/bindings/gate-vector-napi.unit-test.ts @@ -11,6 +11,8 @@ function loadNative() { `../../../../../native/${slug}/kimchi_napi.node`, '../../compiled/node_bindings/kimchi_napi.node', '../../compiled/_node_bindings/kimchi_napi.node', + // the wasm32-wasip1-threads build exposes the same napi module + '../../compiled/node_bindings/kimchi_napi.wasi.cjs', ]; for (const path of candidates) { try { @@ -24,7 +26,7 @@ function loadNative() { if (process.env.O1JS_BACKEND === 'native') { throw new Error('kimchi_napi.node not found but O1JS_BACKEND=native is set'); } - console.warn('kimchi_napi.node not found, skipping gate-vector-napi test'); + console.warn('kimchi napi module not found, skipping gate-vector-napi test'); process.exit(0); } diff --git a/src/bindings/crypto/bindings/kimchi-types.ts b/src/bindings/crypto/bindings/kimchi-types.ts index 54e511225a..b867dbec84 100644 --- a/src/bindings/crypto/bindings/kimchi-types.ts +++ b/src/bindings/crypto/bindings/kimchi-types.ts @@ -5,7 +5,7 @@ import type { Lookup } from './lookup.js'; import type { MlArray, MlOption, MlTuple } from '../../../lib/ml/base.js'; import type { OrInfinity } from './curve.js'; import type { Field } from './field.js'; -import type { WasmFpSrs, WasmFqSrs } from '../../compiled/node_bindings/kimchi_wasm.cjs'; +import type { WasmFpSrs, WasmFqSrs } from '../../compiled/node_bindings/kimchi_napi.wasi.cjs'; export { Field, diff --git a/src/bindings/crypto/bindings/srs.ts b/src/bindings/crypto/bindings/srs.ts deleted file mode 100644 index 25e0e65b63..0000000000 --- a/src/bindings/crypto/bindings/srs.ts +++ /dev/null @@ -1,284 +0,0 @@ -import type { Wasm, RustConversion } from '../bindings.js'; -import { type WasmFpSrs, type WasmFqSrs } from '../../compiled/node_bindings/kimchi_wasm.cjs'; -import { PolyComm } from './kimchi-types.js'; -import { srsCache as cache } from '../cache.js'; -import { - type CacheHeader, - type Cache, - withVersion, - writeCache, - readCache, -} from '../../../lib/proof-system/cache.js'; -import { assert } from '../../../lib/util/errors.js'; -import { MlArray } from '../../../lib/ml/base.js'; -import { OrInfinity, OrInfinityJson } from './curve.js'; - -export { srs }; - -type WasmSrs = WasmFpSrs | WasmFqSrs; - -type SrsStore = Record; - -function empty(): SrsStore { - return {}; -} - -const srsStore = { fp: empty(), fq: empty() }; - -const CacheReadRegister = new Map(); - -const srsVersion = 1; - -function cacheHeaderLagrange(f: 'fp' | 'fq', domainSize: number): CacheHeader { - let id = `lagrange-basis-${f}-${domainSize}`; - return withVersion( - { - kind: 'lagrange-basis', - persistentId: id, - uniqueId: id, - dataType: 'string', - }, - srsVersion - ); -} -function cacheHeaderSrs(f: 'fp' | 'fq', domainSize: number): CacheHeader { - let id = `srs-${f}-${domainSize}`; - return withVersion( - { - kind: 'srs', - persistentId: id, - uniqueId: id, - dataType: 'string', - }, - srsVersion - ); -} - -function srs(wasm: Wasm, conversion: RustConversion<'wasm'>) { - return { - fp: srsPerField('fp', wasm, conversion), - fq: srsPerField('fq', wasm, conversion), - }; -} - -function srsPerField(f: 'fp' | 'fq', wasm: Wasm, conversion: RustConversion<'wasm'>) { - // note: these functions are properly typed, thanks to TS template literal types - let createSrs = (s: number) => wasm[`caml_${f}_srs_create_parallel`](s); - let getSrs = wasm[`caml_${f}_srs_get`]; - let setSrs = wasm[`caml_${f}_srs_set`]; - let isEmptySrs = (srs: WasmSrs) => { - try { - let points = getSrs(srs); - return points == null || points.length <= 1; - } catch { - return true; - } - }; - - let maybeLagrangeCommitment = wasm[`caml_${f}_srs_maybe_lagrange_commitment`]; - let lagrangeCommitment = (srs: WasmFpSrs, domain_size: number, i: number) => - wasm[`caml_${f}_srs_lagrange_commitment`](srs, domain_size, i); - let lagrangeCommitmentsWholeDomainPtr = (srs: WasmSrs, domain_size: number) => - wasm[`caml_${f}_srs_lagrange_commitments_whole_domain_ptr`](srs, domain_size); - let setLagrangeBasis = wasm[`caml_${f}_srs_set_lagrange_basis`]; - let getLagrangeBasis = (srs: WasmSrs, n: number) => - wasm[`caml_${f}_srs_get_lagrange_basis`](srs, n); - let getCommitmentsWholeDomainByPtr = - wasm[`caml_${f}_srs_lagrange_commitments_whole_domain_read_from_ptr`]; - return { - /** - * returns existing stored SRS or falls back to creating a new one - */ - create(size: number): WasmSrs { - let srs = srsStore[f][size] satisfies WasmSrs as WasmSrs | undefined; - - if (srs !== undefined && isEmptySrs(srs)) { - delete srsStore[f][size]; - srs = undefined; - } - - if (srs === undefined) { - if (cache === undefined) { - // if there is no cache, create SRS in memory - srs = createSrs(size); - } else { - let header = cacheHeaderSrs(f, size); - - // try to read SRS from cache / recompute and write if not found - srs = readCache(cache, header, (bytes) => { - // TODO: this takes a bit too long, about 300ms for 2^16 - // `pointsToRust` is the clear bottleneck - let jsonSrs: OrInfinityJson[] = JSON.parse(new TextDecoder().decode(bytes)); - let mlSrs = MlArray.mapTo(jsonSrs, OrInfinity.fromJSON); - let wasmSrs = conversion[f].pointsToRust(mlSrs); - let candidate = setSrs(wasmSrs); - if (isEmptySrs(candidate)) return undefined; - return candidate; - }); - - if (srs === undefined) { - // not in cache - srs = createSrs(size); - - if (cache.canWrite) { - let wasmSrs = getSrs(srs); - let mlSrs = conversion[f].pointsFromRust(wasmSrs); - let jsonSrs = MlArray.mapFrom(mlSrs, OrInfinity.toJSON); - let bytes = new TextEncoder().encode(JSON.stringify(jsonSrs)); - - writeCache(cache, header, bytes); - } - } - } - - srsStore[f][size] = srs; - } - - // TODO should we call freeOnFinalize() and expose a function to clean the SRS cache? - return srsStore[f][size]; - }, - - /** - * returns ith Lagrange basis commitment for a given domain size - */ - lagrangeCommitment(srs: WasmSrs, domainSize: number, i: number): PolyComm { - // happy, fast case: if basis is already stored on the srs, return the ith commitment - let commitment = maybeLagrangeCommitment(srs, domainSize, i); - - if (commitment === undefined) { - if (cache === undefined) { - // if there is no cache, recompute and store basis in memory - commitment = lagrangeCommitment(srs, domainSize, i); - } else { - // try to read lagrange basis from cache / recompute and write if not found - let header = cacheHeaderLagrange(f, domainSize); - let didRead = readCacheLazy( - cache, - header, - conversion, - f, - srs, - domainSize, - setLagrangeBasis - ); - if (didRead !== true) { - // not in cache - if (cache.canWrite) { - try { - let wasmComms = getLagrangeBasis(srs, domainSize); - let mlComms = conversion[f].polyCommsFromRust(wasmComms); - let comms = polyCommsToJSON(mlComms); - let bytes = new TextEncoder().encode(JSON.stringify(comms)); - writeCache(cache, header, bytes); - } catch { - // getLagrangeBasis is unavailable in web workers (WasmVector - // can't cross the SharedArrayBuffer channel). Fall back to - // in-memory computation. - lagrangeCommitment(srs, domainSize, i); - } - } else { - lagrangeCommitment(srs, domainSize, i); - } - } - // here, basis is definitely stored on the srs - let c = maybeLagrangeCommitment(srs, domainSize, i); - assert(c !== undefined, 'commitment exists after setting'); - commitment = c; - } - } - - // edge case for when we have a writeable cache and the basis was already stored on the srs - // but we didn't store it in the cache separately yet - if (commitment && cache && cache.canWrite) { - let header = cacheHeaderLagrange(f, domainSize); - let didRead = readCacheLazy( - cache, - header, - conversion, - f, - srs, - domainSize, - setLagrangeBasis - ); - // only proceed for entries we haven't written to the cache yet - if (didRead !== true) { - try { - let wasmComms = getLagrangeBasis(srs, domainSize); - let mlComms = conversion[f].polyCommsFromRust(wasmComms); - let comms = polyCommsToJSON(mlComms); - let bytes = new TextEncoder().encode(JSON.stringify(comms)); - writeCache(cache, header, bytes); - } catch { - // getLagrangeBasis unavailable in web workers — skip cache write. - } - } - } - return conversion[f].polyCommFromRust(commitment); - }, - - /** - * Returns the Lagrange basis commitments for the whole domain - */ - lagrangeCommitmentsWholeDomain(srs: WasmSrs, domainSize: number) { - // instead of getting the entire commitment directly (which works for nodejs/servers), we get a pointer to the commitment - // and then read the commitment from the pointer - // this is because the web worker implementation currently does not support returning UintXArray's directly - // hence we return a pointer from wasm, funnel it through the web worker - // and then read the commitment from the pointer in the main thread (where UintXArray's are supported) - // see https://github.com/o1-labs/o1js-bindings/blob/09e17b45e0c2ca2b51cd9ed756106e17ca1cf36d/js/web/worker-spec.js#L110-L115 - let ptr = lagrangeCommitmentsWholeDomainPtr(srs, domainSize); - let wasmComms = getCommitmentsWholeDomainByPtr(ptr); - let mlComms = conversion[f].polyCommsFromRust(wasmComms); - return mlComms; - }, - - /** - * adds Lagrange basis for a given domain size - */ - addLagrangeBasis(srs: WasmSrs, logSize: number) { - // this ensures that basis is stored on the srs, no need to duplicate caching logic - this.lagrangeCommitment(srs, 1 << logSize, 0); - }, - }; -} - -type PolyCommJson = { - shifted: OrInfinityJson[]; - unshifted: OrInfinityJson | undefined; -}; - -function polyCommsToJSON(comms: MlArray): PolyCommJson[] { - return MlArray.mapFrom(comms, ([, elems]) => { - return { - shifted: MlArray.mapFrom(elems, OrInfinity.toJSON), - unshifted: undefined, - }; - }); -} - -function polyCommsFromJSON(json: PolyCommJson[]): MlArray { - return MlArray.mapTo(json, ({ shifted, unshifted }) => { - return [0, MlArray.mapTo(shifted, OrInfinity.fromJSON)]; - }); -} - -function readCacheLazy( - cache: Cache, - header: CacheHeader, - conversion: RustConversion<'wasm'>, - f: 'fp' | 'fq', - srs: WasmSrs, - domainSize: number, - setLagrangeBasis: (srs: WasmSrs, domainSize: number, comms: Uint32Array) => void -) { - if (CacheReadRegister.get(header.uniqueId) === true) return true; - return readCache(cache, header, (bytes) => { - let comms: PolyCommJson[] = JSON.parse(new TextDecoder().decode(bytes)); - let mlComms = polyCommsFromJSON(comms); - let wasmComms = conversion[f].polyCommsToRust(mlComms); - - setLagrangeBasis(srs, domainSize, wasmComms); - CacheReadRegister.set(header.uniqueId, true); - return true; - }); -} diff --git a/src/bindings/crypto/native/napi-conversion-verifier-index.ts b/src/bindings/crypto/native/napi-conversion-verifier-index.ts index 571c021c40..d3b415855a 100644 --- a/src/bindings/crypto/native/napi-conversion-verifier-index.ts +++ b/src/bindings/crypto/native/napi-conversion-verifier-index.ts @@ -18,44 +18,21 @@ import type { NapiShiftsShape, NapiVerificationEvalsShape, NapiVerifierIndex, - NapiVerifierIndexClasses, NapiVerifierIndexShape, } from './napi-wrappers.js'; export { napiVerifierIndexConversion }; +// all verifier-index types are `#[napi(object)]` on the Rust side, i.e. plain +// JS objects — no constructors are needed from the FFI module function napiVerifierIndexConversion(napi: Napi, core: ConversionCores) { return { - fp: verifierIndexConversionPerField(core.fp, { - Domain: napi.WasmFpDomain, - VerificationEvals: napi.WasmFpPlonkVerificationEvals, - Shifts: napi.WasmFpShifts, - VerifierIndex: napi.WasmFpPlonkVerifierIndex, - LookupVerifierIndex: napi.WasmFpLookupVerifierIndex, - LookupSelector: napi.WasmFpLookupSelectors, - }), - fq: verifierIndexConversionPerField(core.fq, { - Domain: napi.WasmFqDomain, - VerificationEvals: napi.WasmFqPlonkVerificationEvals, - Shifts: napi.WasmFqShifts, - VerifierIndex: napi.WasmFqPlonkVerifierIndex, - LookupVerifierIndex: napi.WasmFqLookupVerifierIndex, - LookupSelector: napi.WasmFqLookupSelectors, - }), + fp: verifierIndexConversionPerField(core.fp), + fq: verifierIndexConversionPerField(core.fq), }; } -function verifierIndexConversionPerField( - core: ConversionCore, - { - Domain, - VerificationEvals, - Shifts, - VerifierIndex, - LookupVerifierIndex, - LookupSelector, - }: NapiVerifierIndexClasses -) { +function verifierIndexConversionPerField(core: ConversionCore) { function domainToRust([, logSizeOfGroup, groupGen]: Domain): NapiDomain { // In the NAPI backend these types are `#[napi(object)]`, i.e. plain JS objects // (not constructable classes). @@ -178,9 +155,12 @@ function verifierIndexConversionPerField( return [0, lookup, xor, range_check, ffmul]; } - function lookupInfoToRust( - [, maxPerRow, maxJointSize, features]: LookupInfo - ): NapiLookupInfoObject { + function lookupInfoToRust([ + , + maxPerRow, + maxJointSize, + features, + ]: LookupInfo): NapiLookupInfoObject { let [, patterns, joint_lookup_used, uses_runtime_tables] = features; let [, xor, lookup, range_check, foreign_field_mul] = patterns; return { diff --git a/src/bindings/crypto/native/napi-srs.ts b/src/bindings/crypto/native/napi-srs.ts index e4d4cc770c..e836865594 100644 --- a/src/bindings/crypto/native/napi-srs.ts +++ b/src/bindings/crypto/native/napi-srs.ts @@ -52,14 +52,14 @@ function cacheHeaderSrs(f: 'fp' | 'fq', domainSize: number): CacheHeader { ); } -function srs(napi: Napi, conversion: RustConversion<'native'>) { +function srs(napi: Napi, conversion: RustConversion) { return { fp: srsPerField('fp', napi, conversion), fq: srsPerField('fq', napi, conversion), }; } -function srsPerField(f: 'fp' | 'fq', napi: Napi, conversion: RustConversion<'native'>) { +function srsPerField(f: 'fp' | 'fq', napi: Napi, conversion: RustConversion) { // note: these functions are properly typed, thanks to TS template literal types let createSrs = (size: number) => { try { @@ -329,7 +329,7 @@ function polyCommsFromJSON(json: PolyCommJson[]): MlArray { function readCacheLazy( cache: Cache, header: CacheHeader, - conversion: RustConversion<'native'>, + conversion: RustConversion, f: 'fp' | 'fq', srs: NapiSrs, domainSize: number, diff --git a/src/bindings/crypto/native/napi-wrappers.ts b/src/bindings/crypto/native/napi-wrappers.ts index 38920317ea..a3a76f0757 100644 --- a/src/bindings/crypto/native/napi-wrappers.ts +++ b/src/bindings/crypto/native/napi-wrappers.ts @@ -1,4 +1,4 @@ -import type * as napiNamespace from '../../compiled/node_bindings/kimchi_wasm.cjs'; +import type * as napiNamespace from '../../compiled/node_bindings/kimchi_napi.wasi.cjs'; import type { WasmFpDomain as NapiFpDomain, WasmFpLookupCommitments as NapiFpLookupCommitments, @@ -32,7 +32,7 @@ import type { WasmPastaFqLookupTable as NapiPastaFqLookupTable, WasmVecVecFp as NapiVecVecFp, WasmVecVecFq as NapiVecVecFq, -} from '../../compiled/node_bindings/kimchi_wasm.cjs'; +} from '../../compiled/node_bindings/kimchi_napi.wasi.cjs'; export type Napi = typeof napiNamespace; @@ -236,15 +236,6 @@ export type NapiProofClasses = { LookupTable: typeof NapiPastaFpLookupTable | typeof NapiPastaFqLookupTable; }; -export type NapiVerifierIndexClasses = { - Domain: typeof NapiFpDomain | typeof NapiFqDomain; - VerificationEvals: typeof NapiFpPlonkVerificationEvals | typeof NapiFqPlonkVerificationEvals; - Shifts: typeof NapiFpShifts | typeof NapiFqShifts; - VerifierIndex: typeof NapiFpPlonkVerifierIndex | typeof NapiFqPlonkVerifierIndex; - LookupVerifierIndex: typeof NapiFpLookupVerifierIndex | typeof NapiFqLookupVerifierIndex; - LookupSelector: typeof NapiFpLookupSelectors | typeof NapiFqLookupSelectors; -}; - export type NapiOraclesClasses = { RandomOracles: typeof NapiFpRandomOracles | typeof NapiFqRandomOracles; Oracles: typeof NapiFpOracles | typeof NapiFqOracles; diff --git a/src/bindings/js/node/native-backend.js b/src/bindings/js/node/native-backend.js index 7586a467da..b20906f1bb 100644 --- a/src/bindings/js/node/native-backend.js +++ b/src/bindings/js/node/native-backend.js @@ -12,9 +12,12 @@ try { import.meta.url !== undefined ? import.meta.url : pathToFileURL(__filename).href; const require_ = createRequire(moduleUrl); wasm = require_(slug); + wasm.__kimchi_backend = 'native'; wasm.__o1js_backend_preference = 'native'; if (typeof globalThis !== 'undefined') { globalThis.__o1js_backend_preference = 'native'; + // the compiled OCaml artifact (o1js_node.bc.cjs) picks up the FFI module here + globalThis.__o1js_kimchi_ffi = wasm; } } catch (e) { throw new Error( diff --git a/src/bindings/js/node/node-backend.js b/src/bindings/js/node/node-backend.js index 1a781eaaab..9748a86f14 100644 --- a/src/bindings/js/node/node-backend.js +++ b/src/bindings/js/node/node-backend.js @@ -1,199 +1,52 @@ -import { createRequire } from 'module'; -import os from 'os'; -import { dirname, join } from 'path'; -import { fileURLToPath } from 'url'; -import { Worker, isMainThread, parentPort, workerData } from 'worker_threads'; +import { createRequire } from 'node:module'; +import os from 'node:os'; +import { fileURLToPath } from 'node:url'; import { WithThreadPool, workers } from '../../../lib/proof-system/workers.js'; + +export { wasm, withThreadPool }; + let url = import.meta.url; let filename = url !== undefined ? fileURLToPath(url) : __filename; const require = createRequire(filename); -const wasm_ = requireKimchiWasm(!isMainThread ? workerData?.memory : undefined); -/** - * @type {import("../../compiled/node_bindings/kimchi_wasm.cjs")} - */ -const wasm = wasm_; +// The 'wasm' backend is the wasm32-wasip1-threads build of the same kimchi-napi +// crate that powers the native backend. The generated .wasi.cjs loader +// (via @napi-rs/wasm-runtime) instantiates the module synchronously and spawns +// worker_threads on demand for Rust std::thread/rayon — no manual memory +// sharing or worker bootstrapping is needed here. +// +// Rayon reads its thread-pool size from the environment at first use, so it +// must be configured before any parallel binding call. +setRayonThreadCount(); + +const wasm = requireKimchiNapiWasm(); + +// Both backends expose the napi object model, so they share the TS conversion +// layer (src/bindings/crypto/native/). +wasm.__kimchi_backend = 'native'; wasm.__o1js_backend_preference = 'wasm'; if (typeof globalThis !== 'undefined') { globalThis.__o1js_backend_preference = 'wasm'; + // the compiled OCaml artifact (o1js_node.bc.cjs) picks up the FFI module here + globalThis.__o1js_kimchi_ffi = wasm; } -export { wasm, withThreadPool }; +// The wasm runtime manages its own threads; nothing to set up or tear down. +const withThreadPool = WithThreadPool({ + initThreadPool: async () => {}, + exitThreadPool: async () => {}, +}); -function requireKimchiWasm(memoryOverride) { +function requireKimchiNapiWasm() { let modulePath = filename.endsWith('index.cjs') - ? './bindings/compiled/node_bindings/kimchi_wasm.cjs' - : '../../compiled/node_bindings/kimchi_wasm.cjs'; - if (memoryOverride === undefined) return require(modulePath); - - let OriginalMemory = WebAssembly.Memory; - WebAssembly.Memory = new Proxy(OriginalMemory, { - construct(_target, _args, _newTarget) { - return memoryOverride; - }, - }); - try { - return require(modulePath); - } finally { - WebAssembly.Memory = OriginalMemory; - } -} - -function getWorkerSource() { - return filename.endsWith('index.cjs') - ? join(dirname(filename), 'bindings/js/node/node-backend.js') - : filename; -} - -let workersReadyResolve; -let workersReady; -let wasmThreadPoolRunning = false; - -// expose this globally so that it can be referenced from wasm -globalThis.startWorkers = startWorkers; -globalThis.terminateWorkers = terminateWorkers; - -if (!isMainThread) { - parentPort.postMessage({ type: 'wasm_bindgen_worker_ready' }); - wasm.wbg_rayon_start_worker(workerData.receiver); -} - -// state machine to enable calling multiple functions that need a thread pool at once -const withThreadPool = WithThreadPool({ initThreadPool, exitThreadPool }); - -async function initThreadPool() { - if (!isMainThread) return; - if (wasmThreadPoolRunning) return; - const numThreads = Math.max(1, workers.numWorkers ?? (os.availableParallelism() ?? 1) - 1); - workersReady = new Promise((resolve) => (workersReadyResolve = resolve)); - try { - await wasm.initThreadPool(numThreads, getWorkerSource()); - await workersReady; - wasmThreadPoolRunning = true; - } catch (error) { - wasmThreadPoolRunning = false; - throw error; - } finally { - workersReady = undefined; - workersReadyResolve = undefined; - } -} - -async function exitThreadPool() { - if (!isMainThread) return; - if (!wasmThreadPoolRunning) return; - // Keep the pool alive across compile/prove calls. - // Explicit teardown can deadlock or trigger finalizer crashes depending on - // toolchain/runtime combinations. -} - -/** - * @type {Worker[]} - */ -let wasmWorkers = []; - -function getWorkerMemory() { - // Use the canonical memory object from the loaded JS module instead of the - // callback argument coming through wasm-bindgen externref glue. - return typeof wasm.get_memory === 'function' ? wasm.get_memory() : wasm.__wasm.memory; -} - -function isCloneError(error) { - return ( - error?.name === 'DataCloneError' || - String(error?.message ?? error).includes('could not be cloned') - ); -} - -function describeMemory(value) { - return { - type: value?.constructor?.name, - hasBuffer: !!value?.buffer, - sharedBuffer: value?.buffer instanceof SharedArrayBuffer, - byteLength: value?.buffer?.byteLength, - }; -} - -function cloneCheck(value) { - try { - structuredClone(value); - return 'ok'; - } catch (error) { - return `${error?.name ?? 'Error'}: ${error?.message ?? String(error)}`; - } -} - -async function startWorkers(src, memory, builder) { - wasmWorkers = []; - const startupTimeoutMs = 30_000; - let workerMemory = getWorkerMemory(); - await Promise.all( - Array.from({ length: builder.numThreads() }, () => { - let worker = new Worker(src, { - workerData: { memory: workerMemory, receiver: builder.receiver() }, - }); - wasmWorkers.push(worker); - return new Promise((resolve, reject) => { - let timer = setTimeout(() => { - cleanup(); - reject(new Error('Timed out waiting for wasm worker startup')); - }, startupTimeoutMs); - let ready = false; - - function cleanup() { - clearTimeout(timer); - worker.off('message', onReady); - worker.off('error', onError); - worker.off('exit', onExit); - } - - function onReady(data) { - if (data == null || data.type !== 'wasm_bindgen_worker_ready') return; - ready = true; - cleanup(); - // Do not keep the process alive solely because pool workers exist. - worker.unref(); - resolve(worker); - } - - function onError(error) { - cleanup(); - reject(error); - } - - function onExit(code) { - cleanup(); - if (ready) { - // Some wasm-bindgen/node combinations exit worker threads as soon as - // startup work is done. Treat a clean exit as successful startup. - resolve(worker); - return; - } - reject(new Error(`WASM worker exited before ready (code ${code})`)); - } - - worker.on('message', onReady); - worker.once('error', onError); - worker.once('exit', onExit); - }); - }) - ); - builder.build(); - workersReadyResolve(); + ? './bindings/compiled/node_bindings/kimchi_napi.wasi.cjs' + : '../../compiled/node_bindings/kimchi_napi.wasi.cjs'; + return require(modulePath); } -function terminateWorkers() { - let workersToTerminate = wasmWorkers ?? []; - wasmWorkers = []; - wasmThreadPoolRunning = false; - for (let worker of workersToTerminate) { - try { - let terminated = worker.terminate(); - if (terminated && typeof terminated.catch === 'function') { - terminated.catch(() => {}); - } - } catch { - // Ignore shutdown races. - } - } +function setRayonThreadCount() { + if (typeof process === 'undefined') return; + if (process.env.RAYON_NUM_THREADS !== undefined) return; + let numThreads = Math.max(1, workers.numWorkers ?? (os.availableParallelism?.() ?? 1) - 1); + process.env.RAYON_NUM_THREADS = String(numThreads); } diff --git a/src/bindings/js/web/web-backend.js b/src/bindings/js/web/web-backend.js index 7754edc031..d41e536485 100644 --- a/src/bindings/js/web/web-backend.js +++ b/src/bindings/js/web/web-backend.js @@ -1,222 +1,42 @@ import o1jsWebSrc from 'string:../../../web_bindings/o1js_web.bc.js'; -import { WithThreadPool, workers } from '../../../lib/proof-system/workers.js'; -import kimchiWasm from '../../../web_bindings/kimchi_wasm.js'; -import { inlineWorker, srcFromFunctionModule, waitForMessage } from './worker-helpers.js'; -import { workerSpec } from './worker-spec.js'; +import { WithThreadPool } from '../../../lib/proof-system/workers.js'; +import * as kimchiNapi from '../../../web_bindings/kimchi_napi.wasi-browser.js'; export { initializeBindings, wasm, withThreadPool }; let wasm; -/** - * @type {Promise} - */ -let workerPromise; -/** - * @type {number | undefined} - */ -let numWorkers = undefined; -let wasmThreadPoolRunning = false; - async function initializeBindings() { - wasm = kimchiWasm(); - globalThis.kimchi_wasm = wasm; - let init = wasm.default; - - const memory = allocateWasmMemoryForUserAgent(navigator.userAgent); - await init(undefined, memory); - - let module = init.__wbindgen_wasm_module; - - // we have two approaches to run the .bc.js code after its dependencies are ready, without fetching an additional script: - - // 1. wrap it inside a function and just include that function in the bundle - // this could be nice and simple, but breaks because the .bc.js code uses `(function(){return this}())` to access `window` - // (probably as a cross-platform way to get the global object before globalThis existed) - // that obsolete hack doesn't work here because inside an ES module, this === undefined instead of this === window - // it seems to work when we patch the source code (replace IIFEs with `window`) - - // 2. include the code as string and eval it: - // (this works because it breaks out of strict mode) - new Function(o1jsWebSrc)(); - - workerPromise = new Promise((resolve, reject) => { - setTimeout(async () => { - let worker = inlineWorker(srcFromFunctionModule(mainWorker)); - let onError = (error) => { - reject(new Error(`Failed to start o1js web worker: ${error.message}`)); - }; - worker.addEventListener('error', onError, { once: true }); - try { - await workerCall(worker, 'start', { memory, module }); - worker.removeEventListener('error', onError); - if (worker._o1jsBlobUrl !== undefined) { - URL.revokeObjectURL(worker._o1jsBlobUrl); - delete worker._o1jsBlobUrl; - } - overrideBindings(globalThis.kimchi_wasm, worker); - resolve(worker); - } catch (error) { - worker.removeEventListener('error', onError); - reject(error); - } - }, 0); - }); -} - -async function initThreadPool() { - if (workerPromise === undefined) throw Error('need to initialize worker first'); - if (wasmThreadPoolRunning) return; - let worker = await workerPromise; - numWorkers ??= Math.max(1, workers.numWorkers ?? (navigator.hardwareConcurrency ?? 1) - 1); - await workerCall(worker, 'initThreadPool', numWorkers); - wasmThreadPoolRunning = true; -} - -async function exitThreadPool() { - if (workerPromise === undefined) throw Error('need to initialize worker first'); - if (!wasmThreadPoolRunning) return; - // Keep the pool alive across compile/prove calls. - // Explicit teardown can deadlock on some runtime/toolchain combinations. -} - -const withThreadPool = WithThreadPool({ initThreadPool, exitThreadPool }); - -async function mainWorker() { - const wasm = kimchiWasm(); - let init = wasm.default; - - let spec = workerSpec(wasm); - - let isInitialized = false; - let data = await waitForMessage(self, 'start'); - let { module, memory } = data.message; - - onMessage(self, 'run', ({ name, args, u32_ptr }) => { - let functionSpec = spec[name]; - let specArgs = functionSpec.args; - let resArgs = args; - for (let i = 0, l = specArgs.length; i < l; i++) { - let specArg = specArgs[i]; - if (specArg && specArg.__wrap) { - // Reconstruct the class wrapper from the raw pointer. - // IMPORTANT: Do NOT use specArg.__wrap() here — in wasm-bindgen - // >= 0.2.100, __wrap() registers the object with a FinalizationRegistry. - // When the worker GC collects these temporary wrappers, the finalizer - // frees memory that the main thread still owns, causing use-after-free. - // Instead, create a bare prototype wrapper that borrows the pointer. - let obj = Object.create(specArg.prototype); - obj.__wbg_ptr = args[i].__wbg_ptr; - resArgs[i] = obj; - } else { - resArgs[i] = args[i]; - } - } - let res = wasm[name].apply(wasm, resArgs); - if (functionSpec.res && functionSpec.res.__wrap) { - // Transfer ownership of the result from the worker's wasm instance. - // __destroy_into_raw() unregisters from the worker's FinalizationRegistry - // and returns the raw pointer. Without this, the worker's GC would - // eventually free the result while the main thread still holds it. - res = typeof res.__destroy_into_raw === 'function' ? res.__destroy_into_raw() : res.__wbg_ptr; - } else if (functionSpec.res && functionSpec.res.there) { - res = functionSpec.res.there(res); - } - /* Here be undefined behavior dragons. */ - wasm.set_u32_ptr(u32_ptr, res); - /*postMessage(res);*/ - }); - - workerExport(self, { - async initThreadPool(numWorkers) { - if (!isInitialized) { - isInitialized = true; - await wasm.initThreadPool(numWorkers); - } - }, - async exitThreadPool() { - if (isInitialized) { - isInitialized = false; - await wasm.exitThreadPool(); - } - }, - }); - - await init(module, memory); - postMessage({ type: data.id }); -} -mainWorker.deps = [kimchiWasm, workerSpec, workerExport, onMessage, waitForMessage]; - -function overrideBindings(kimchi_wasm, worker) { - let spec = workerSpec(kimchi_wasm); - for (let key in spec) { - kimchi_wasm[key] = (...args) => { - if (spec[key].disabled) throw Error(`Wasm method '${key}' is disabled on the web.`); - let u32_ptr = wasm.create_zero_u32_ptr(); - worker.postMessage({ - type: 'run', - message: { name: key, args, u32_ptr }, - }); - /* Here be undefined behavior dragons. */ - let res = wasm.wait_until_non_zero(u32_ptr); - wasm.free_u32_ptr(u32_ptr); - let res_spec = spec[key].res; - if (res_spec && res_spec.__wrap) { - return spec[key].res.__wrap(res); - } else if (res_spec && res_spec.back) { - return res_spec.back(res); - } else { - return res; - } - }; - } -} - -// helpers for main thread <-> worker communication - -function onMessage(worker, type, onMsg) { - worker.addEventListener('message', function ({ data }) { - if (data?.type !== type) return; - onMsg(data.message); - }); -} - -function workerExport(worker, exportObject) { - for (let key in exportObject) { - worker.addEventListener('message', async function ({ data }) { - if (data?.type !== key) return; - try { - let result = await exportObject[key](data.message); - postMessage({ type: data.id, result }); - } catch (error) { - postMessage({ type: data.id, error: String(error?.stack ?? error) }); - } - }); + if (wasm !== undefined) return; + + // The wasm backend is the wasm32-wasip1-threads build of the kimchi-napi + // crate — the same crate that powers the native backend on Node. The + // generated .wasi-browser.js loader (via @napi-rs/wasm-runtime) instantiates + // the module on this thread and spawns Web Workers on demand for Rust + // std::thread. SharedArrayBuffer (COOP/COEP headers) is required, same as + // with the previous wasm-bindgen backend. + // + // NOTE on threading: browser main threads cannot block, so rayon-parallel + // sections that make the calling thread wait must not run on the main + // thread with a multi-threaded pool. Until worker-hosted execution lands + // (see PLAN.md, web Option B), the pool is limited to inline execution. + wasm = kimchiNapi.default ?? kimchiNapi; + wasm.__kimchi_backend = 'native'; + + if (typeof globalThis !== 'undefined') { + globalThis.__o1js_backend_preference = 'wasm'; + // the compiled OCaml artifact (o1js_web.bc.js) picks up the FFI module here + globalThis.__o1js_kimchi_ffi = wasm; } -} -async function workerCall(worker, type, message) { - let id = Math.random(); - let promise = waitForMessage(worker, id); - worker.postMessage({ type, id, message }); - let response = await promise; - if (response.error) throw new Error(response.error); - return response.result; + // Evaluate the compiled OCaml artifact. It is included as a string and + // eval'd because the js_of_ocaml output uses `(function(){return this}())` + // to reach the global object, which breaks inside ES modules (strict mode). + new Function(o1jsWebSrc)(); } -function allocateWasmMemoryForUserAgent(userAgent) { - const isIOSDevice = /iPad|iPhone|iPod/.test(userAgent); - if (isIOSDevice) { - return new WebAssembly.Memory({ - initial: 20, - maximum: 16384, // 1 GiB - shared: true, - }); - } else { - return new WebAssembly.Memory({ - initial: 20, - maximum: 65536, // 4 GiB - shared: true, - }); - } -} +// The wasm runtime manages its own threads; nothing to set up or tear down. +const withThreadPool = WithThreadPool({ + initThreadPool: async () => {}, + exitThreadPool: async () => {}, +}); diff --git a/src/bindings/js/web/worker-helpers.js b/src/bindings/js/web/worker-helpers.js deleted file mode 100644 index 8f223e0709..0000000000 --- a/src/bindings/js/web/worker-helpers.js +++ /dev/null @@ -1,136 +0,0 @@ -import wasm from '../../../web_bindings/kimchi_wasm.js'; - -export { - inlineWorker, - srcFromFunctionModule, - startWorkers, - terminateWorkers, - waitForMessage, - workerHelperMain, -}; - -function srcFromFunctionModule(fun) { - let deps = collectDependencies(fun, []); - if (!deps.includes(fun)) deps.push(fun); - let sources = deps.map((dep) => { - let src = dep.toString(); - if (dep.deps) { - let depsList = dep.deps.map((d) => d.name).join(','); - src += `\n${dep.name}.deps = [${depsList}]`; - } - return src; - }); - return sources.join('\n') + `\n${fun.name}();`; -} -srcFromFunctionModule.deps = [collectDependencies]; - -function collectDependencies(fun, deps) { - for (let dep of fun.deps ?? []) { - if (deps.includes(dep)) continue; - deps.push(dep); - collectDependencies(dep, deps); - } - return deps; -} - -function inlineWorker(src) { - let blob = new Blob([src], { type: 'application/javascript' }); - let url = URL.createObjectURL(blob); - let worker = new Worker(url); - // Keep the blob URL alive for the lifetime of the worker. Revoking it - // immediately can race worker startup in headless browsers. - worker._o1jsBlobUrl = url; - return worker; -} - -function waitForMessage(target, type) { - return new Promise((resolve) => { - target.addEventListener('message', function onMsg({ data }) { - if (data?.type !== type) return; - target.removeEventListener('message', onMsg); - resolve(data); - }); - }); -} - -function workerHelperMain() { - let { default: init, wbg_rayon_start_worker } = wasm(); - - waitForMessage(self, 'wasm_bindgen_worker_init') - .then(async (data) => { - await init(data.module, data.memory); - postMessage({ type: 'wasm_bindgen_worker_ready' }); - wbg_rayon_start_worker(data.receiver); - }) - .catch((err) => { - postMessage({ - type: 'wasm_bindgen_worker_error', - error: String(err?.stack ?? err), - }); - }); -} -workerHelperMain.deps = [wasm, waitForMessage]; - -async function startWorkers(module, memory, builder) { - const startupTimeoutMs = 30_000; - const workerInit = { - type: 'wasm_bindgen_worker_init', - module, - memory, - receiver: builder.receiver(), - }; - let workerSrc = srcFromFunctionModule(workerHelperMain); - - self._workers = []; // not used, prevents Firefox bug - - let blob = new Blob([workerSrc], { type: 'application/javascript' }); - let url = URL.createObjectURL(blob); - self._workerBlobUrl = url; - for (let i = 0; i < builder.numThreads(); i++) { - let worker = new Worker(url); - worker.postMessage(workerInit); - self._workers.push(worker); - } - URL.revokeObjectURL(url); - - await Promise.all( - self._workers.map( - (w) => - new Promise((resolve, reject) => { - let timer = setTimeout(() => { - cleanup(); - reject(new Error('Timed out waiting for wasm worker startup')); - }, startupTimeoutMs); - - function cleanup() { - clearTimeout(timer); - w.removeEventListener('message', onMessage); - } - - function onMessage({ data }) { - if (data?.type === 'wasm_bindgen_worker_ready') { - cleanup(); - resolve(w); - } else if (data?.type === 'wasm_bindgen_worker_error') { - cleanup(); - reject(new Error('WASM worker init failed: ' + data.error)); - } - } - - w.addEventListener('message', onMessage); - }) - ) - ); - builder.build(); -} -startWorkers.deps = [srcFromFunctionModule, waitForMessage, workerHelperMain]; - -async function terminateWorkers() { - self._workers.forEach((worker) => { - worker.terminate(); - }); - if (self._workerBlobUrl !== undefined) { - URL.revokeObjectURL(self._workerBlobUrl); - delete self._workerBlobUrl; - } -} diff --git a/src/bindings/js/web/worker-spec.js b/src/bindings/js/web/worker-spec.js deleted file mode 100644 index 8eb069fd01..0000000000 --- a/src/bindings/js/web/worker-spec.js +++ /dev/null @@ -1,169 +0,0 @@ -export { workerSpec }; - -function workerSpec(wasm) { - let bool = { - // We avoid returning zero for false to ensure that the - // wait_until_non_zero call below terminates. - there: (bool) => (bool ? 2 : 1), - back: (u32) => u32 !== 1, - }; - return { - caml_pasta_fp_plonk_index_create: { - args: [ - // gates - wasm.WasmFpGateVector, - // public_ - undefined /* number */, - // lookup_tables - undefined /*Uint32Array*/, - // runtime_table_cfgs - undefined /*Uint32Array*/, - // prev_challenges - undefined /* number */, - // srs - wasm.WasmFpSrs, - // lazy_mode - undefined /* boolean */, - ], - res: wasm.WasmPastaFpPlonkIndex, - }, - caml_pasta_fq_plonk_index_create: { - args: [ - // gates - wasm.WasmFqGateVector, - // public_ - undefined /* number */, - // lookup_tables - undefined /*Uint32Array*/, - // runtime_table_cfgs - undefined /*Uint32Array*/, - // prev_challenges - undefined /* number */, - // srs - wasm.WasmFqSrs, - // lazy_mode - undefined /* boolean */, - ], - res: wasm.WasmPastaFqPlonkIndex, - }, - caml_pasta_fp_plonk_verifier_index_create: { - args: [wasm.WasmPastaFpPlonkIndex], - res: wasm.WasmFpPlonkVerifierIndex, - }, - caml_pasta_fq_plonk_verifier_index_create: { - args: [wasm.WasmPastaFqPlonkIndex], - res: wasm.WasmFqPlonkVerifierIndex, - }, - caml_pasta_fp_plonk_proof_create: { - args: [ - // index - wasm.WasmPastaFpPlonkIndex, - // witness - wasm.WasmVecVecFp, - // runtime tables - undefined /*Uint32Array*/, - // prev_challenges - undefined /*Uint8Array*/, - // prev_svgs - undefined /*Uint32Array*/, - ], - res: wasm.WasmFpProverProof, - }, - caml_pasta_fq_plonk_proof_create: { - args: [ - // index - wasm.WasmPastaFqPlonkIndex, - // witness - wasm.WasmVecVecFq, - // runtime tables - undefined /*Uint32Array*/, - // prev_challenges - undefined /*Uint8Array*/, - // prev_svgs - undefined /*Uint32Array*/, - ], - res: wasm.WasmFqProverProof, - }, - caml_pasta_fp_plonk_proof_verify: { - args: [wasm.WasmFpPlonkVerifierIndex, wasm.WasmFpProverProof], - res: bool, - }, - caml_pasta_fq_plonk_proof_verify: { - args: [wasm.WasmFqPlonkVerifierIndex, wasm.WasmFqProverProof], - res: bool, - }, - caml_pasta_fp_plonk_proof_batch_verify: { - args: [undefined /* UintXArray */, undefined /* UintXArray */], - res: bool, - }, - caml_pasta_fq_plonk_proof_batch_verify: { - args: [undefined /* UintXArray */, undefined /* UintXArray */], - res: bool, - }, - caml_fp_srs_create_parallel: { - args: [undefined /*number*/], - res: wasm.WasmFpSrs, - }, - caml_fq_srs_create_parallel: { - args: [undefined /*number*/], - res: wasm.WasmFqSrs, - }, - caml_fp_srs_get_lagrange_basis: { - disabled: true, - args: [wasm.WasmFpSrs, undefined /* number */], - // TODO: returning a UintXArray does not work: - // the worker wrapper excepts the return value to be a number - // that can be stored in a single u32. - // A UintXArray is coerced into a 0 pointer, which doesn't trigger `wait_until_non_zero()`, - // which means the main worker just keeps spinning waiting for a response. - // A proper solution would be to wrap the return value in a pointer! - res: undefined /* UintXArray */, - }, - caml_fq_srs_get_lagrange_basis: { - disabled: true, - args: [wasm.WasmFqSrs, undefined /* number */], - // TODO: returning a UintXArray does not work, see above - res: undefined /* UintXArray */, - }, - caml_fp_srs_b_poly_commitment: { - args: [wasm.WasmFpSrs, undefined /*Uint8Array*/], - res: wasm.WasmFpPolyComm, - }, - caml_fq_srs_b_poly_commitment: { - args: [wasm.WasmFqSrs, undefined /*Uint8Array*/], - res: wasm.WasmFqPolyComm, - }, - fp_oracles_create: { - args: [undefined /* Uint32Array */, wasm.WasmFpPlonkVerifierIndex, wasm.WasmFpProverProof], - res: wasm.WasmFpOracles, - }, - fq_oracles_create: { - args: [undefined /* Uint32Array */, wasm.WasmFqPlonkVerifierIndex, wasm.WasmFqProverProof], - res: wasm.WasmFqOracles, - }, - caml_fp_srs_batch_accumulator_check: { - args: [wasm.WasmFpSrs, undefined /* UintXArray */, undefined /* UintXArray */], - res: bool, - }, - caml_fq_srs_batch_accumulator_check: { - args: [wasm.WasmFqSrs, undefined /* UintXArray */, undefined /* UintXArray */], - res: bool, - }, - caml_fp_srs_lagrange_commitment: { - args: [wasm.WasmFpSrs, undefined /* number */, undefined /* number */], - res: wasm.WasmFpPolyComm, - }, - caml_fq_srs_lagrange_commitment: { - args: [wasm.WasmFqSrs, undefined /* number */, undefined /* number */], - res: wasm.WasmFqPolyComm, - }, - caml_fp_srs_lagrange_commitments_whole_domain: { - args: [wasm.WasmFpSrs, undefined /* number */], - res: undefined /* number, ptr */, - }, - caml_fq_srs_lagrange_commitments_whole_domain: { - args: [wasm.WasmFqSrs, undefined /* number */], - res: undefined /* number, ptr */, - }, - }; -} diff --git a/src/build/build-example.js b/src/build/build-example.js index 91e034631e..17ae91309b 100644 --- a/src/build/build-example.js +++ b/src/build/build-example.js @@ -129,7 +129,7 @@ function makeO1jsExternal() { } function makeJsooExternal() { - let isJsoo = /(bc.cjs|kimchi_wasm.cjs)$/; + let isJsoo = /(bc.cjs|kimchi_napi.wasi.cjs)$/; return { name: 'plugin-external', setup(build) { diff --git a/src/build/build-node.js b/src/build/build-node.js index c4ac171ad8..1e7a2088c7 100644 --- a/src/build/build-node.js +++ b/src/build/build-node.js @@ -56,7 +56,7 @@ function makeNodeModulesExternal() { } function makeJsooExternal() { - let isJsoo = /(bc.cjs|kimchi_wasm.cjs)$/; + let isJsoo = /(bc.cjs|kimchi_napi.wasi.cjs)$/; return { name: 'plugin-external', setup(build) { diff --git a/src/build/build-web.js b/src/build/build-web.js index 0d7a93a5e3..b39a36dfed 100644 --- a/src/build/build-web.js +++ b/src/build/build-web.js @@ -2,7 +2,7 @@ import esbuild from 'esbuild'; import fse, { move } from 'fs-extra'; import glob from 'glob'; import { exec } from 'node:child_process'; -import { readFile, unlink, writeFile } from 'node:fs/promises'; +import { readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -24,36 +24,47 @@ if (isMain) { async function buildWeb({ production }) { let minify = !!production; - // prepare kimchi_wasm.js with bundled wasm in function-wrapped form - let bindings = await readFile('./src/bindings/compiled/web_bindings/kimchi_wasm.js', 'utf8'); - bindings = rewriteWasmBindings(bindings); - let tmpBindingsPath = 'src/bindings/compiled/web_bindings/kimchi_wasm.tmp.js'; - await writeFile(tmpBindingsPath, bindings); - await esbuild.build({ - entryPoints: [tmpBindingsPath], - bundle: true, - format: 'esm', - outfile: tmpBindingsPath, - target: 'esnext', - plugins: [wasmPlugin()], - allowOverwrite: true, - sourcemap: true, - }); - bindings = await readFile(tmpBindingsPath, 'utf8'); - bindings = rewriteBundledWasmBindings(bindings); - await writeFile(tmpBindingsPath, bindings); - // run typescript await execPromise('npx tsc -p tsconfig.web.json'); // copy over pure js files await copy({ - './src/bindings/compiled/web_bindings/': './dist/web/web_bindings/', './src/bindings.d.ts': './dist/web/bindings.d.ts', './src/bindings.web.js': './dist/web/bindings.js', './src/bindings/js/web/': './dist/web/bindings/js/web/', }); + // bundle the napi-rs wasm loaders so that their `@napi-rs/wasm-runtime` + // imports are resolved; the .wasm binary itself stays a separate file which + // the loader fetches relative to `import.meta.url` + await esbuild.build({ + entryPoints: ['./src/bindings/compiled/web_bindings/kimchi_napi.wasi-browser.js'], + bundle: true, + format: 'esm', + outfile: './dist/web/web_bindings/kimchi_napi.wasi-browser.js', + target: 'esnext', + external: ['*.wasm', '*.mjs'], + logLevel: 'error', + minify, + sourcemap: true, + }); + await esbuild.build({ + entryPoints: ['./src/bindings/compiled/web_bindings/wasi-worker-browser.mjs'], + bundle: true, + format: 'esm', + outfile: './dist/web/web_bindings/wasi-worker-browser.mjs', + target: 'esnext', + external: ['*.wasm'], + logLevel: 'error', + minify, + sourcemap: true, + }); + await copy({ + './src/bindings/compiled/web_bindings/kimchi_napi.wasm32-wasi.wasm': + './dist/web/web_bindings/kimchi_napi.wasm32-wasi.wasm', + './src/bindings/compiled/web_bindings/o1js_web.bc.js': './dist/web/web_bindings/o1js_web.bc.js', + }); + if (minify) { let o1jsWebPath = './dist/web/web_bindings/o1js_web.bc.js'; let o1jsWeb = await readFile(o1jsWebPath, 'utf8'); @@ -65,10 +76,6 @@ async function buildWeb({ production }) { await writeFile(o1jsWebPath, code); } - // overwrite kimchi_wasm with bundled version - await copy({ [tmpBindingsPath]: './dist/web/web_bindings/kimchi_wasm.js' }); - await unlink(tmpBindingsPath); - // move all .web.js files to their .js counterparts let webFiles = glob.sync('./dist/web/**/*.web.js'); await Promise.all( @@ -83,7 +90,7 @@ async function buildWeb({ production }) { format: 'esm', outfile: 'dist/web/index.js', resolveExtensions: ['.js', '.ts'], - plugins: [wasmPlugin(), srcStringPlugin()], + plugins: [makeWasiLoaderExternal(), srcStringPlugin()], dropLabels: ['CJS'], external: ['*.bc.js'], target, @@ -120,52 +127,22 @@ function execPromise(cmd) { ); } -function rewriteWasmBindings(src) { - src = src - .replace("new URL('kimchi_wasm_bg.wasm', import.meta.url)", 'wasmCode') - .replace('import.meta.url', '"/"'); - return `import wasmCode from './kimchi_wasm_bg.wasm'; - let startWorkers, terminateWorkers; -${src}`; -} -function rewriteBundledWasmBindings(src) { - let i = src.indexOf('export {'); - let exportSlice = src.slice(i); - let defaultExport = exportSlice.match(/\w* as default/)[0]; - exportSlice = exportSlice - .replace(defaultExport, `default: __wbg_init`) - .replace('export', 'return'); - src = src.slice(0, i) + exportSlice; - - src = src.replace('var startWorkers;\n', ''); - src = src.replace('var terminateWorkers;\n', ''); - - // Force wasm-bindgen thread stack size to 1 MiB for web, matching the node - // build patch in fix-wasm-bindings-node.js. wasm-bindgen >= 0.2.100 - // defaults to 2 MiB which doubles memory pressure during worker startup. - src = src.replace( - 'wasm.__wbindgen_start(thread_stack_size)', - 'wasm.__wbindgen_start(thread_stack_size ?? 1048576)' - ); - - return `import { startWorkers, terminateWorkers } from '../bindings/js/web/worker-helpers.js' -export {kimchiWasm as default}; -function kimchiWasm() { - ${src} -} -kimchiWasm.deps = [startWorkers, terminateWorkers]`; -} - -function wasmPlugin() { +// keep the (pre-bundled) napi-rs wasm loader external to the main bundle, and +// rewrite its import path relative to the bundle output (dist/web/index.js) +function makeWasiLoaderExternal() { + let isWasiLoader = /kimchi_napi\.wasi-browser\.js$/; return { - name: 'wasm-plugin', + name: 'plugin-wasi-external', setup(build) { - build.onLoad({ filter: /\.wasm$/ }, async ({ path }) => { - return { - contents: await readFile(path), - loader: 'binary', - }; - }); + build.onResolve({ filter: isWasiLoader }, ({ path: filePath, resolveDir }) => ({ + path: + './' + + path.relative( + path.resolve('.', 'dist/web'), + path.resolve(resolveDir, filePath).replace('/compiled/web_bindings/', '/web_bindings/') + ), + external: true, + })); }, }; } diff --git a/src/build/copy-to-dist.js b/src/build/copy-to-dist.js index 09249d8fc8..0c03451247 100644 --- a/src/build/copy-to-dist.js +++ b/src/build/copy-to-dist.js @@ -6,7 +6,7 @@ await copyFromTo( 'src/bindings.d.ts', 'src/bindings/compiled/_node_bindings', 'src/bindings/compiled/native', - 'src/bindings/compiled/node_bindings/kimchi_wasm.d.cts', + 'src/bindings/compiled/node_bindings/kimchi_napi.wasi.d.cts', ], 'src/', 'dist/node/' diff --git a/src/build/fix-wasm-bindings-node.js b/src/build/fix-wasm-bindings-node.js deleted file mode 100644 index 505c1a88d4..0000000000 --- a/src/build/fix-wasm-bindings-node.js +++ /dev/null @@ -1,50 +0,0 @@ -import fs from 'node:fs/promises'; - -const file = process.argv[2]; - -let src = await fs.readFile(file, 'utf8'); - -// wasm-bindgen <= 0.2.99 pattern -src = src.replace( - "imports['env'] = require('env');", - ` -let { isMainThread, workerData } = require('worker_threads'); - -let env = {}; -if (isMainThread) { - env.memory = new WebAssembly.Memory({ - initial: 20, - maximum: 65536, - shared: true, - }); -} else { - env.memory = workerData.memory; -} - -imports['env'] = env; -` -); - -// wasm-bindgen >= 0.2.100 pattern -src = src.replace( - /imports\.wbg\s*=\s*\{\s*memory:\s*new WebAssembly\.Memory\((\{[\s\S]*?\})\)\s*\};/, - ` -let { isMainThread, workerData } = require('worker_threads'); -let wbgMemory = isMainThread - ? new WebAssembly.Memory($1) - : workerData.memory; -imports.wbg = { memory: wbgMemory }; -` -); - -// Force wasm-bindgen thread stack size explicitly for node target. -// 1 MiB was the previous default before wasm-bindgen raised it to 2 MiB. -src = src.replace( - 'wasm.__wbindgen_start();', - ` -const __o1jsThreadStackSize = Number(process?.env?.O1JS_WASM_THREAD_STACK_SIZE ?? 1048576); -wasm.__wbindgen_start(__o1jsThreadStackSize); -` -); - -await fs.writeFile(file, src, 'utf8'); diff --git a/src/lib/proof-system/prover-keys.ts b/src/lib/proof-system/prover-keys.ts index 9872262aed..cf52a98b53 100644 --- a/src/lib/proof-system/prover-keys.ts +++ b/src/lib/proof-system/prover-keys.ts @@ -7,9 +7,10 @@ */ import { Pickles, wasm } from '../../bindings.js'; import { + ExternalObject, WasmPastaFpPlonkIndex, WasmPastaFqPlonkIndex, -} from '../../bindings/compiled/node_bindings/kimchi_wasm.cjs'; +} from '../../bindings/compiled/node_bindings/kimchi_napi.wasi.cjs'; // TODO: include conversion bundle to decide between wasm and napi conversion import { getRustConversion } from '../../bindings/crypto/bindings.js'; import { VerifierIndex } from '../../bindings/crypto/bindings/kimchi-types.js'; @@ -35,9 +36,9 @@ type SnarkKeyHeader = | [KeyType.WrapVerificationKey, MlWrapVerificationKeyHeader]; type SnarkKey = - | [KeyType.StepProvingKey, MlBackendKeyPair] + | [KeyType.StepProvingKey, MlBackendKeyPair>] | [KeyType.StepVerificationKey, VerifierIndex] - | [KeyType.WrapProvingKey, MlBackendKeyPair] + | [KeyType.WrapProvingKey, MlBackendKeyPair>] | [KeyType.WrapVerificationKey, MlWrapVerificationKey]; /** @@ -56,9 +57,7 @@ function parseHeader( let methodIndex = header[1][3]; let methodName = methods[methodIndex].methodName; let persistentId = sanitize(`${kind}-${programName}-${methodName}`); - let uniqueId = sanitize( - `${kind}-${programName}-${methodIndex}-${methodName}-${hash}` - ); + let uniqueId = sanitize(`${kind}-${programName}-${methodIndex}-${methodName}-${hash}`); return { version: cacheHeaderVersion, uniqueId, diff --git a/src/mina b/src/mina index b26d0f9530..8182bf2628 160000 --- a/src/mina +++ b/src/mina @@ -1 +1 @@ -Subproject commit b26d0f95307bef300046435e598287d7784c2e47 +Subproject commit 8182bf2628f395f58966c7537015925f2dbb6cc7 From 3b5224af954e8149b0543ea43291c732567662e2 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 2 Jul 2026 22:53:42 +0700 Subject: [PATCH 02/14] move to napi-rs wasm --- .github/workflows/build.yml | 3 +- PLAN.md | 30 +++++---- README-dev.md | 33 ++++------ flake.nix | 19 +----- package-lock.json | 29 ++++++--- scripts/build/wasm/build-node.sh | 64 +++++++++++++++++++ src/bindings/js/web/buffer-polyfill.js | 25 ++++++++ .../js/web/wasm-runtime-no-threads.js | 49 ++++++++++++++ src/bindings/js/web/web-backend.js | 6 +- src/bindings/ocaml/jsoo_exports/dune | 58 ----------------- src/build/build-web.js | 21 +++++- src/examples/plain-html/server.js | 16 +++-- src/examples/zkprogram/program.ts | 3 +- 13 files changed, 228 insertions(+), 128 deletions(-) create mode 100644 src/bindings/js/web/buffer-polyfill.js create mode 100644 src/bindings/js/web/wasm-runtime-no-threads.js diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd94de780a..394b7473c3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,7 +42,6 @@ jobs: lmdb \ pkgconf \ postgresql@15 \ - wasm-pack \ wasm-tools \ wabt \ bash \ @@ -135,7 +134,7 @@ jobs: uses: dtolnay/rust-toolchain@master with: toolchain: nightly-2025-12-11 - targets: wasm32-unknown-unknown + targets: wasm32-unknown-unknown,wasm32-wasip1-threads components: rust-src - name: Setup go diff --git a/PLAN.md b/PLAN.md index 8e20e73da4..812042e414 100644 --- a/PLAN.md +++ b/PLAN.md @@ -315,17 +315,23 @@ lands. - `tsc -p tsconfig.node.json` and full-repo `tsc` clean (modulo pre-existing examples-vs-dist errors). -**Still needed (CI / follow-up):** - -1. Rebuild JSOO artifacts (`npm run build:jsoo` in CI) — old compiled artifacts - reference the removed `kimchi_wasm.cjs` and won't work with this branch. -2. Run the full jest + vk/perf-regression suites for `O1JS_BACKEND=wasm|native`; - re-dump wasm perf baselines (napi-wasm ≠ wasm-bindgen perf profile). -3. Web: benchmark Option A; implement Option B (worker-hosted RPC) if web - proving parallelism is required; validate SharedArrayBuffer/COOP-COEP e2e via - playwright. +**Still needed (CI / follow-up)** — see `STATE.md` for the up-to-date status: + +1. ~~Rebuild JSOO artifacts~~ — done locally (`build:bindings-all` passes + end-to-end; the OCaml toolchain was available after all). +2. Run the vk/perf-regression suites for `O1JS_BACKEND=wasm|native`; re-dump + wasm perf baselines (napi-wasm ≠ wasm-bindgen perf profile). Jest suites + pass 18/18 on both backends; web playwright e2e passes 5/5. +3. ~~Web: validate Option A via playwright~~ — done (needed three fixes: + explicit single-threaded rayon init, disabled wasi thread-spawn, Buffer + polyfill — see AGENT_LOG.md). Option B (worker-hosted RPC) remains the + follow-up if web proving parallelism is required. 4. Decide on iOS memory ceiling (generated loader hardcodes max 4 GiB; old code used 1 GiB on iOS) — post-process or napi config per-target if needed. -5. Review the o1-labs napi-rs fork delta vs upstream before web rollout; the - `kimchi_wasm` crate and its dune targets in the mina repo can be retired once - no consumer remains. +5. Review the o1-labs napi-rs fork delta vs upstream (now also motivated by a + load-sensitive rayon-worker trap on node-wasm, see AGENT_LOG.md). + ~~Retire the `kimchi_wasm` crate and its dune targets~~ — done in the + submodule working trees (proof-systems crate + xtask build-wasm + + wasm-pack dep deleted; mina `kimchi_bindings/js` packaging, `o1js_stub` + and test `link_deps` removed); mina-side nix/buildkite plumbing remains + for the upstream merge. diff --git a/README-dev.md b/README-dev.md index 6b74f3d533..d094383c35 100644 --- a/README-dev.md +++ b/README-dev.md @@ -174,25 +174,20 @@ located in the Mina repo under `src/mina`. See the [Kimchi README](https://github.com/o1-labs/proof-systems/blob/master/README.md) for more information. -To compile the Wasm code, a combination of Cargo and Dune is used. Both build -files are located under `src/mina/src/lib/crypto/kimchi`, where the `wasm` -folder contains the Rust code that is compiled to Wasm, and the `js` folder that -contains a wrapper around the Wasm code which allows Js_of_ocaml to compile -against the Wasm backend. - -For the Wasm build, the output files are: - -- `kimchi_wasm_bg.wasm`: The compiled WebAssembly binary. -- `kimchi_wasm_bg.wasm.d.ts`: TypeScript definition files describing the types - of .wasm or .js files. -- `kimchi_wasm.js`: JavaScript file that wraps the Wasm code for use in Node.js. -- `kimchi_wasm.d.ts`: TypeScript definition file for kimchi_wasm.js. - -Similarly, for internal development and debugging, you can manually build the -WASM bindings for Node and Web using `npm run build:wasm:node` and -`npm run build:wasm:web`, respectively. For typical local development, however, -running the standard build commands automatically generates these bindings as -part of the overall build process. +The wasm backend is the `wasm32-wasip1-threads` build of the `kimchi-napi` +crate — the same napi-rs crate that powers the native (`.node`) backend — built +via the napi-rs CLI (see `scripts/build/wasm/build-kimchi-napi-wasm.sh`). The +crate lives in the proof-systems submodule under +`src/mina/src/lib/crypto/proof-systems/kimchi-napi`. + +For the wasm build, the output files are: + +- `kimchi_napi.wasm32-wasi.wasm`: The compiled WebAssembly binary. +- `kimchi_napi.wasi.cjs` / `kimchi_napi.wasi-browser.js`: generated loaders for + Node.js and the browser, backed by `@napi-rs/wasm-runtime`. +- `wasi-worker.mjs` / `wasi-worker-browser.mjs`: worker files used by the + runtime to spawn threads. +- `kimchi_napi.wasi.d.cts`: TypeScript definitions generated from the crate. Similarly, for internal development and debugging, you can manually build the WASM bindings for Node and Web using `npm run build:wasm:node` and diff --git a/flake.nix b/flake.nix index ca989107b7..10dc0d901f 100644 --- a/flake.nix +++ b/flake.nix @@ -117,16 +117,7 @@ info noTestSkipping prj; - prj = prj_ // { - pkgs = prj_.pkgs // { - __ocaml-js__ = prj_.pkgs.__ocaml-js__.overrideAttrs { - PREBUILT_KIMCHI_BINDINGS_JS_WEB = - "${mina.files.src-lib-crypto-kimchi_bindings-js-web}/src/lib/crypto/kimchi_bindings/js/web"; - PREBUILT_KIMCHI_BINDINGS_JS_NODE_JS = - "${mina.files.src-lib-crypto-kimchi_bindings-js-node_js}/src/lib/crypto/kimchi_bindings/js/node_js"; - }; - }; - }; + prj = prj_; rust-channel = ((pkgs.rustChannelOf { @@ -137,6 +128,7 @@ { targets = [ "wasm32-unknown-unknown" + "wasm32-wasip1-threads" "x86_64-unknown-linux-gnu" "aarch64-apple-darwin" "x86_64-apple-darwin" @@ -173,7 +165,6 @@ typescript nodePackages.typescript-language-server rustup - wasm-pack binaryen # provides wasm-opt dune_3 ] ++ commonOverrides.buildInputs; @@ -324,15 +315,9 @@ ]; }); inherit (inputs.mina.devShells."${system}".default) - KIMCHI_WASM_NODEJS - KIMCHI_WASM_WEB KIMCHI_STUBS KIMCHI_STUBS_STATIC_LIB ; - PREBUILT_KIMCHI_BINDINGS_JS_WEB = - "${mina.files.src-lib-crypto-kimchi_bindings-js-web}/src/lib/crypto/kimchi_bindings/js/web"; - PREBUILT_KIMCHI_BINDINGS_JS_NODE_JS = - "${mina.files.src-lib-crypto-kimchi_bindings-js-node_js}/src/lib/crypto/kimchi_bindings/js/node_js"; EXPORT_TEST_VECTORS = "${test-vectors}/bin/export_test_vectors"; SKIP_MINA_COMMIT = true; SKIP_NATIVE_BUILD = true; diff --git a/package-lock.json b/package-lock.json index 7aaaeeb044..4ef39d97a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -102,6 +102,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.23.3.tgz", "integrity": "sha512-Jg+msLuNuCJDyBvFv5+OKOUjWMZgd85bKjbICd3zWrKAo+bJ49HJufi7CQE0q0uR8NGyO6xkCACScNqyjHSZew==", "dev": true, + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.22.13", @@ -567,28 +568,28 @@ "dev": true }, "node_modules/@emnapi/core": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.7.0.tgz", - "integrity": "sha512-pJdKGq/1iquWYtv1RRSljZklxHCOCAJFJrImO5ZLKPJVJlVUcs8yFwNQlqS0Lo8xT1VAXXTCZocF9n26FWEKsw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", "license": "MIT", "dependencies": { - "@emnapi/wasi-threads": "1.1.0", + "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.0.tgz", - "integrity": "sha512-oAYoQnCYaQZKVS53Fq23ceWMRxq5EhQsE0x0RdQ55jT7wagMu5k+fS39v1fiSLrtrLQlXwVINenqhLMtTrV/1Q==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", "license": "MIT", "dependencies": { "tslib": "^2.4.0" } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", - "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", "license": "MIT", "dependencies": { "tslib": "^2.4.0" @@ -3490,6 +3491,7 @@ "integrity": "sha512-1LFfa/qnMQvEOAdzlQymH0ulepxbxnCYAKJZfMci/5XJyIHWgEYnDmgnKakbTh7CH2tFQ5O60oYDvns4i9RAIg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.1.0", @@ -3719,6 +3721,7 @@ "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -4207,6 +4210,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.9.tgz", "integrity": "sha512-0f5klcuImLnG4Qreu9hPj/rEfFq6YRc5n2mAjSsH+ec/mJL+3voBH0+8T7o8RpFjH7ovc+TRsL/c7OYIQsPTfQ==", "dev": true, + "peer": true, "dependencies": { "undici-types": "~5.26.4" } @@ -4486,6 +4490,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001541", "electron-to-chromium": "^1.4.535", @@ -5630,6 +5635,7 @@ "resolved": "https://registry.npmjs.org/jest/-/jest-28.1.3.tgz", "integrity": "sha512-N4GT5on8UkZgH0O5LUavMRV1EDEhNTL0KEfRmDIeZHSV7p2XgLoY9t9VDUgL6o+yfdgYHVxuz81G8oB9VG5uyA==", "dev": true, + "peer": true, "dependencies": { "@jest/core": "^28.1.3", "@jest/types": "^28.1.3", @@ -7641,6 +7647,7 @@ "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -8393,6 +8400,7 @@ "integrity": "sha512-5PzUddaA9FbaarUzIsEc4wNXCiO4Ot3bJNeMF2qKpYlTmM9TTaSHQ7162w756ERCkXER/+o2purRG6YOAv6EMA==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@gerrit0/mini-shiki": "^3.2.2", "lunr": "^2.3.9", @@ -8465,6 +8473,7 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz", "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==", "dev": true, + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/scripts/build/wasm/build-node.sh b/scripts/build/wasm/build-node.sh index bc30f2d7cf..94c362fcca 100755 --- a/scripts/build/wasm/build-node.sh +++ b/scripts/build/wasm/build-node.sh @@ -34,4 +34,68 @@ cp $ARTIFACTS_PATH/kimchi_napi.wasi.cjs $BINDINGS_PATH/ cp $ARTIFACTS_PATH/wasi-worker.mjs $BINDINGS_PATH/ cp $ARTIFACTS_PATH/index.d.ts $BINDINGS_PATH/kimchi_napi.wasi.d.cts +info "hardening generated loader against silent thread death..." + +# a rayon worker thread that dies (wasm trap) or never starts (racy thread +# bootstrap) deadlocks the process silently: the parent's event loop is +# blocked inside the synchronous wasm call, so worker 'error'/'exit' events +# and forwarded stderr are never delivered, and rayon waits forever for the +# missing thread (all threads end up parked in Atomics.wait). three fixes: +# +# 1. waitThreadStart: block each spawn (allowed on node) until the thread has +# actually started — a thread that cannot start becomes a loud EAGAIN +# instead of a phantom tid. +# 2/3. 'error' and 'exit' handlers that crash the process with a stack trace +# (fs.writeSync bypasses the blocked event loop). rayon threads live for +# the process lifetime, so any worker death while the process runs is +# fatal anyway — a crash beats an undebuggable hang. +node -e ' + let fs = require("fs"); + let path = process.argv[1]; + let src = fs.readFileSync(path, "utf8"); + + let reuseAnchor = " reuseWorker: true,"; + if (!src.includes(reuseAnchor)) throw Error("waitThreadStart anchor not found in " + path); + src = src.replace(reuseAnchor, reuseAnchor + "\n waitThreadStart: 10000,"); + + let workerDied = (event) => ` worker.on("${event}", (e) => { + try { + require("fs").writeSync( + 2, + "[o1js] fatal: wasm worker thread ${event === "exit" ? "exited" : "crashed"} (this would deadlock rayon):\\n" + + (e && (e.stack || e.message || e)) + "\\n" + ); + } catch (_) {} + process.exit(70); + }); +`; + let anchor = " worker.onmessage = ({ data }) => {"; + if (!src.includes(anchor)) throw Error("loader hardening anchor not found in " + path); + src = src.replace(anchor, workerDied("error") + workerDied("exit") + anchor); + fs.writeFileSync(path, src); +' $BINDINGS_PATH/kimchi_napi.wasi.cjs + +# the parent-side handlers above only fire when the main thread is between +# wasm calls — during proving it is blocked inside one, events queue forever. +# worker_threads share the OS pid, so the WORKER can break the deadlock +# itself: log the trap with a synchronous write (bypasses all event loops) +# and abort the whole process. +cat >> $BINDINGS_PATH/wasi-worker.mjs <<'WORKER_HARDENING' + +// o1js hardening: a wasm trap on this thread would otherwise deadlock rayon +// silently — the parent cannot observe worker death while it is blocked +// inside a synchronous wasm call. abort the whole process (worker_threads +// share the pid) so the failure is loud and carries a stack trace. +process.on('uncaughtException', (e) => { + try { + fs.writeSync( + 2, + '[o1js] fatal: wasm worker thread crashed (this would deadlock rayon):\n' + + (e && (e.stack || e.message || e)) + '\n' + ); + } catch (_) {} + process.kill(process.pid, 'SIGABRT'); +}); +WORKER_HARDENING + success "WASM node build success!" diff --git a/src/bindings/js/web/buffer-polyfill.js b/src/bindings/js/web/buffer-polyfill.js new file mode 100644 index 0000000000..7464099fed --- /dev/null +++ b/src/bindings/js/web/buffer-polyfill.js @@ -0,0 +1,25 @@ +// emnapi needs a `Buffer` to implement napi_create_buffer & friends (kimchi +// exposes Buffer-typed values, e.g. verifier-index shifts). browsers don't +// have one; emnapi only uses `from`, `alloc` and instanceof checks, so a thin +// Uint8Array subclass is enough. +// +// @emnapi/runtime captures `Buffer` at module evaluation time, so this module +// must be imported (for its side effect) BEFORE @napi-rs/wasm-runtime. +class BufferPolyfill extends Uint8Array { + static alloc(size) { + return new BufferPolyfill(size); + } + static from(value, offset, length) { + if (value instanceof ArrayBuffer || value instanceof SharedArrayBuffer) { + return offset === undefined + ? new BufferPolyfill(value) + : new BufferPolyfill(value, offset, length); + } + // typed array / array-like: copy + return new BufferPolyfill(value); + } + static isBuffer(value) { + return value instanceof BufferPolyfill; + } +} +if (globalThis.Buffer === undefined) globalThis.Buffer = BufferPolyfill; diff --git a/src/bindings/js/web/wasm-runtime-no-threads.js b/src/bindings/js/web/wasm-runtime-no-threads.js new file mode 100644 index 0000000000..dc8eb38208 --- /dev/null +++ b/src/bindings/js/web/wasm-runtime-no-threads.js @@ -0,0 +1,49 @@ +// wrapper around @napi-rs/wasm-runtime for the browser MAIN thread. +// +// browser main threads cannot block (`Atomics.wait` / `memory.atomic.wait32` +// trap), but wasi thread spawning needs the spawning thread to block or yield +// before the worker can start, and rayon blocks the calling thread at every +// join point. so with real thread support, kimchi's first rayon call traps +// with "Atomics.wait cannot be called in this context". +// +// instead, we make `wasi::thread-spawn` fail with ENOSYS (52). rust std maps +// that to io::ErrorKind::Unsupported, which makes rayon-core's global-pool +// init fall back to a sequential pool on the current thread (see +// rayon-core/src/registry.rs, default_global_registry) — everything runs +// inline on the main thread, no blocking needed. this is "Option A" of the +// napi-wasm web design, see PLAN.md §3 Phase 2. +// the polyfill must run before @emnapi/runtime is evaluated (it captures +// `Buffer` at module load), hence the import order here +import './buffer-polyfill.js'; + +export * from '@napi-rs/wasm-runtime'; +import { + instantiateNapiModule as _instantiateNapiModule, + instantiateNapiModuleSync as _instantiateNapiModuleSync, +} from '@napi-rs/wasm-runtime'; + +export { instantiateNapiModule, instantiateNapiModuleSync }; + +const ENOSYS = 52; // wasi preview1 errno + +function disableThreadSpawn(options) { + let userOverwrite = options.overwriteImports; + return { + ...options, + overwriteImports(importObject) { + if (typeof userOverwrite === 'function') { + importObject = userOverwrite(importObject) ?? importObject; + } + importObject.wasi = { ...importObject.wasi, 'thread-spawn': () => -ENOSYS }; + return importObject; + }, + }; +} + +function instantiateNapiModule(wasmInput, options) { + return _instantiateNapiModule(wasmInput, disableThreadSpawn(options)); +} + +function instantiateNapiModuleSync(wasmInput, options) { + return _instantiateNapiModuleSync(wasmInput, disableThreadSpawn(options)); +} diff --git a/src/bindings/js/web/web-backend.js b/src/bindings/js/web/web-backend.js index d41e536485..890a9788d0 100644 --- a/src/bindings/js/web/web-backend.js +++ b/src/bindings/js/web/web-backend.js @@ -19,8 +19,12 @@ async function initializeBindings() { // NOTE on threading: browser main threads cannot block, so rayon-parallel // sections that make the calling thread wait must not run on the main // thread with a multi-threaded pool. Until worker-hosted execution lands - // (see PLAN.md, web Option B), the pool is limited to inline execution. + // (see PLAN.md, web Option B), the pool is limited to inline execution: + // the bundled loader disables wasi thread spawning (see + // wasm-runtime-no-threads.js) and rayon's global pool is pinned to the + // current thread before the first rayon call. wasm = kimchiNapi.default ?? kimchiNapi; + wasm.camlRayonInitSingleThreaded(); wasm.__kimchi_backend = 'native'; if (typeof globalThis !== 'undefined') { diff --git a/src/bindings/ocaml/jsoo_exports/dune b/src/bindings/ocaml/jsoo_exports/dune index a00b9649cd..76a093782f 100644 --- a/src/bindings/ocaml/jsoo_exports/dune +++ b/src/bindings/ocaml/jsoo_exports/dune @@ -3,62 +3,6 @@ (js_of_ocaml (compilation_mode whole_program)))) -(rule - (enabled_if - (= %{env:PREBUILT_KIMCHI_BINDINGS_JS_NODE_JS=n} n)) - (targets node_js_kimchi_wasm.js node_js_kimchi_wasm_bg.wasm) - (deps - (:d1 ../../../mina/src/lib/crypto/kimchi_bindings/js/node_js/kimchi_wasm.js) - (:d2 - ../../../mina/src/lib/crypto/kimchi_bindings/js/node_js/kimchi_wasm_bg.wasm)) - (action - (progn - (run cp %{d1} node_js_kimchi_wasm.js) - (run cp %{d2} node_js_kimchi_wasm_bg.wasm)))) - -(rule - (enabled_if - (= %{env:PREBUILT_KIMCHI_BINDINGS_JS_WEB=n} n)) - (targets web_kimchi_wasm.js web_kimchi_wasm_bg.wasm) - (deps - (:d1 ../../../mina/src/lib/crypto/kimchi_bindings/js/web/kimchi_wasm.js) - (:d2 - ../../../mina/src/lib/crypto/kimchi_bindings/js/web/kimchi_wasm_bg.wasm)) - (action - (progn - (run cp %{d1} web_kimchi_wasm.js) - (run cp %{d2} web_kimchi_wasm_bg.wasm)))) - -(rule - (enabled_if - (<> %{env:PREBUILT_KIMCHI_BINDINGS_JS_NODE_JS=n} n)) - (targets node_js_kimchi_wasm.js node_js_kimchi_wasm_bg.wasm) - (action - (progn - (run - cp - %{env:PREBUILT_KIMCHI_BINDINGS_JS_NODE_JS=n}/kimchi_wasm.js - node_js_kimchi_wasm.js) - (run - cp - %{env:PREBUILT_KIMCHI_BINDINGS_JS_NODE_JS=n}/kimchi_wasm_bg.wasm - node_js_kimchi_wasm_bg.wasm)))) - -(rule - (enabled_if - (<> %{env:PREBUILT_KIMCHI_BINDINGS_JS_WEB=n} n)) - (targets web_kimchi_wasm.js web_kimchi_wasm_bg.wasm) - (action - (progn - (run - cp - %{env:PREBUILT_KIMCHI_BINDINGS_JS_WEB=n}/kimchi_wasm.js - web_kimchi_wasm.js) - (run - cp - %{env:PREBUILT_KIMCHI_BINDINGS_JS_WEB=n}/kimchi_wasm_bg.wasm - web_kimchi_wasm_bg.wasm)))) - (executable (name o1js_node) (modules o1js_node) @@ -71,7 +15,6 @@ (libraries o1js_bindings.lib bindings_js.node_backend) - (link_deps node_js_kimchi_wasm.js node_js_kimchi_wasm_bg.wasm) (instrumentation (backend bisect_ppx)) (forbidden_libraries async core re2 ctypes) @@ -87,7 +30,6 @@ (flags :standard +toplevel.js +dynlink.js) (javascript_files overrides.js)) (libraries o1js_bindings.lib bindings_js.web_backend) - (link_deps web_kimchi_wasm.js web_kimchi_wasm_bg.wasm) (instrumentation (backend bisect_ppx)) (forbidden_libraries async core re2 ctypes) diff --git a/src/build/build-web.js b/src/build/build-web.js index b39a36dfed..96709edd00 100644 --- a/src/build/build-web.js +++ b/src/build/build-web.js @@ -36,7 +36,10 @@ async function buildWeb({ production }) { // bundle the napi-rs wasm loaders so that their `@napi-rs/wasm-runtime` // imports are resolved; the .wasm binary itself stays a separate file which - // the loader fetches relative to `import.meta.url` + // the loader fetches relative to `import.meta.url`. + // the main-thread loader gets a wrapped runtime that disables wasi thread + // spawning — browser main threads cannot block, so rayon must run inline + // (see src/bindings/js/web/wasm-runtime-no-threads.js) await esbuild.build({ entryPoints: ['./src/bindings/compiled/web_bindings/kimchi_napi.wasi-browser.js'], bundle: true, @@ -44,6 +47,7 @@ async function buildWeb({ production }) { outfile: './dist/web/web_bindings/kimchi_napi.wasi-browser.js', target: 'esnext', external: ['*.wasm', '*.mjs'], + plugins: [noThreadsWasmRuntimePlugin()], logLevel: 'error', minify, sourcemap: true, @@ -127,6 +131,21 @@ function execPromise(cmd) { ); } +// redirect the main-thread loader's `@napi-rs/wasm-runtime` import to the +// no-threads wrapper (the wrapper itself still resolves the real runtime) +function noThreadsWasmRuntimePlugin() { + let shim = path.resolve('./src/bindings/js/web/wasm-runtime-no-threads.js'); + return { + name: 'no-threads-wasm-runtime', + setup(build) { + build.onResolve({ filter: /^@napi-rs\/wasm-runtime$/ }, ({ importer }) => { + if (importer === shim) return null; + return { path: shim }; + }); + }, + }; +} + // keep the (pre-bundled) napi-rs wasm loader external to the main bundle, and // rewrite its import path relative to the bundle output (dist/web/index.js) function makeWasiLoaderExternal() { diff --git a/src/examples/plain-html/server.js b/src/examples/plain-html/server.js index dc7679626c..157d81d50a 100644 --- a/src/examples/plain-html/server.js +++ b/src/examples/plain-html/server.js @@ -16,7 +16,8 @@ const server = http.createServer(async (req, res) => { if (file === './') file = './index.html'; let content; try { - content = await fs.readFile(path.resolve('./dist/web', file), 'utf8'); + // read as a buffer — .wasm files are binary and must not go through utf8 + content = await fs.readFile(path.resolve('./dist/web', file)); } catch (err) { res.writeHead(404, defaultHeaders); res.write('404'); @@ -25,11 +26,14 @@ const server = http.createServer(async (req, res) => { } const extension = path.basename(file).split('.').pop(); - const contentType = { - html: 'text/html', - js: 'application/javascript', - map: 'application/json', - }[extension]; + const contentType = + { + html: 'text/html', + js: 'application/javascript', + mjs: 'application/javascript', + map: 'application/json', + wasm: 'application/wasm', + }[extension] ?? 'application/octet-stream'; const headers = { ...defaultHeaders, 'content-type': contentType }; res.writeHead(200, headers); diff --git a/src/examples/zkprogram/program.ts b/src/examples/zkprogram/program.ts index 7fe3861cb7..26de846d87 100644 --- a/src/examples/zkprogram/program.ts +++ b/src/examples/zkprogram/program.ts @@ -1,5 +1,4 @@ -import { Field, ZkProgram, Cache, verify } from 'o1js'; - +import { Cache, Field, ZkProgram, verify } from 'o1js'; let MyProgram = ZkProgram({ name: 'example-with-output', publicOutput: Field, From ade35b872793f51c8c5ff873d4650fa6a9bf3070 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 4 Jul 2026 10:29:09 +0700 Subject: [PATCH 03/14] fix bindings --- .../checkpoint-2026-07-03T15-41-47-636Z.json | 16 ++ run-in-browser.js | 25 +- .../bindings/gate-vector-napi.unit-test.ts | 3 +- .../crypto/native/napi-conversion-core.ts | 53 ++-- src/bindings/js/node/node-backend.js | 14 +- src/bindings/js/web/ffi-proxy.js | 205 ++++++++++++++++ src/bindings/js/web/ffi-worker-host.js | 232 ++++++++++++++++++ src/bindings/js/web/web-backend.js | 50 ++-- src/build/build-web.js | 14 ++ src/examples/plain-html/server.js | 3 +- 10 files changed, 555 insertions(+), 60 deletions(-) create mode 100644 .omc/state/checkpoints/checkpoint-2026-07-03T15-41-47-636Z.json create mode 100644 src/bindings/js/web/ffi-proxy.js create mode 100644 src/bindings/js/web/ffi-worker-host.js diff --git a/.omc/state/checkpoints/checkpoint-2026-07-03T15-41-47-636Z.json b/.omc/state/checkpoints/checkpoint-2026-07-03T15-41-47-636Z.json new file mode 100644 index 0000000000..abb96893f3 --- /dev/null +++ b/.omc/state/checkpoints/checkpoint-2026-07-03T15-41-47-636Z.json @@ -0,0 +1,16 @@ +{ + "created_at": "2026-07-03T15:41:47.635Z", + "trigger": "manual", + "active_modes": {}, + "todo_summary": { + "pending": 0, + "in_progress": 0, + "completed": 0 + }, + "wisdom_exported": false, + "background_jobs": { + "active": [], + "recent": [], + "stats": null + } +} \ No newline at end of file diff --git a/run-in-browser.js b/run-in-browser.js index 5e275b606f..06b6dd1689 100755 --- a/run-in-browser.js +++ b/run-in-browser.js @@ -1,8 +1,8 @@ #!/usr/bin/env node +import minimist from 'minimist'; import fs from 'node:fs/promises'; -import path from 'node:path'; import http from 'node:http'; -import minimist from 'minimist'; +import path from 'node:path'; import { build } from './src/build/build-example.js'; let { @@ -44,7 +44,7 @@ const indexHtml = ` `; -const port = 8000; +const port = 8001; const defaultHeaders = { 'content-type': 'text/html', 'Cross-Origin-Embedder-Policy': 'require-corp', @@ -52,7 +52,8 @@ const defaultHeaders = { }; const server = http.createServer(async (req, res) => { - let file = '.' + req.url; + // strip query strings; the ffi worker assets may be loaded with params + let file = '.' + new URL(req.url, 'http://localhost').pathname; if (file === './') file = './index.html'; // console.log('serving', file); @@ -60,7 +61,8 @@ const server = http.createServer(async (req, res) => { if (file === './index.html') content = indexHtml; else { try { - content = await fs.readFile(path.resolve('./dist/web', file), 'utf8'); + // read as a buffer — .wasm files are binary and must not go through utf8 + content = await fs.readFile(path.resolve('./dist/web', file)); } catch (err) { res.writeHead(404, defaultHeaders); res.write('404'); @@ -70,11 +72,14 @@ const server = http.createServer(async (req, res) => { } const extension = path.basename(file).split('.').pop(); - const contentType = { - html: 'text/html', - js: 'application/javascript', - map: 'application/json', - }[extension]; + const contentType = + { + html: 'text/html', + js: 'application/javascript', + mjs: 'application/javascript', + map: 'application/json', + wasm: 'application/wasm', + }[extension] ?? 'application/octet-stream'; const headers = { ...defaultHeaders, 'content-type': contentType }; res.writeHead(200, headers); diff --git a/src/bindings/crypto/bindings/gate-vector-napi.unit-test.ts b/src/bindings/crypto/bindings/gate-vector-napi.unit-test.ts index d8776c328f..0f661d4508 100644 --- a/src/bindings/crypto/bindings/gate-vector-napi.unit-test.ts +++ b/src/bindings/crypto/bindings/gate-vector-napi.unit-test.ts @@ -71,7 +71,8 @@ const sampleGate: Gate = [ const vector = gateVectorCreate(); expect(gateVectorLen(vector)).toBe(0); -gateVectorAdd(vector, fp.gateToRust(sampleGate)); +let flatGate = fp.gateToRust(sampleGate); +gateVectorAdd(vector, flatGate.typ, flatGate.wires, flatGate.coeffs); expect(gateVectorLen(vector)).toBe(1); const gate0 = gateVectorGet(vector, 0); diff --git a/src/bindings/crypto/native/napi-conversion-core.ts b/src/bindings/crypto/native/napi-conversion-core.ts index 20e9213d88..7759acd5db 100644 --- a/src/bindings/crypto/native/napi-conversion-core.ts +++ b/src/bindings/crypto/native/napi-conversion-core.ts @@ -6,7 +6,6 @@ import { fieldsToRustFlat, } from '../bindings/conversion-base.js'; import { Field, Gate, OrInfinity, PolyComm, Wire } from '../bindings/kimchi-types.js'; -import { mapTuple } from '../bindings/util.js'; import type { Napi, NapiAffine, @@ -59,23 +58,17 @@ function conversionCorePerField({ makeAffine, PolyComm }: NapiCoreClasses) { const vectorToRust = (fields: MlArray) => fieldsToRustFlat(fields); const vectorFromRust = fieldsFromRustFlat; + // flat encoding matching gate_vector_add (typ + Int32Array of 14 wire ints + + // coeff bytes) — nested objects cost one napi call per property read on wasm const gateToRust = (gate: Gate) => { const [, typ, [, ...wires], coeffs] = gate; - const mapped = mapTuple(wires, wireToRust); - const nativeWires = { - w0: mapped[0], - w1: mapped[1], - w2: mapped[2], - w3: mapped[3], - w4: mapped[4], - w5: mapped[5], - w6: mapped[6], - } as const; - return { - typ, - wires: nativeWires, - coeffs: Array.from(fieldsToRustFlat(coeffs)), - }; + let wireInts = new Int32Array(14); + for (let i = 0; i < 7; i++) { + let [, row, col] = wires[i]; + wireInts[2 * i] = row; + wireInts[2 * i + 1] = col; + } + return { typ, wires: wireInts, coeffs: fieldsToRustFlat(coeffs) }; }; const gateFromRust = (gate: { @@ -108,23 +101,21 @@ function conversionCorePerField({ makeAffine, PolyComm }: NapiCoreClasses) { return [0, gate.typ, wiresTuple, coeffs]; }; + // `WasmGVesta` / `WasmGPallas` are `#[napi(object)]` (plain JS objects), so points can be + // built in JS without an ffi call per point. the infinity case reuses the generator's + // coordinate bytes fetched once via makeAffine() — the rust side ignores x/y when + // infinity is set, and only ever copies out of the buffers, so sharing them is fine. + let affineOne: NapiAffine | undefined; const affineToRust = (pt: OrInfinity): NapiAffine => { - function isFinitePoint(point: OrInfinity): point is [0, [0, Field, Field]] { - return Array.isArray(point); - } - let res = makeAffine(); - if (!isFinitePoint(pt)) { - res.infinity = true; - } else { - const [, pair] = pt; - const [, x, y] = pair; - // `WasmGVesta` / `WasmGPallas` are `#[napi(object)]` (plain JS objects), so assigning the - // same backing buffer to both `x` and `y` corrupts the point. Always use distinct byte - // arrays for each coordinate. - res.x = fieldToRust(x); - res.y = fieldToRust(y); + if (!Array.isArray(pt)) { + affineOne ??= makeAffine(); + return { x: affineOne.x, y: affineOne.y, infinity: true }; } - return res; + const [, pair] = pt as [0, [0, Field, Field]]; + const [, x, y] = pair; + // distinct byte arrays for each coordinate — assigning the same backing buffer to + // both `x` and `y` corrupts the point + return { x: fieldToRust(x), y: fieldToRust(y), infinity: false }; }; const affineFromRust = (pt: NapiAffine): OrInfinity => { if (pt.infinity) return 0; diff --git a/src/bindings/js/node/node-backend.js b/src/bindings/js/node/node-backend.js index 9748a86f14..9e6eb87922 100644 --- a/src/bindings/js/node/node-backend.js +++ b/src/bindings/js/node/node-backend.js @@ -47,6 +47,18 @@ function requireKimchiNapiWasm() { function setRayonThreadCount() { if (typeof process === 'undefined') return; if (process.env.RAYON_NUM_THREADS !== undefined) return; - let numThreads = Math.max(1, workers.numWorkers ?? (os.availableParallelism?.() ?? 1) - 1); + // wasm32 memory is capped at 4 GiB and per-thread proving memory adds up — + // on many-core machines a full-width pool can exhaust the heap on + // proof-heavy workloads past the point where extra threads help. measured + // on a 12-core M4 (dynamic-call.unit-test, arkworks `parallel` features + + // flat ffi encodings on): 2 threads 37s, 4 -> 23s, 8 -> 19s (stable, node + // 22 and 24), 11 -> 19s. throughput plateaus by 8 while wider pools only + // add memory pressure, so 8 is the sweet spot — identical to the old + // cpus-1 default on <=8-core machines. setNumberOfWorkers() or + // RAYON_NUM_THREADS override. + let numThreads = Math.max( + 1, + workers.numWorkers ?? Math.min(8, (os.availableParallelism?.() ?? 1) - 1) + ); process.env.RAYON_NUM_THREADS = String(numThreads); } diff --git a/src/bindings/js/web/ffi-proxy.js b/src/bindings/js/web/ffi-proxy.js new file mode 100644 index 0000000000..cda1ef766f --- /dev/null +++ b/src/bindings/js/web/ffi-proxy.js @@ -0,0 +1,205 @@ +// main-thread side of the worker-hosted kimchi FFI (web "Option B"). +// +// the napi-wasm module lives in a dedicated Web Worker where blocking is +// allowed, so rayon gets real threads. JSOO calls the FFI synchronously from +// the main thread, which cannot block in Atomics.wait — so every call is +// posted to the worker and the result is awaited by SPINNING on a +// SharedArrayBuffer flag (same trick the old wasm-bindgen backend used). +// +// FFI objects (napi class instances and externals) cannot cross the worker +// boundary; they stay in the worker's handle table and are represented here +// by stub objects carrying a handle id. stub classes (methods, getters, +// statics) are generated at runtime from a spec the worker introspects off +// the real module, so nothing here needs to track the kimchi API surface. + +export { createFfiProxy }; + +// SAB layout: [0] state (int32), [1] payload length, [2] needed size on grow +const STATE_PENDING = 0; +const STATE_DONE = 1; +const STATE_ERROR = 2; +const STATE_GROW = 3; +const HEADER_BYTES = 12; +const INITIAL_SAB_BYTES = 4 * 1024 * 1024 + HEADER_BYTES; + +async function createFfiProxy(workerUrl, threads) { + let worker = new Worker(workerUrl, { type: 'module', name: 'o1js-kimchi-ffi-host' }); + + // 'boot' arrives once the module (incl. the wasm fetch in its imports) has + // evaluated and its message handler exists — only then send 'init' + let spec = await new Promise((resolve, reject) => { + worker.onmessage = ({ data }) => { + if (data?.type === 'boot') worker.postMessage({ type: 'init', threads }); + else if (data?.type === 'ready') resolve(data.spec); + else if (data?.type === 'init-error') reject(new Error(data.message)); + }; + worker.onerror = (e) => reject(new Error(`kimchi ffi worker failed to load: ${e.message}`)); + }); + worker.onmessage = null; + worker.onerror = null; + + let sab = new SharedArrayBuffer(INITIAL_SAB_BYTES); + + // collect garbage-collected stubs and free their worker-side handles in + // batches, so long sessions don't leak the handle table + let pendingFrees = []; + let registry = new FinalizationRegistry((id) => { + pendingFrees.push(id); + if (pendingFrees.length >= 64) { + worker.postMessage({ type: 'free', ids: pendingFrees }); + pendingFrees = []; + } + }); + + let stubClasses = {}; + + function makeHandle(id, cls) { + let stub; + if (cls && stubClasses[cls]) { + stub = Object.create(stubClasses[cls].prototype); + stub.__h = id; + } else { + if (cls) console.warn(`o1js ffi-proxy: no stub class for '${cls}', methods unavailable`); + stub = { __h: id }; + } + registry.register(stub, id); + return stub; + } + + // args go through postMessage (structured clone handles typed arrays and + // plain data); only handles need replacing with markers + function encodeArg(value) { + if (value === null || typeof value !== 'object') return value; + if (typeof value.__h === 'number') return { $: 'h', id: value.__h }; + if (ArrayBuffer.isView(value) || value instanceof ArrayBuffer) return value; + if (Array.isArray(value)) return value.map(encodeArg); + if (Object.getPrototypeOf(value) === Object.prototype) { + let out = {}; + for (let k of Object.keys(value)) out[k] = encodeArg(value[k]); + return out; + } + return value; + } + + // results come back as JSON with markers (see ffi-worker-host.js) + function decodeResult(value) { + if (value === null || typeof value !== 'object') return value; + if (Array.isArray(value)) return value.map(decodeResult); + switch (value.$) { + case 'h': + return makeHandle(value.id, value.cls); + case 'undef': + return undefined; + case 'big': + return BigInt(value.v); + case 'ta': { + let bytes = base64ToBytes(value.b); + if (value.t === 'u8') return bytes; + if (value.t === 'i32') return new Int32Array(bytes.buffer, 0, bytes.byteLength >> 2); + if (value.t === 'u32') return new Uint32Array(bytes.buffer, 0, bytes.byteLength >> 2); + if (value.t === 'f64') return new Float64Array(bytes.buffer, 0, bytes.byteLength >> 3); + throw Error(`unknown typed array tag '${value.t}'`); + } + default: { + let out = {}; + for (let k of Object.keys(value)) out[k] = decodeResult(value[k]); + return out; + } + } + } + + let textDecoder = new TextDecoder(); + + function callSync(target, args) { + let i32 = new Int32Array(sab, 0, 3); + Atomics.store(i32, 0, STATE_PENDING); + worker.postMessage({ type: 'call', target, args: args.map(encodeArg), sab }); + + // the worker sets the state flag when the result is in the buffer. + // Atomics.wait is not allowed here (main thread), so spin. + let state; + while ((state = Atomics.load(i32, 0)) === STATE_PENDING) {} + + if (state === STATE_GROW) { + // result didn't fit; allocate a bigger buffer and ask for a resend + let needed = Atomics.load(i32, 2); + sab = new SharedArrayBuffer(needed + HEADER_BYTES); + let ni32 = new Int32Array(sab, 0, 3); + Atomics.store(ni32, 0, STATE_PENDING); + worker.postMessage({ type: 'resend', sab }); + while ((state = Atomics.load(ni32, 0)) === STATE_PENDING) {} + i32 = ni32; + } + + let length = Atomics.load(i32, 1); + let bytes = new Uint8Array(length); + bytes.set(new Uint8Array(sab, HEADER_BYTES, length)); + let payload = JSON.parse(textDecoder.decode(bytes)); + + if (state === STATE_ERROR) { + let error = new Error(payload.message); + if (payload.stack) error.stack = payload.stack; + throw error; + } + return decodeResult(payload); + } + + // build the ffi object: plain functions, constants, and stub classes + let ffi = {}; + + for (let name of spec.functions) { + ffi[name] = (...args) => callSync({ kind: 'fn', name }, args); + } + for (let [name, value] of Object.entries(spec.constants)) { + ffi[name] = value; + } + for (let [name, cls] of Object.entries(spec.classes)) { + let Stub = function (...args) { + // the decoded result is already a registered stub of this class; + // returning it overrides `this` (constructor-return-object semantics) + return callSync({ kind: 'construct', name }, args); + }; + for (let member of cls.methods) { + Stub.prototype[member] = function (...args) { + return callSync({ kind: 'method', name, member, id: this.__h }, args); + }; + } + for (let member of cls.getters) { + Object.defineProperty(Stub.prototype, member, { + get() { + return callSync({ kind: 'get', name, member, id: this.__h }, []); + }, + configurable: true, + }); + } + for (let member of cls.setters) { + let descriptor = Object.getOwnPropertyDescriptor(Stub.prototype, member) ?? { + configurable: true, + }; + descriptor.set = function (value) { + callSync({ kind: 'set', name, member, id: this.__h }, [value]); + }; + Object.defineProperty(Stub.prototype, member, descriptor); + } + for (let member of cls.statics.methods) { + Stub[member] = (...args) => callSync({ kind: 'static', name, member }, args); + } + for (let member of cls.statics.getters) { + Object.defineProperty(Stub, member, { + get: () => callSync({ kind: 'static-get', name, member }, []), + configurable: true, + }); + } + stubClasses[name] = Stub; + ffi[name] = Stub; + } + + return ffi; +} + +function base64ToBytes(b64) { + let bin = atob(b64); + let bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + return bytes; +} diff --git a/src/bindings/js/web/ffi-worker-host.js b/src/bindings/js/web/ffi-worker-host.js new file mode 100644 index 0000000000..9e19f6b4f6 --- /dev/null +++ b/src/bindings/js/web/ffi-worker-host.js @@ -0,0 +1,232 @@ +// worker side of the worker-hosted kimchi FFI (web "Option B"). +// +// this worker owns the real napi-wasm module. blocking is allowed here, so +// rayon runs with a real thread pool (nested workers, sized via the +// ?threads= query on this module's URL). FFI objects never leave this +// worker: they live in a handle table and cross to the main thread as +// handle ids. +// +// IMPORTANT: nested workers cannot finish loading while this worker is +// blocked inside a wasm call, so the rayon pool is spawned NOW — while the +// event loop is idle — and we poll until all threads are running before +// signaling readiness. +// +// protocol (see ffi-proxy.js for the main-thread side): +// in: {type:'call', target, args, sab} | {type:'resend', sab} | {type:'free', ids} +// out: results are serialized to JSON and written into the SharedArrayBuffer, +// then the state flag is set — the main thread is spinning on it and +// cannot receive messages. + +// the polyfill must be evaluated before @emnapi/runtime (imported by the +// loader below), which captures `Buffer` at module load +import './buffer-polyfill.js'; +import * as kimchiNapi from '../../compiled/web_bindings/kimchi_napi.wasi-browser.js'; + +const STATE_DONE = 1; +const STATE_ERROR = 2; +const STATE_GROW = 3; +const HEADER_BYTES = 12; + +let ffi = kimchiNapi.default ?? kimchiNapi; + +// handle table +let nextHandle = 1; +let handles = new Map(); + +function registerHandle(value) { + let id = nextHandle++; + handles.set(id, value); + return id; +} + +// classify a runtime value coming out of the FFI: +// - emnapi externals have a null prototype -> handle +// - class instances (non-plain prototype) -> handle, with class name +// - everything else is data (primitives, typed arrays, arrays, plain objects) +function encodeResult(value) { + if (value === undefined) return { $: 'undef' }; + if (value === null || typeof value === 'number' || typeof value === 'string' || typeof value === 'boolean') { + return value; + } + if (typeof value === 'bigint') return { $: 'big', v: value.toString() }; + if (value instanceof Uint8Array) return { $: 'ta', t: 'u8', b: bytesToBase64(value) }; + if (value instanceof Int32Array) { + return { $: 'ta', t: 'i32', b: bytesToBase64(new Uint8Array(value.buffer, value.byteOffset, value.byteLength)) }; + } + if (value instanceof Uint32Array) { + return { $: 'ta', t: 'u32', b: bytesToBase64(new Uint8Array(value.buffer, value.byteOffset, value.byteLength)) }; + } + if (value instanceof Float64Array) { + return { $: 'ta', t: 'f64', b: bytesToBase64(new Uint8Array(value.buffer, value.byteOffset, value.byteLength)) }; + } + if (Array.isArray(value)) return value.map(encodeResult); + let proto = Object.getPrototypeOf(value); + if (proto === Object.prototype) { + let out = {}; + for (let k of Object.keys(value)) out[k] = encodeResult(value[k]); + return out; + } + // external (proto === null) or napi class instance + let cls = proto === null ? null : (proto.constructor?.name ?? null); + return { $: 'h', id: registerHandle(value), cls }; +} + +function decodeArg(value) { + if (value === null || typeof value !== 'object') return value; + if (value.$ === 'h') { + let real = handles.get(value.id); + if (real === undefined) throw Error(`ffi-worker-host: unknown handle ${value.id}`); + return real; + } + if (ArrayBuffer.isView(value) || value instanceof ArrayBuffer) return value; + if (Array.isArray(value)) return value.map(decodeArg); + let out = {}; + for (let k of Object.keys(value)) out[k] = decodeArg(value[k]); + return out; +} + +function execute(target, args) { + switch (target.kind) { + case 'fn': + return ffi[target.name](...args); + case 'construct': + // the constructed instance goes straight into the handle table; the + // stub on the main thread adopts the handle + return new ffi[target.name](...args); + case 'method': { + let self = handles.get(target.id); + return self[target.member](...args); + } + case 'get': { + let self = handles.get(target.id); + return self[target.member]; + } + case 'set': { + let self = handles.get(target.id); + self[target.member] = args[0]; + return undefined; + } + case 'static': + return ffi[target.name][target.member](...args); + case 'static-get': + return ffi[target.name][target.member]; + default: + throw Error(`ffi-worker-host: unknown call kind '${target.kind}'`); + } +} + +let textEncoder = new TextEncoder(); +let lastPayload = null; // kept for the grow/resend protocol + +function respond(sab, state, payloadBytes) { + let i32 = new Int32Array(sab, 0, 3); + if (payloadBytes.byteLength > sab.byteLength - HEADER_BYTES) { + lastPayload = { state, bytes: payloadBytes }; + Atomics.store(i32, 2, payloadBytes.byteLength); + Atomics.store(i32, 0, STATE_GROW); + return; + } + new Uint8Array(sab, HEADER_BYTES, payloadBytes.byteLength).set(payloadBytes); + Atomics.store(i32, 1, payloadBytes.byteLength); + Atomics.store(i32, 0, state); +} + +onmessage = ({ data }) => { + if (data.type === 'init') { + spawnRayonPool(data.threads) + .then((threads) => postMessage({ type: 'ready', spec: buildSpec(), threads })) + .catch((e) => postMessage({ type: 'init-error', message: e?.message ?? String(e) })); + return; + } + if (data.type === 'free') { + for (let id of data.ids) handles.delete(id); + return; + } + if (data.type === 'resend') { + let { state, bytes } = lastPayload; + lastPayload = null; + respond(data.sab, state, bytes); + return; + } + if (data.type !== 'call') return; + let state, payload; + try { + let result = execute(data.target, data.args.map(decodeArg)); + state = STATE_DONE; + payload = encodeResult(result); + } catch (e) { + state = STATE_ERROR; + payload = { message: e?.message ?? String(e), stack: e?.stack }; + } + respond(data.sab, state, textEncoder.encode(JSON.stringify(payload))); +}; + +function bytesToBase64(bytes) { + let bin = ''; + const chunk = 0x8000; + for (let i = 0; i < bytes.length; i += chunk) { + bin += String.fromCharCode.apply(null, bytes.subarray(i, i + chunk)); + } + return btoa(bin); +} + +// nested workers only start while THIS worker's event loop is responsive: +// their script evaluation is scheduled by their owner, and their own spawn +// requests arrive here as messages. so camlRayonSpawnPool builds the pool on +// a helper wasi thread (never blocking us) and we poll until the threads are +// actually up. +async function spawnRayonPool(threads) { + if (!(threads > 0)) threads = Math.max(1, (globalThis.navigator?.hardwareConcurrency ?? 4) - 1); + ffi.camlRayonSpawnPool(threads); + let deadline = Date.now() + 30_000; + let started; + while ((started = ffi.camlRayonStartedThreads()) < threads) { + if (Date.now() > deadline) { + console.warn(`o1js ffi-worker-host: only ${started}/${threads} rayon threads started`); + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + return started; +} + +// introspect the module and send the api spec to the main thread +function buildSpec() { + let functions = []; + let classes = {}; + let constants = {}; + for (let [name, value] of Object.entries(ffi)) { + if (name === 'default') continue; + if (typeof value === 'function') { + if (value.prototype && Object.getOwnPropertyNames(value.prototype).length > 1) { + // a napi class: enumerate prototype + static members + let methods = []; + let getters = []; + let setters = []; + for (let [member, d] of Object.entries(Object.getOwnPropertyDescriptors(value.prototype))) { + if (member === 'constructor') continue; + if (typeof d.value === 'function') methods.push(member); + if (d.get) getters.push(member); + if (d.set) setters.push(member); + } + let statics = { methods: [], getters: [] }; + for (let [member, d] of Object.entries(Object.getOwnPropertyDescriptors(value))) { + if (['length', 'name', 'prototype'].includes(member)) continue; + if (typeof d.value === 'function') statics.methods.push(member); + if (d.get) statics.getters.push(member); + } + classes[name] = { methods, getters, setters, statics }; + } else { + functions.push(name); + } + } else if (typeof value !== 'object' || value === null) { + constants[name] = value; + } + } + return { functions, classes, constants }; +} + +// the module body only runs after the (imported) loader finished its +// top-level await; messages sent before that are lost — so announce +// readiness-to-init and let the main thread send 'init' only now +postMessage({ type: 'boot' }); diff --git a/src/bindings/js/web/web-backend.js b/src/bindings/js/web/web-backend.js index 890a9788d0..25f65e63d9 100644 --- a/src/bindings/js/web/web-backend.js +++ b/src/bindings/js/web/web-backend.js @@ -1,6 +1,6 @@ import o1jsWebSrc from 'string:../../../web_bindings/o1js_web.bc.js'; -import { WithThreadPool } from '../../../lib/proof-system/workers.js'; -import * as kimchiNapi from '../../../web_bindings/kimchi_napi.wasi-browser.js'; +import { WithThreadPool, workers } from '../../../lib/proof-system/workers.js'; +import { createFfiProxy } from './ffi-proxy.js'; export { initializeBindings, wasm, withThreadPool }; @@ -10,21 +10,39 @@ async function initializeBindings() { if (wasm !== undefined) return; // The wasm backend is the wasm32-wasip1-threads build of the kimchi-napi - // crate — the same crate that powers the native backend on Node. The - // generated .wasi-browser.js loader (via @napi-rs/wasm-runtime) instantiates - // the module on this thread and spawns Web Workers on demand for Rust - // std::thread. SharedArrayBuffer (COOP/COEP headers) is required, same as - // with the previous wasm-bindgen backend. + // crate — the same crate that powers the native backend on Node. // - // NOTE on threading: browser main threads cannot block, so rayon-parallel - // sections that make the calling thread wait must not run on the main - // thread with a multi-threaded pool. Until worker-hosted execution lands - // (see PLAN.md, web Option B), the pool is limited to inline execution: - // the bundled loader disables wasi thread spawning (see - // wasm-runtime-no-threads.js) and rayon's global pool is pinned to the - // current thread before the first rayon call. - wasm = kimchiNapi.default ?? kimchiNapi; - wasm.camlRayonInitSingleThreaded(); + // Browser main threads cannot block (Atomics.wait traps), and JSOO calls + // the FFI synchronously, so there are two modes: + // + // - worker-hosted (default, needs SharedArrayBuffer i.e. COOP/COEP + // headers): the module lives in a dedicated Web Worker where blocking is + // allowed, so rayon gets a real thread pool and proving is parallel. + // Calls are proxied over a handle-table RPC; the main thread awaits each + // result by spinning on a SharedArrayBuffer flag (see ffi-proxy.js). + // + // - main-thread fallback (no cross-origin isolation): instantiate the + // module here with thread spawning disabled and rayon pinned to the + // current thread — everything works, but proving is single-threaded. + if (typeof SharedArrayBuffer !== 'undefined' && globalThis.crossOriginIsolated) { + let threads = + workers.numWorkers ?? Math.max(1, (globalThis.navigator?.hardwareConcurrency ?? 4) - 1); + let url = new URL('./web_bindings/ffi-worker-host.js', import.meta.url); + wasm = await createFfiProxy(url, threads); + } else { + console.warn( + 'o1js: page is not cross-origin isolated (COOP/COEP headers missing) — ' + + 'falling back to single-threaded proving on the main thread.' + ); + let kimchiNapi = await import('../../../web_bindings/kimchi_napi.wasi-browser.js'); + wasm = kimchiNapi.default ?? kimchiNapi; + // pin rayon to this thread before the first rayon call — see + // wasm-runtime-no-threads.js for why the pool must not spawn + wasm.camlRayonInitSingleThreaded(); + } + + // Both backends expose the napi object model, so they share the TS + // conversion layer (src/bindings/crypto/native/). wasm.__kimchi_backend = 'native'; if (typeof globalThis !== 'undefined') { diff --git a/src/build/build-web.js b/src/build/build-web.js index 96709edd00..2b9bdb5425 100644 --- a/src/build/build-web.js +++ b/src/build/build-web.js @@ -63,6 +63,20 @@ async function buildWeb({ production }) { minify, sourcemap: true, }); + // the ffi host worker (web "Option B") keeps the default runtime with + // real thread support — it runs in a worker where blocking is allowed; + // rayon pool sizing/startup is handled inside ffi-worker-host.js + await esbuild.build({ + entryPoints: ['./src/bindings/js/web/ffi-worker-host.js'], + bundle: true, + format: 'esm', + outfile: './dist/web/web_bindings/ffi-worker-host.js', + target: 'esnext', + external: ['*.wasm'], + logLevel: 'error', + minify, + sourcemap: true, + }); await copy({ './src/bindings/compiled/web_bindings/kimchi_napi.wasm32-wasi.wasm': './dist/web/web_bindings/kimchi_napi.wasm32-wasi.wasm', diff --git a/src/examples/plain-html/server.js b/src/examples/plain-html/server.js index 157d81d50a..9230ff1029 100644 --- a/src/examples/plain-html/server.js +++ b/src/examples/plain-html/server.js @@ -10,7 +10,8 @@ const defaultHeaders = { }; const server = http.createServer(async (req, res) => { - let file = '.' + req.url; + // strip query strings (e.g. the ffi worker is loaded with ?threads=N) + let file = '.' + new URL(req.url, 'http://localhost').pathname; console.log(file); if (file === './') file = './index.html'; From 09a3b927740e785507bfa071fe2036ee6bfb6027 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 4 Jul 2026 10:30:19 +0700 Subject: [PATCH 04/14] Update mina --- src/mina | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mina b/src/mina index 8182bf2628..dfc4135b44 160000 --- a/src/mina +++ b/src/mina @@ -1 +1 @@ -Subproject commit 8182bf2628f395f58966c7537015925f2dbb6cc7 +Subproject commit dfc4135b44701c2886c5524bac15f587ac544a4b From 52a24e1a00db63bc6567d66f8915c2d0c521085b Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 4 Jul 2026 11:06:03 +0700 Subject: [PATCH 05/14] fix depds hash --- npmDepsHash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npmDepsHash b/npmDepsHash index c90e8ee219..70fb70fdef 100644 --- a/npmDepsHash +++ b/npmDepsHash @@ -1 +1 @@ -sha256-NYTcfClNuqJNCQgC4/YPyMt6Z3umTwo4w+pn4MHQ5Ts= +sha256-Py0fY+xe/vw7Xiuy3TfKOA1QAiLeQ9Gb4ExJ6Zj3FXw= \ No newline at end of file From 809ace7e189d10d83288babea13128d6df07de58 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 4 Jul 2026 15:30:49 +0700 Subject: [PATCH 06/14] Update mina --- src/mina | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mina b/src/mina index dfc4135b44..5a3ade2648 160000 --- a/src/mina +++ b/src/mina @@ -1 +1 @@ -Subproject commit dfc4135b44701c2886c5524bac15f587ac544a4b +Subproject commit 5a3ade2648089cd66e18da5b77614daeb15152a3 From c179f3c8511a43d5dae3a6a2e543867ae742fc98 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 4 Jul 2026 15:47:40 +0700 Subject: [PATCH 07/14] set cargo home/home variable --- scripts/build/wasm/build-kimchi-napi-wasm.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/build/wasm/build-kimchi-napi-wasm.sh b/scripts/build/wasm/build-kimchi-napi-wasm.sh index b3fcde6889..b9543e1ad6 100755 --- a/scripts/build/wasm/build-kimchi-napi-wasm.sh +++ b/scripts/build/wasm/build-kimchi-napi-wasm.sh @@ -24,6 +24,14 @@ PROOF_SYSTEMS_PATH=$MINA_PATH/src/lib/crypto/proof-systems NAPI=$(pwd)/node_modules/.bin/napi ARTIFACTS_PATH=$NATIVE_PATH/artifacts-wasm +# Nix sandboxes commonly set HOME to /homeless-shelter, which Cargo cannot +# write to when fetching git dependencies through napi-rs. +if [ -z "${HOME:-}" ] || [ ! -w "${HOME:-}" ]; then + export CARGO_HOME="${CARGO_HOME:-$(pwd)/.cargo}" + mkdir -p "$CARGO_HOME" + export HOME="$(pwd)" +fi + info "building kimchi-napi for wasm32-wasip1-threads..." ( From a79193e1c2ee17405e33540e79ae879e88f063ed Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 4 Jul 2026 16:50:05 +0700 Subject: [PATCH 08/14] improve compile time --- src/bindings/crypto/native/napi-srs.ts | 61 ++++++++++---------------- src/bindings/js/node/node-backend.js | 5 +++ src/bindings/js/web/ffi-proxy.js | 16 +++++-- src/mina | 2 +- 4 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/bindings/crypto/native/napi-srs.ts b/src/bindings/crypto/native/napi-srs.ts index e836865594..d5731e380b 100644 --- a/src/bindings/crypto/native/napi-srs.ts +++ b/src/bindings/crypto/native/napi-srs.ts @@ -9,7 +9,7 @@ import { import { srsCache as cache } from '../cache.js'; import { assert } from '../../../lib/util/errors.js'; import type { RustConversion } from '../bindings.js'; -import type { Napi, NapiAffine, NapiPolyComm, NapiPolyComms, NapiSrs } from './napi-wrappers.js'; +import type { Napi, NapiPolyComm, NapiPolyComms, NapiSrs } from './napi-wrappers.js'; import { OrInfinity, OrInfinityJson } from '../bindings/curve.js'; import { PolyComm } from '../bindings/kimchi-types.js'; @@ -39,6 +39,12 @@ function cacheHeaderLagrange(f: 'fp' | 'fq', domainSize: number): CacheHeader { srsVersion ); } +// v3: raw uncompressed bytes via caml_*_srs_{to,from}_raw_bytes. the v1 +// JSON-of-hex-points format spent ~400ms per field in per-point conversions on +// every cache read; v2 (serde/rmp) was worse — compressed points cost a sqrt +// each to load. raw affine coordinates make the read ~a memcpy. +const srsBlobVersion = 3; + function cacheHeaderSrs(f: 'fp' | 'fq', domainSize: number): CacheHeader { let id = `srs-${f}-${domainSize}`; return withVersion( @@ -46,9 +52,9 @@ function cacheHeaderSrs(f: 'fp' | 'fq', domainSize: number): CacheHeader { kind: 'srs', persistentId: id, uniqueId: id, - dataType: 'string', + dataType: 'bytes', }, - srsVersion + srsBlobVersion ); } @@ -70,32 +76,14 @@ function srsPerField(f: 'fp' | 'fq', napi: Napi, conversion: RustConversion) { } }; - let getSrs = (srs: NapiSrs): NapiAffine[] => { - try { - let fn = napi[`caml_${f}_srs_get`] as unknown as (value: NapiSrs) => NapiAffine[]; - return fn(srs); - } catch (error) { - console.error(`Error in SRS get for field ${f}`); - throw error; - } - }; let isEmptySrs = (srs: NapiSrs) => { try { - let points = getSrs(srs); - return points == null || points.length <= 1; + let fn = napi[`caml_${f}_srs_length`] as unknown as (value: NapiSrs) => number; + return fn(srs) === 0; } catch { return true; } }; - let setSrs = (points: NapiAffine[]) => { - try { - let fn = napi[`caml_${f}_srs_set`] as unknown as (value: NapiAffine[]) => NapiSrs; - return fn(points); - } catch (error) { - console.error(`Error in SRS set for field ${f} args ${points}`); - throw error; - } - }; let maybeLagrangeCommitment = ( srs: NapiSrs, @@ -177,26 +165,25 @@ function srsPerField(f: 'fp' | 'fq', napi: Napi, conversion: RustConversion) { // try to read SRS from cache / recompute and write if not found srs = readCache(cache, header, (bytes: Uint8Array) => { - // TODO: this takes a bit too long, about 300ms for 2^16 - // `pointsToRust` is the clear bottleneck - let jsonSrs: OrInfinityJson[] = JSON.parse(new TextDecoder().decode(bytes)); - let mlSrs = MlArray.mapTo(jsonSrs, OrInfinity.fromJSON); - let wasmSrs = conversion[f].pointsToRust(mlSrs); - let candidate = setSrs(wasmSrs); - if (isEmptySrs(candidate)) return undefined; - return candidate; + try { + let fn = napi[`caml_${f}_srs_from_raw_bytes`] as unknown as ( + b: Uint8Array + ) => NapiSrs; + let candidate = fn(bytes); + if (isEmptySrs(candidate)) return undefined; + return candidate; + } catch { + // unreadable/corrupt blob — treat as cache miss + return undefined; + } }); if (srs === undefined) { // not in cache srs = createSrs(size); if (cache.canWrite) { - let wasmSrs = getSrs(srs); - let mlSrs = conversion[f].pointsFromRust(wasmSrs); - let jsonSrs = MlArray.mapFrom(mlSrs, OrInfinity.toJSON); - let bytes = new TextEncoder().encode(JSON.stringify(jsonSrs)); - - writeCache(cache, header, bytes); + let fn = napi[`caml_${f}_srs_to_raw_bytes`] as unknown as (s: NapiSrs) => Uint8Array; + writeCache(cache, header, fn(srs)); } } } diff --git a/src/bindings/js/node/node-backend.js b/src/bindings/js/node/node-backend.js index 9e6eb87922..34cca049cd 100644 --- a/src/bindings/js/node/node-backend.js +++ b/src/bindings/js/node/node-backend.js @@ -32,6 +32,11 @@ if (typeof globalThis !== 'undefined') { } // The wasm runtime manages its own threads; nothing to set up or tear down. +// note: do NOT pre-warm the pool via camlRayonSpawnPool here — its helper +// thread's spawn requests are serviced by the main thread's event loop, which +// compile immediately blocks with synchronous wasm calls → deadlock. rayon's +// inline build on first use works because the MAIN thread spawns workers, +// which node can do without pumping its own event loop. const withThreadPool = WithThreadPool({ initThreadPool: async () => {}, exitThreadPool: async () => {}, diff --git a/src/bindings/js/web/ffi-proxy.js b/src/bindings/js/web/ffi-proxy.js index cda1ef766f..6a35306a8d 100644 --- a/src/bindings/js/web/ffi-proxy.js +++ b/src/bindings/js/web/ffi-proxy.js @@ -110,6 +110,17 @@ async function createFfiProxy(workerUrl, threads) { let textDecoder = new TextDecoder(); + // Atomics.pause (V8 13+/recent Firefox) hints the CPU during spin-waits — + // long ffi calls (proving) otherwise burn a full core competing with the + // rayon workers doing the actual work + let pause = typeof Atomics.pause === 'function' ? Atomics.pause : () => {}; + + function spinUntilNotPending(i32) { + let state; + while ((state = Atomics.load(i32, 0)) === STATE_PENDING) pause(); + return state; + } + function callSync(target, args) { let i32 = new Int32Array(sab, 0, 3); Atomics.store(i32, 0, STATE_PENDING); @@ -117,8 +128,7 @@ async function createFfiProxy(workerUrl, threads) { // the worker sets the state flag when the result is in the buffer. // Atomics.wait is not allowed here (main thread), so spin. - let state; - while ((state = Atomics.load(i32, 0)) === STATE_PENDING) {} + let state = spinUntilNotPending(i32); if (state === STATE_GROW) { // result didn't fit; allocate a bigger buffer and ask for a resend @@ -127,7 +137,7 @@ async function createFfiProxy(workerUrl, threads) { let ni32 = new Int32Array(sab, 0, 3); Atomics.store(ni32, 0, STATE_PENDING); worker.postMessage({ type: 'resend', sab }); - while ((state = Atomics.load(ni32, 0)) === STATE_PENDING) {} + state = spinUntilNotPending(ni32); i32 = ni32; } diff --git a/src/mina b/src/mina index 5a3ade2648..6e40b81dda 160000 --- a/src/mina +++ b/src/mina @@ -1 +1 @@ -Subproject commit 5a3ade2648089cd66e18da5b77614daeb15152a3 +Subproject commit 6e40b81dda789da490e7d1ad1d68e0a33b798d8c From 48b3395549d0ec9e0df7b1f9a45be969d883dd81 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 4 Jul 2026 17:02:56 +0700 Subject: [PATCH 09/14] fxix CI --- .github/actions/build-native/action.yml | 3 ++- .github/actions/build-wasm/action.yml | 5 ++++- .github/workflows/pull_requests.yml | 2 +- README-dev.md | 8 ++++---- src/bindings/README.md | 8 ++++---- src/bindings/crypto/native/napi-srs.ts | 10 +++------- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/actions/build-native/action.yml b/.github/actions/build-native/action.yml index 889630370d..fb72f81c0d 100644 --- a/.github/actions/build-native/action.yml +++ b/.github/actions/build-native/action.yml @@ -31,7 +31,8 @@ runs: if: steps.native-cache.outputs.cache-hit != 'true' uses: actions/setup-node@v4 with: - node-version: '20' + # keep in sync with the package engines floor (>=22.19.5) + node-version: '22' - name: Install npm dependencies if: steps.native-cache.outputs.cache-hit != 'true' diff --git a/.github/actions/build-wasm/action.yml b/.github/actions/build-wasm/action.yml index d8eb81d450..ae25ec9b2e 100644 --- a/.github/actions/build-wasm/action.yml +++ b/.github/actions/build-wasm/action.yml @@ -5,7 +5,10 @@ inputs: node_version: description: 'Node.js version to use' required: false - default: '20' + # must satisfy the package engines floor (>=22.19.5) — the napi-wasm + # backend (worker_threads + WASI + shared wasm memory) is not supported + # on older node + default: '22' proof_systems_commit: description: 'proof_systems commit to use' required: false diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml index cd33797c68..d8bec680d1 100644 --- a/.github/workflows/pull_requests.yml +++ b/.github/workflows/pull_requests.yml @@ -66,7 +66,7 @@ jobs: - name: Setup Node.JS uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 - name: Install Dependencies run: npm ci diff --git a/README-dev.md b/README-dev.md index d094383c35..012ba5f7f7 100644 --- a/README-dev.md +++ b/README-dev.md @@ -174,10 +174,10 @@ located in the Mina repo under `src/mina`. See the [Kimchi README](https://github.com/o1-labs/proof-systems/blob/master/README.md) for more information. -The wasm backend is the `wasm32-wasip1-threads` build of the `kimchi-napi` -crate — the same napi-rs crate that powers the native (`.node`) backend — built -via the napi-rs CLI (see `scripts/build/wasm/build-kimchi-napi-wasm.sh`). The -crate lives in the proof-systems submodule under +The wasm backend is the `wasm32-wasip1-threads` build of the `kimchi-napi` crate +— the same napi-rs crate that powers the native (`.node`) backend — built via +the napi-rs CLI (see `scripts/build/wasm/build-kimchi-napi-wasm.sh`). The crate +lives in the proof-systems submodule under `src/mina/src/lib/crypto/proof-systems/kimchi-napi`. For the wasm build, the output files are: diff --git a/src/bindings/README.md b/src/bindings/README.md index 16db74c1b3..acccce4262 100644 --- a/src/bindings/README.md +++ b/src/bindings/README.md @@ -8,10 +8,10 @@ OCaml. - `/compiled` - compiled JS and Wasm artifacts produced by `js_of_ocaml` (from OCaml source code) and by `napi-rs` (the `wasm32-wasip1-threads` build of the - `kimchi-napi` Rust crate — the same crate that powers the native backend). - We keep these artifacts in the source tree so that developing on o1js can be - done with standard JS tooling and doesn't require setting up the full - OCaml/Rust build pipeline. + `kimchi-napi` Rust crate — the same crate that powers the native backend). We + keep these artifacts in the source tree so that developing on o1js can be done + with standard JS tooling and doesn't require setting up the full OCaml/Rust + build pipeline. - `/crypto` - pure TS implementations of a subset of the crypto primitives we use, including finite field and elliptic curve arithmetic. This is used by mina-signer (a pure TS package) to hash and sign transactions. Also includes diff --git a/src/bindings/crypto/native/napi-srs.ts b/src/bindings/crypto/native/napi-srs.ts index d5731e380b..c35d9cef7d 100644 --- a/src/bindings/crypto/native/napi-srs.ts +++ b/src/bindings/crypto/native/napi-srs.ts @@ -6,12 +6,12 @@ import { type Cache, type CacheHeader, } from '../../../lib/proof-system/cache.js'; -import { srsCache as cache } from '../cache.js'; import { assert } from '../../../lib/util/errors.js'; import type { RustConversion } from '../bindings.js'; -import type { Napi, NapiPolyComm, NapiPolyComms, NapiSrs } from './napi-wrappers.js'; import { OrInfinity, OrInfinityJson } from '../bindings/curve.js'; import { PolyComm } from '../bindings/kimchi-types.js'; +import { srsCache as cache } from '../cache.js'; +import type { Napi, NapiPolyComm, NapiPolyComms, NapiSrs } from './napi-wrappers.js'; export { srs }; @@ -102,11 +102,7 @@ function srsPerField(f: 'fp' | 'fq', napi: Napi, conversion: RustConversion) { throw error; } }; - let lagrangeCommitment = ( - srs: NapiSrs, - domain_size: number, - i: number - ): NapiPolyComm => { + let lagrangeCommitment = (srs: NapiSrs, domain_size: number, i: number): NapiPolyComm => { try { let fn = napi[`caml_${f}_srs_lagrange_commitment`] as unknown as ( srsValue: NapiSrs, From 77a3ef893a68ec91a9000a4cf82e5c4f90fb7c14 Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 4 Jul 2026 17:17:07 +0700 Subject: [PATCH 10/14] add watchdog timeout --- src/bindings/js/web/buffer-polyfill.js | 7 +++++++ src/bindings/js/web/ffi-proxy.js | 24 ++++++++++++++++++++---- src/bindings/js/web/ffi-worker-host.js | 7 +++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/bindings/js/web/buffer-polyfill.js b/src/bindings/js/web/buffer-polyfill.js index 7464099fed..5eaabc288a 100644 --- a/src/bindings/js/web/buffer-polyfill.js +++ b/src/bindings/js/web/buffer-polyfill.js @@ -5,6 +5,13 @@ // // @emnapi/runtime captures `Buffer` at module evaluation time, so this module // must be imported (for its side effect) BEFORE @napi-rs/wasm-runtime. +// TEMP CI diagnosis (remove): first evaluated statement in the ffi worker +// host's import graph — proves the worker script started evaluating. +try { + if (typeof WorkerGlobalScope !== 'undefined') + void fetch('/__o1js_boot_stage/worker-evaluating'); +} catch {} + class BufferPolyfill extends Uint8Array { static alloc(size) { return new BufferPolyfill(size); diff --git a/src/bindings/js/web/ffi-proxy.js b/src/bindings/js/web/ffi-proxy.js index 6a35306a8d..3c013679d1 100644 --- a/src/bindings/js/web/ffi-proxy.js +++ b/src/bindings/js/web/ffi-proxy.js @@ -26,14 +26,30 @@ async function createFfiProxy(workerUrl, threads) { let worker = new Worker(workerUrl, { type: 'module', name: 'o1js-kimchi-ffi-host' }); // 'boot' arrives once the module (incl. the wasm fetch in its imports) has - // evaluated and its message handler exists — only then send 'init' + // evaluated and its message handler exists — only then send 'init'. + // the watchdog turns a stalled worker (module evaluation wedged — no boot, + // no error event) into a loud failure instead of hanging o1js + // initialization forever with no message anywhere. let spec = await new Promise((resolve, reject) => { + let watchdog = setTimeout( + () => + reject( + new Error( + 'kimchi ffi worker: not ready within 120s — worker module evaluation or rayon pool startup stalled' + ) + ), + 120_000 + ); + let done = (fn) => (value) => { + clearTimeout(watchdog); + fn(value); + }; worker.onmessage = ({ data }) => { if (data?.type === 'boot') worker.postMessage({ type: 'init', threads }); - else if (data?.type === 'ready') resolve(data.spec); - else if (data?.type === 'init-error') reject(new Error(data.message)); + else if (data?.type === 'ready') done(resolve)(data.spec); + else if (data?.type === 'init-error') done(reject)(new Error(data.message)); }; - worker.onerror = (e) => reject(new Error(`kimchi ffi worker failed to load: ${e.message}`)); + worker.onerror = (e) => done(reject)(new Error(`kimchi ffi worker failed to load: ${e.message}`)); }); worker.onmessage = null; worker.onerror = null; diff --git a/src/bindings/js/web/ffi-worker-host.js b/src/bindings/js/web/ffi-worker-host.js index 9e19f6b4f6..27fab22ccc 100644 --- a/src/bindings/js/web/ffi-worker-host.js +++ b/src/bindings/js/web/ffi-worker-host.js @@ -22,6 +22,13 @@ import './buffer-polyfill.js'; import * as kimchiNapi from '../../compiled/web_bindings/kimchi_napi.wasi-browser.js'; +// TEMP CI diagnosis (remove): the loader import above just finished — its +// top-level await fetched + instantiated the wasm. observable in playwright +// network traces even when worker messaging is wedged. +try { + void fetch('/__o1js_boot_stage/loader-evaluated'); +} catch {} + const STATE_DONE = 1; const STATE_ERROR = 2; const STATE_GROW = 3; From e1b9d72acb0f407fcd864d56fb8dfc6c84e65a0d Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 4 Jul 2026 22:51:39 +0700 Subject: [PATCH 11/14] add diagonistics --- .../actions/release-pkg-pr-version/action.yml | 7 ++++--- .github/workflows/checks.yml | 20 ++++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/actions/release-pkg-pr-version/action.yml b/.github/actions/release-pkg-pr-version/action.yml index e56d7dea02..f7ef1cbf93 100644 --- a/.github/actions/release-pkg-pr-version/action.yml +++ b/.github/actions/release-pkg-pr-version/action.yml @@ -9,11 +9,12 @@ runs: uses: actions/cache@v4 with: path: . - key: repo-${{ github.sha }}-node-20 - - name: Setup Node.JS 20 + # must match the key build-wasm/action.yml saves under + key: repo-${{ github.sha }}-node-22 + - name: Setup Node.JS 22 uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 - name: build uses: ./.github/actions/build-wasm - name: Build o1js and mina-signer diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ac59f847f1..5021355101 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -329,7 +329,11 @@ jobs: for ((i=start_index; i-node-, currently 22) + key: repo-${{ github.sha }}-node-22 - - name: Setup Node.JS 20 + - name: Setup Node.JS 22 uses: actions/setup-node@v4 with: node-version: 22 @@ -372,7 +378,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.npm - key: ${{ runner.OS }}-node-20-${{ hashFiles('**/package-lock.json') }} + key: ${{ runner.OS }}-node-22-${{ hashFiles('**/package-lock.json') }} - name: Cache Playwright browsers uses: actions/cache@v4 @@ -515,7 +521,11 @@ jobs: for ((i=start_index; i Date: Sun, 5 Jul 2026 01:57:35 +0700 Subject: [PATCH 12/14] attempt CI fix --- .github/workflows/checks.yml | 2 ++ scripts/build/wasm/build-node.sh | 21 +++++++++++++++++++++ scripts/build/wasm/build-web.sh | 23 +++++++---------------- src/bindings/js/node/node-backend.js | 10 ++++++++++ 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 5021355101..698028ed21 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -300,6 +300,7 @@ jobs: TOTAL_TESTS: ${{ steps.count_tests.outputs.test_count }} CHUNK: ${{ matrix.chunk }} CHUNKS: 8 + O1JS_CI_DIAG: 1 shell: bash run: | echo "Total tests: $TOTAL_TESTS" @@ -498,6 +499,7 @@ jobs: CHUNK: ${{ matrix.chunk }} CHUNKS: 4 O1JS_BACKEND: native + O1JS_CI_DIAG: 1 shell: bash run: | echo "Total tests: $TOTAL_TESTS" diff --git a/scripts/build/wasm/build-node.sh b/scripts/build/wasm/build-node.sh index 94c362fcca..543c6d2111 100755 --- a/scripts/build/wasm/build-node.sh +++ b/scripts/build/wasm/build-node.sh @@ -96,6 +96,27 @@ process.on('uncaughtException', (e) => { } catch (_) {} process.kill(process.pid, 'SIGABRT'); }); + +// TEMP CI diagnosis (remove): synchronous stderr breadcrumb, gated on env — +// shows in the CI job log even when everything else is wedged. +if (process.env.O1JS_CI_DIAG) { + try { fs.writeSync(2, '[o1js-diag] wasi worker module evaluated\n'); } catch (_) {} +} WORKER_HARDENING +# TEMP CI diagnosis (remove): breadcrumbs around loader evaluation and thread +# spawning, gated on O1JS_CI_DIAG. a hung CI test's log then shows the last +# startup stage reached before the per-test timeout kills it. +node -e ' + let fs = require("fs"); + let path = process.argv[1]; + let src = fs.readFileSync(path, "utf8"); + let diag = (msg) => `(process.env.O1JS_CI_DIAG && (() => { try { require("fs").writeSync(2, "[o1js-diag] ${msg}\\n"); } catch (_) {} })());\n`; + let spawnAnchor = "const worker = new Worker("; + if (!src.includes(spawnAnchor)) throw Error("diag spawn anchor not found in " + path); + src = src.replace(spawnAnchor, diag("spawning wasi worker thread") + spawnAnchor); + src += "\n" + diag("node loader evaluated"); + fs.writeFileSync(path, src); +' $BINDINGS_PATH/kimchi_napi.wasi.cjs + success "WASM node build success!" diff --git a/scripts/build/wasm/build-web.sh b/scripts/build/wasm/build-web.sh index 9835600622..410e6a197f 100755 --- a/scripts/build/wasm/build-web.sh +++ b/scripts/build/wasm/build-web.sh @@ -8,7 +8,6 @@ set -Eeuo pipefail # - Copies the wasm binary, the generated browser loader # (`kimchi_napi.wasi-browser.js`, backed by @napi-rs/wasm-runtime) and its # worker file into `src/bindings/compiled/web_bindings/`. -# - Optimizes the WebAssembly binary with `wasm-opt` when available. # # Usage: # npm run build:wasm:web @@ -32,20 +31,12 @@ cp $ARTIFACTS_PATH/kimchi_napi.wasi-browser.js $BINDINGS_PATH/ cp $ARTIFACTS_PATH/wasi-worker-browser.mjs $BINDINGS_PATH/ cp $ARTIFACTS_PATH/index.d.ts $BINDINGS_PATH/kimchi_napi.wasi-browser.d.ts -if command -v wasm-opt >/dev/null 2>&1; then - info "optimizing wasm with wasm-opt..." - run_cmd wasm-opt \ - --detect-features \ - --enable-mutable-globals \ - --enable-threads \ - --enable-bulk-memory \ - -O4 \ - -o $BINDINGS_PATH/kimchi_napi.wasm32-wasi.wasm.opt \ - $BINDINGS_PATH/kimchi_napi.wasm32-wasi.wasm - run_cmd mv $BINDINGS_PATH/kimchi_napi.wasm32-wasi.wasm.opt $BINDINGS_PATH/kimchi_napi.wasm32-wasi.wasm - ok "wasm optimized" -else - warn "wasm-opt not found — skipping wasm optimization" -fi +# NOTE: do NOT run wasm-opt on this binary. wasm-opt -O4 miscompiles the +# wasm32-wasip1-threads build: the optimized binary wedges forever inside +# WebAssembly instantiation in the browser worker host (verified by A/B in +# an otherwise identical environment — raw binary passes, optimized binary +# hangs; this is what made Build-And-Test-Web time out on CI, where wasm-opt +# happened to be installed, while local builds without wasm-opt worked). +# It was also measured to make no runtime performance difference. success "WASM web build success!" diff --git a/src/bindings/js/node/node-backend.js b/src/bindings/js/node/node-backend.js index 34cca049cd..29ad44646a 100644 --- a/src/bindings/js/node/node-backend.js +++ b/src/bindings/js/node/node-backend.js @@ -19,7 +19,17 @@ const require = createRequire(filename); // must be configured before any parallel binding call. setRayonThreadCount(); +// TEMP CI diagnosis (remove): startup breadcrumbs, gated on O1JS_CI_DIAG +diag('requiring kimchi napi wasm loader'); const wasm = requireKimchiNapiWasm(); +diag('kimchi napi wasm loader required'); + +function diag(msg) { + if (typeof process === 'undefined' || !process.env.O1JS_CI_DIAG) return; + try { + require('node:fs').writeSync(2, `[o1js-diag] ${msg}\n`); + } catch (_) {} +} // Both backends expose the napi object model, so they share the TS conversion // layer (src/bindings/crypto/native/). From 0543949e34d1619f34ef6c198e33fcae81484be5 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 5 Jul 2026 12:54:06 +0700 Subject: [PATCH 13/14] fix build --- .github/actions/build-wasm/action.yml | 6 +++++- .github/workflows/checks.yml | 22 ++++++++++++++++++++++ scripts/build/wasm/build-node.sh | 8 +++++++- scripts/build/wasm/build-web.sh | 6 +++--- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/actions/build-wasm/action.yml b/.github/actions/build-wasm/action.yml index ae25ec9b2e..30abb5f1c8 100644 --- a/.github/actions/build-wasm/action.yml +++ b/.github/actions/build-wasm/action.yml @@ -65,7 +65,11 @@ runs: ~/.npm node_modules dist - key: ${{ runner.OS }}-node-${{ inputs.node_version }}-v2-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }} + # v3: the key must also hash the build scripts — this cache includes + # `dist`, and on a hit the build is skipped entirely, so a build-script + # fix (e.g. removing the wasm-opt miscompilation) never took effect + # while ts/js were unchanged + key: ${{ runner.OS }}-node-${{ inputs.node_version }}-v3-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js', 'scripts/**/*.sh', '.github/actions/**') }} - name: Build examples if: ${{ steps.cache.outputs.cache-hit != 'true' }} shell: bash diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 698028ed21..2f482ba1a8 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -301,8 +301,19 @@ jobs: CHUNK: ${{ matrix.chunk }} CHUNKS: 8 O1JS_CI_DIAG: 1 + # SPECULATIVE FIX (evaluate, keep or revert): V8's wasm trap-handler + # reserves a ~10GB virtual-address guard cage; the loader needs >8GB VA + # to instantiate (measured locally). If the CI runner constrains VA, + # instantiation wedges. Inline bounds checks drop the requirement to + # ~4GB (the shared-memory reservation) at a small runtime cost. + NODE_OPTIONS: '--disable-wasm-trap-handler' shell: bash run: | + # TEMP CI diagnosis (remove): runner resource limits — the wasm loader + # needs >8GB of virtual address space (trap-handler cage + 4GiB + # shared-memory reservation); a constrained runner wedges at load + echo "=== runner limits:"; ulimit -a; free -h || true + echo "Total tests: $TOTAL_TESTS" echo "Current chunk: $CHUNK" echo "Total chunks: $CHUNKS" @@ -500,8 +511,19 @@ jobs: CHUNKS: 4 O1JS_BACKEND: native O1JS_CI_DIAG: 1 + # SPECULATIVE FIX (evaluate, keep or revert): V8's wasm trap-handler + # reserves a ~10GB virtual-address guard cage; the loader needs >8GB VA + # to instantiate (measured locally). If the CI runner constrains VA, + # instantiation wedges. Inline bounds checks drop the requirement to + # ~4GB (the shared-memory reservation) at a small runtime cost. + NODE_OPTIONS: '--disable-wasm-trap-handler' shell: bash run: | + # TEMP CI diagnosis (remove): runner resource limits — the wasm loader + # needs >8GB of virtual address space (trap-handler cage + 4GiB + # shared-memory reservation); a constrained runner wedges at load + echo "=== runner limits:"; ulimit -a; free -h || true + echo "Total tests: $TOTAL_TESTS" echo "Current chunk: $CHUNK" echo "Total chunks: $CHUNKS" diff --git a/scripts/build/wasm/build-node.sh b/scripts/build/wasm/build-node.sh index 543c6d2111..07fb631a95 100755 --- a/scripts/build/wasm/build-node.sh +++ b/scripts/build/wasm/build-node.sh @@ -111,10 +111,16 @@ node -e ' let fs = require("fs"); let path = process.argv[1]; let src = fs.readFileSync(path, "utf8"); - let diag = (msg) => `(process.env.O1JS_CI_DIAG && (() => { try { require("fs").writeSync(2, "[o1js-diag] ${msg}\\n"); } catch (_) {} })());\n`; + let diag = (msg) => `;(process.env.O1JS_CI_DIAG && (() => { try { require("fs").writeSync(2, "[o1js-diag] ${msg}\\n"); } catch (_) {} })());\n`; let spawnAnchor = "const worker = new Worker("; if (!src.includes(spawnAnchor)) throw Error("diag spawn anchor not found in " + path); src = src.replace(spawnAnchor, diag("spawning wasi worker thread") + spawnAnchor); + let memAnchor = "const __sharedMemory = new WebAssembly.Memory({"; + if (!src.includes(memAnchor)) throw Error("diag memory anchor not found in " + path); + src = src.replace(memAnchor, diag("creating shared wasm memory (4GiB max)") + memAnchor); + let instAnchor = "const { instance: __napiInstance"; + if (!src.includes(instAnchor)) throw Error("diag instantiate anchor not found in " + path); + src = src.replace(instAnchor, diag("memory ok, instantiating napi module") + instAnchor); src += "\n" + diag("node loader evaluated"); fs.writeFileSync(path, src); ' $BINDINGS_PATH/kimchi_napi.wasi.cjs diff --git a/scripts/build/wasm/build-web.sh b/scripts/build/wasm/build-web.sh index 410e6a197f..22bd94a88b 100755 --- a/scripts/build/wasm/build-web.sh +++ b/scripts/build/wasm/build-web.sh @@ -35,8 +35,8 @@ cp $ARTIFACTS_PATH/index.d.ts $BINDINGS_PATH/kimchi_napi.wasi-browser.d.ts # wasm32-wasip1-threads build: the optimized binary wedges forever inside # WebAssembly instantiation in the browser worker host (verified by A/B in # an otherwise identical environment — raw binary passes, optimized binary -# hangs; this is what made Build-And-Test-Web time out on CI, where wasm-opt -# happened to be installed, while local builds without wasm-opt worked). -# It was also measured to make no runtime performance difference. +# hangs). It also ran only where wasm-opt happened to be installed (CI but +# not local machines), so CI silently shipped a binary nobody else executed. +# Measured to make no runtime performance difference, so nothing is lost. success "WASM web build success!" From 56120345c1b421600967f84d1f254654f0407f6b Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 6 Jul 2026 13:03:13 +0700 Subject: [PATCH 14/14] debug --- .github/workflows/checks.yml | 46 +++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2f482ba1a8..1a81f9d1f9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -339,13 +339,55 @@ jobs: set -o pipefail + # TEMP CI diagnosis (remove): the wasm loader wedges inside module + # instantiation only on the CI runner — not reproducible locally on + # any arch / node version / cache state. Print exact runtime versions + # so we can see what the runner actually installed vs the lockfile. + echo "=== o1js-diag runtime versions:" + node -e 'console.log("node", process.version, process.arch, "cpus", require("os").cpus().length); + for (const p of ["@emnapi/core","@emnapi/runtime","@napi-rs/wasm-runtime","@tybys/wasm-util"]) { + try { console.log(p, require(p + "/package.json").version); } catch (e) { console.log(p, "MISSING"); } + }' || true + + # TEMP CI diagnosis (remove): if a test is still alive after 120s + # (normal tests finish well under that), dump every thread's kernel + # wait-state from /proc — no gdb/symbols needed. futex_wait = lock + # deadlock, mmap/mprotect = memory reservation, epoll/poll = I/O. + # This shows exactly what the wasm instantiation wedges on. + dump_hang() { + local label="$1" + local np="" + # the pattern also matches the `timeout` wrapper (label is in its + # argv), so pick the process whose comm is actually `node` + for p in $(pgrep -f "enable-source-maps ${label}"); do + if [ "$(cat /proc/"$p"/comm 2>/dev/null)" = "node" ]; then np="$p"; break; fi + done + [ -z "$np" ] && return 0 + { + echo "===== HANG DIAG: ${label} (pid ${np}) =====" + for t in /proc/"$np"/task/*; do + local tid; tid=$(basename "$t") + printf "tid=%s comm=%s state=%s wchan=%s syscall=%s\n" \ + "$tid" "$(cat "$t"/comm 2>/dev/null)" \ + "$(awk '{print $3}' "$t"/stat 2>/dev/null)" \ + "$(cat "$t"/wchan 2>/dev/null)" \ + "$(cut -d' ' -f1 "$t"/syscall 2>/dev/null)" + done + echo "--- kernel stacks (sudo):" + sudo bash -c 'for t in /proc/'"$np"'/task/*; do echo "== $t"; cat "$t"/stack 2>/dev/null; done' 2>/dev/null || echo "(no sudo/stack)" + } >> hang-diag.txt 2>&1 + } + for ((i=start_index; i/dev/null || true done continue-on-error: false - name: Upload test results @@ -353,7 +395,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: test-results-${{ matrix.chunk }} - path: profiling.md + path: | + profiling.md + hang-diag.txt - name: Add to job summary if: always() shell: bash