Blender→three authoring pipeline

.github/workflows/ci.yml

@@ -24,6 +24,10 @@ jobs:
       - run: pnpm build
       - run: pnpm exec eslint src/
       - run: pnpm exec playwright install --with-deps chromium webkit
+      # Bake round-trips need only node + the Chromium installed above; the
+      # Blender-native suite (test:blender) can't run here and stays local.
+      - run: pnpm test:bake
+      - run: pnpm test:bridge
       - run: pnpm test:e2e
         env:
           # Audio/model assets are gitignored and absent in CI; fetch them from the

.gitignore

@@ -46,3 +46,9 @@ yarn-error.log*
 
 # perf run artifacts
 tools/perf/reports/
+
+# blender contract-test fixture (generated headlessly)
+/public/blender-contract.glb
+
+# blender bake-audio generated artifacts
+/tools/blender/bake/__generated__

CLAUDE.md

@@ -11,3 +11,15 @@
   down the audio engine and the scene, so a credit link or a Discord invite
   would silently end whatever is playing. In-app routing stays on wouter's
   `<Link>`, which must not get these attributes.
+
+## three.js versions
+
+This project runs two copies of three.js side by side:
+
+- `three` — the latest release, used by new content under `src/viz/runtime/`.
+- `three-legacy` — `three@0.108.0`, used by the original scenes (Moonrise,
+  Mornings, Swamp). It is pinned **deliberately**; those scenes depend on its
+  behavior. Do not "upgrade" it or fold it back into `three`.
+
+New code imports `three` and `three/addons/<path>.js`. Legacy code imports
+`three-legacy` and `three-legacy/examples/jsm/<path>`.

blender-bake-audio.html

@@ -0,0 +1,10 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>Blender Bake-Audio Harness</title>
+  </head>
+  <body>
+    <script type="module" src="/src/viz/runtime/dev/bakeAudioHarness.ts"></script>
+  </body>
+</html>

blender-bake.html

@@ -0,0 +1,10 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>Blender Bake Harness</title>
+  </head>
+  <body>
+    <script type="module" src="/src/viz/runtime/bake/bakeHarness.ts"></script>
+  </body>
+</html>

blender-contract.html

@@ -0,0 +1,10 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>Blender Contract Harness</title>
+  </head>
+  <body>
+    <script type="module" src="/src/viz/runtime/dev/contractHarness.ts"></script>
+  </body>
+</html>

blender-hello.html

@@ -0,0 +1,18 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>Blender Parity Harness</title>
+    <style>
+      html,
+      body {
+        margin: 0;
+        background: #000;
+      }
+    </style>
+  </head>
+  <body>
+    <canvas id="blender-hello-canvas" width="512" height="512"></canvas>
+    <script type="module" src="/src/viz/runtime/dev/blenderHello.ts"></script>
+  </body>
+</html>

blender-runtime.html

@@ -0,0 +1,18 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>Blender Runtime Harness</title>
+    <style>
+      html,
+      body {
+        margin: 0;
+        background: #000;
+      }
+    </style>
+  </head>
+  <body>
+    <canvas id="blender-runtime-canvas" width="256" height="256"></canvas>
+    <script type="module" src="/src/viz/runtime/dev/runtimeHarness.ts"></script>
+  </body>
+</html>

e2e/blender-bake.spec.js

@@ -0,0 +1,48 @@
+import { test, expect } from "@playwright/test";
+
+// The harness bakes 1s of broadband noise (first 0.5s) then silence at 30fps, 8
+// buckets, twice: raw (smoothing 0) and smoothed (smoothing 0.8, a real-band value).
+// Frame 5 (~0.18s) is inside the noise; frame 20 (~0.68s) is fully into silence
+// (past the FFT window), frame 25 (~0.85s) deeper still.
+test("headless bake produces a faithful per-frame band signal from audio", async ({ page }) => {
+  const pageErrors = [];
+  page.on("pageerror", (err) => pageErrors.push(err));
+
+  await page.goto("/blender-bake.html", { waitUntil: "load" });
+  await page.waitForFunction(() => window.__blenderBake?.ready === true, null, { timeout: 30000 });
+
+  const { raw, smoothed } = await page.evaluate(() => window.__blenderBake);
+
+  // --- raw pass (smoothing 0): instantaneous, silence is ~0 ---
+  expect(raw.fps).toBe(30);
+  expect(raw.frames.length).toBe(30);
+  for (const f of raw.frames) {
+    expect(Number.isFinite(f.volume)).toBe(true);
+    expect(f.volume).toBeGreaterThanOrEqual(0);
+    expect(f.buckets.length).toBe(8);
+    for (const b of f.buckets) {
+      expect(Number.isFinite(b)).toBe(true);
+      expect(b).toBeGreaterThanOrEqual(0);
+    }
+  }
+  const active = raw.frames[5];
+  const silence = raw.frames[25];
+  expect(active.volume).toBeGreaterThan(0.05);
+  expect(silence.volume).toBeLessThan(0.02);
+  expect(active.volume).toBeGreaterThan(silence.volume);
+  expect(Math.max(...active.buckets)).toBeGreaterThan(0.05);
+
+  // --- smoothed pass (smoothing 0.8): the technique must work for real bands too ---
+  expect(smoothed.frames.length).toBe(30);
+  for (const f of smoothed.frames) {
+    expect(Number.isFinite(f.volume)).toBe(true);
+    expect(f.volume).toBeGreaterThanOrEqual(0);
+    for (const b of f.buckets) expect(Number.isFinite(b)).toBe(true);
+  }
+  expect(smoothed.frames[5].volume).toBeGreaterThan(0.05); // tracks audio
+  // A frame fully into silence: the smoothed bake retains a decaying residual the
+  // raw bake does not -> proves the offline render captures the analyser's smoothing.
+  expect(smoothed.frames[20].volume).toBeGreaterThan(raw.frames[20].volume + 0.02);
+
+  expect(pageErrors, pageErrors.map((e) => e.message).join("\n")).toHaveLength(0);
+});

e2e/blender-contract.spec.js

@@ -0,0 +1,39 @@
+import { test, expect } from "@playwright/test";
+import { execFileSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+
+const BLENDER =
+  process.env.BLENDER_BIN || "/Applications/Blender.app/Contents/MacOS/Blender";
+const FIXTURE = resolve("public/blender-contract.glb");
+const SCRIPT = resolve("tools/blender/tests/export_contract_fixture.py");
+const HAS_BLENDER = existsSync(BLENDER);
+
+test.describe("blender glTF export contract", () => {
+  test.skip(!HAS_BLENDER, `Blender not found at ${BLENDER}`);
+
+  test.beforeAll(() => {
+    if (!HAS_BLENDER) return;
+    execFileSync(BLENDER, ["--background", "--python", SCRIPT], {
+      env: { ...process.env, SOUNDSCAPE_FIXTURE_OUT: FIXTURE },
+      stdio: "inherit",
+      timeout: 60_000,
+      killSignal: "SIGKILL",
+    });
+    if (!existsSync(FIXTURE)) throw new Error("fixture not generated: " + FIXTURE);
+  });
+
+  test("Blender export carries bindings onto the mesh as an object", async ({ page }) => {
+    await page.goto("/blender-contract.html", { waitUntil: "load" });
+    await page.waitForFunction(() => window.__blenderContract?.ready === true);
+    const r = await page.evaluate(() => window.__blenderContract);
+
+    expect(r.error, r.error).toBeUndefined();
+    // (a) nested object, not a JSON string
+    expect(r.userDataType).toBe("[object Object]");
+    // (b) on the mesh, not a parent Group
+    expect(r.carrierIsMesh).toBe(true);
+    // schema-valid for the Phase-2 runtime
+    expect(r.valid, r.errors.join("; ")).toBe(true);
+  });
+});

e2e/blender-hello.spec.js

@@ -0,0 +1,32 @@
+import { test, expect } from "@playwright/test";
+
+// sRGB #3366cc — the HelloScene cube color. With ColorManagement on and
+// NoToneMapping + sRGB output, an unlit sRGB color round-trips to the same
+// framebuffer bytes, so the center pixel must read back as ~#3366cc.
+const EXPECTED = [0x33, 0x66, 0xcc];
+const TOLERANCE = 4; // 8-bit sRGB<->linear round-trip quantization
+
+test("blender-hello harness renders the parity cube with correct sRGB color", async ({
+  page,
+}) => {
+  const pageErrors = [];
+  page.on("pageerror", (err) => pageErrors.push(err));
+
+  await page.goto("/blender-hello.html", { waitUntil: "load" });
+  await expect(page.locator("#blender-hello-canvas")).toBeAttached();
+  await page.waitForFunction(() => !!window.__blenderHello);
+
+  const pixel = await page.evaluate(() => window.__blenderHello.centerPixel);
+
+  for (let i = 0; i < 3; i++) {
+    expect(
+      Math.abs(pixel[i] - EXPECTED[i]),
+      `channel ${i}: got ${pixel[i]}, expected ~${EXPECTED[i]}`
+    ).toBeLessThanOrEqual(TOLERANCE);
+  }
+
+  expect(
+    pageErrors,
+    pageErrors.map((e) => e.message).join("\n")
+  ).toHaveLength(0);
+});

e2e/blender-runtime.spec.js

@@ -0,0 +1,25 @@
+import { test, expect } from "@playwright/test";
+
+// The fixture binding maps emissiveIntensity <- bass volume over [0,1] with no
+// smoothing, so a constant signal of 0 yields 0 and 1 yields ~1 in one frame.
+test("glTF extras drive the runtime end-to-end in WebGL", async ({ page }) => {
+  const pageErrors = [];
+  page.on("pageerror", (err) => pageErrors.push(err));
+
+  await page.goto("/blender-runtime.html", { waitUntil: "load" });
+  await page.waitForFunction(() => window.__blenderRuntime?.ready === true);
+
+  const low = await page.evaluate(() => {
+    window.__blenderRuntime.setSignal(0);
+    return window.__blenderRuntime.tick();
+  });
+  expect(low).toBeCloseTo(0, 5);
+
+  const high = await page.evaluate(() => {
+    window.__blenderRuntime.setSignal(1);
+    return window.__blenderRuntime.tick();
+  });
+  expect(high).toBeGreaterThan(0.99);
+
+  expect(pageErrors, pageErrors.map((e) => e.message).join("\n")).toHaveLength(0);
+});

e2e/runtime-scene.spec.js

@@ -0,0 +1,33 @@
+import { test, expect } from "@playwright/test";
+
+// RuntimeScene loads a GLB with two position.y-bound cubes and drives them from
+// two signal-source shapes: a page-scripted level and a synthetic baked bake
+// whose playback position the page advances. Both must move their binding.
+test("RuntimeScene drives bindings from scripted and baked sources", async ({
+  page,
+}) => {
+  const pageErrors = [];
+  page.on("pageerror", (err) => pageErrors.push(err));
+
+  await page.goto("/runtime-scene.html", { waitUntil: "load" });
+  await page.waitForFunction(() => window.__runtimeScene?.loaded === true);
+
+  const before = await page.evaluate(() => window.__runtimeScene.sample());
+  await page.evaluate(() => window.__runtimeScene.setLevel(1));
+  await page.waitForFunction(
+    (b) => Math.abs(window.__runtimeScene.sample() - b) > 0.1,
+    before
+  );
+
+  await page.evaluate(() => window.__runtimeScene.setBakedPosition(0));
+  const quiet = await page.evaluate(() => window.__runtimeScene.sampleBaked());
+  await page.evaluate(() => window.__runtimeScene.setBakedPosition(0.75));
+  await page.waitForFunction(
+    (q) => Math.abs(window.__runtimeScene.sampleBaked() - q) > 0.1,
+    quiet
+  );
+
+  expect(pageErrors, pageErrors.map((e) => e.message).join("\n")).toHaveLength(
+    0
+  );
+});

e2e/stem-inspector.spec.js

@@ -0,0 +1,63 @@
+import { test, expect } from "@playwright/test";
+
+// 1s mono 44100 wav: 0.5s LCG noise then silence (same fixture recipe as
+// tools/blender/bake/test-bake-audio.mjs).
+function fixtureWav() {
+  const sampleRate = 44100;
+  const n = sampleRate;
+  const active = n / 2;
+  const buf = Buffer.alloc(44 + n * 2);
+  buf.write("RIFF", 0); buf.writeUInt32LE(36 + n * 2, 4); buf.write("WAVE", 8);
+  buf.write("fmt ", 12); buf.writeUInt32LE(16, 16); buf.writeUInt16LE(1, 20); buf.writeUInt16LE(1, 22);
+  buf.writeUInt32LE(sampleRate, 24); buf.writeUInt32LE(sampleRate * 2, 28); buf.writeUInt16LE(2, 32); buf.writeUInt16LE(16, 34);
+  buf.write("data", 36); buf.writeUInt32LE(n * 2, 40);
+  let s = 1 >>> 0;
+  for (let i = 0; i < active; i++) {
+    s = (s * 1664525 + 1013904223) >>> 0;
+    const v = (s / 0xffffffff) * 2 - 1;
+    buf.writeInt16LE(Math.round(v < 0 ? v * 0x8000 : v * 0x7fff), 44 + i * 2);
+  }
+  return buf;
+}
+
+test("stem inspector lists, bakes, draws, and plays a stem", async ({ page }) => {
+  const pageErrors = [];
+  page.on("pageerror", (err) => pageErrors.push(err));
+
+  await page.route("**/__stems", (route) =>
+    route.fulfill({ contentType: "application/json", body: JSON.stringify([{ song: "e2e", name: "tone.wav" }]) })
+  );
+  await page.route("**/audio/wav/e2e/tone.wav", (route) =>
+    route.fulfill({ contentType: "audio/wav", body: fixtureWav() })
+  );
+
+  await page.goto("/stem-inspector.html", { waitUntil: "load" });
+  await page.waitForFunction(() => window.__stemInspector?.ready === true);
+
+  // fixture stem is unknown to app-config -> unassigned bucket
+  await expect(page.locator("#sidebar h4", { hasText: "unassigned" })).toBeVisible();
+  await page.locator("#sidebar button", { hasText: "tone.wav" }).click();
+
+  // both bakes ran: 1s at 30fps
+  await page.waitForFunction(() => window.__stemInspector?.frames === 30);
+  expect(await page.evaluate(() => window.__stemInspector.lastError)).toBeNull();
+
+  // canvas actually drew something (waveform + curves -> non-blank pixels)
+  const drawn = await page.evaluate(() => {
+    const c = document.getElementById("canvas");
+    const d = c.getContext("2d").getImageData(0, 0, c.width, c.height).data;
+    let nonBlank = 0;
+    for (let i = 3; i < d.length; i += 4) if (d[i] !== 0) nonBlank++;
+    return nonBlank;
+  });
+  expect(drawn).toBeGreaterThan(100);
+
+  // playback advances the readout
+  await page.locator("#play").click();
+  await page.waitForTimeout(400);
+  await expect(page.locator("#readout")).toContainText("t=0.");
+  const t = await page.locator("#readout").textContent();
+  expect(Number(t.match(/t=([\d.]+)s/)[1])).toBeGreaterThan(0);
+
+  expect(pageErrors, pageErrors.map((e) => e.message).join("\n")).toHaveLength(0);
+});

package.json

@@ -20,29 +20,38 @@
     "react": "19.2.7",
     "react-dom": "19.2.7",
     "stats.js": "0.17.0",
-    "three": "0.108.0",
+    "three": "npm:three@0.185.0",
+    "three-legacy": "npm:three@0.108.0",
     "wouter": "3.10.0",
     "zustand": "5.0.14"
   },
   "scripts": {
     "start": "vite",
+    "stems": "vite --open /stem-inspector.html",
     "build": "vite build",
     "preview": "vite preview",
     "lint": "eslint src/ --fix",
+    "test": "vitest run",
     "typecheck": "tsc --noEmit",
     "format": "prettier --write src",
     "test:unit": "vitest run",
-    "test:e2e": "PLAYWRIGHT_FORCE_ASYNC_LOADER=1 playwright test"
+    "test:e2e": "PLAYWRIGHT_FORCE_ASYNC_LOADER=1 playwright test",
+    "test:blender": "for f in tools/blender/tests/test_*.py; do out=$(\"${BLENDER_BIN:-/Applications/Blender.app/Contents/MacOS/Blender}\" --background --python \"$f\" 2>&1); echo \"$out\"; echo \"$out\" | grep -q '^OK ' || { echo \"FAILED: $f\"; exit 1; }; done",
+    "bake:song": "node tools/bake-song.mjs",
+    "test:bake": "node tools/blender/bake/test-bake-audio.mjs && node tools/test-bake-song.mjs",
+    "test:bridge": "python3 tools/blender/bake/test-bake-bridge.py"
   },
   "devDependencies": {
     "@eslint/js": "^9.39.4",
     "@playwright/test": "^1.61.0",
     "@types/chroma-js": "^3.1.2",
     "@types/d3-ease": "^3.0.2",
     "@types/d3-scale-chromatic": "^3.1.0",
+    "@types/node": "^26.0.1",
     "@types/react": "19.2.17",
     "@types/react-dom": "19.2.3",
-    "@types/three": "0.103.2",
+    "@types/three": "0.185.0",
+    "@types/three-legacy": "npm:@types/three@0.103.2",
     "@vanilla-extract/css": "1.20.1",
     "@vanilla-extract/vite-plugin": "5.2.2",
     "@vitejs/plugin-react": "^4.7.0",