fix(music): stream AAC on macOS once the equalizer or crossfade is in use

changelog/v0.1.0-beta3.txt

@@ -71,6 +71,8 @@ MACOS
 - Flow uses the native macOS window controls and titlebar again. Closing the window hides it and keeps playback and downloads running until you quit, and clicking the Dock icon brings it back.
 - The window background now follows your theme. A Tauri bug was clearing it, which showed a mismatched color around the app while resizing.
 - On 120 Hz ProMotion screens Flow now draws at 120 frames per second. WebKit was holding every animation, scroll and lyric line to 60.
+- The music equalizer and crossfade are now heard on macOS. WebKit cannot pass Opus songs through audio processing, so once either is on, songs stream as AAC and the song playing reloads once from where it is.
+- The volume slider keeps working on a downloaded Opus song while the equalizer is on.
 
 LINUX
 - The AppImage is built on Ubuntu 24.04 with a newer WebKitGTK that no longer crashes on color emoji fonts, the cause of the freeze on Fedora 44 and Bazzite. It needs glibc 2.39 or newer, so on Ubuntu 22.04, Mint 21 or Debian 12 use the .deb instead.

src/lib/audio/musicAudioEngine.eq.test.ts

@@ -4,6 +4,14 @@ import { graphSpec, EQ_GRAPH_OFF } from "./eq/graphPlan";
 import { defaultEqState, selectPreset } from "./eq/stateOps";
 import { musicAudioEngine } from "./musicAudioEngine";
 
+const platform = vi.hoisted(() => ({ macos: false }));
+vi.mock("../platform", () => ({
+  IS_LINUX_RUNTIME: false,
+  get IS_MACOS_RUNTIME() {
+    return platform.macos;
+  },
+}));
+
 // A minimal WebAudio stand-in: enough to see which nodes exist and what they were told.
 class FakeParam {
   constructor(public value = 0) {}
@@ -92,6 +100,7 @@ beforeEach(() => {
 });
 
 afterEach(() => {
+  platform.macos = false;
   vi.useRealTimers();
   vi.unstubAllGlobals();
 });
@@ -124,8 +133,8 @@ describe("music EQ graph", () => {
     expect(ctx.biquads[0]!.gain.value).toBe(rock.bands[0]!.gain);
     expect(ctx.biquads[5]!.gain.value).toBe(0);
 
-    await musicAudioEngine.load("http://127.0.0.1/stream/a");
-    musicAudioEngine.preload("b", "http://127.0.0.1/stream/b", null);
+    await musicAudioEngine.load("http://127.0.0.1/stream/a", null);
+    musicAudioEngine.preload("b", "http://127.0.0.1/stream/b", null, null);
     Object.defineProperty(pair[1], "readyState", { value: HTMLMediaElement.HAVE_FUTURE_DATA, configurable: true });
     expect(musicAudioEngine.activatePreloaded("b")).toBe(true);
 
@@ -138,6 +147,27 @@ describe("music EQ graph", () => {
     expect(ctx.biquads.every((f) => f.gain.value === 0)).toBe(true);
   });
 
+  it("asks for AAC on macOS, and keeps a WebM song it cannot hear on the element's volume", async () => {
+    platform.macos = true;
+    const pair = elements();
+    musicAudioEngine.attach(pair);
+    musicAudioEngine.setEq(EQ_GRAPH_OFF, false);
+    expect(musicAudioEngine.wantsAac()).toBe(false);
+    musicAudioEngine.setEq(rock, true);
+    expect(musicAudioEngine.wantsAac()).toBe(true);
+
+    await musicAudioEngine.load("http://127.0.0.1/stream/opus", 'audio/webm; codecs="opus"');
+    await musicAudioEngine.play();
+    musicAudioEngine.setLoudness(null, true);
+    musicAudioEngine.setVolume(0.5);
+    expect(musicAudioEngine.activeBypassesGraph()).toBe(true);
+    expect(pair[0].volume).toBeCloseTo(0.25, 2);
+
+    await musicAudioEngine.load("http://127.0.0.1/stream/aac", 'audio/mp4; codecs="mp4a.40.2"');
+    expect(musicAudioEngine.activeBypassesGraph()).toBe(false);
+    expect(pair[0].volume).toBe(1);
+  });
+
   it("starts the element even when the context will not resume", async () => {
     vi.useFakeTimers();
     const pair = elements();
@@ -175,8 +205,8 @@ describe("music EQ graph", () => {
     Object.defineProperty(pair[0], "paused", { value: false, configurable: true });
     musicAudioEngine.setCrossfadeEnabled(true);
     const ctx = lastContext();
-    await musicAudioEngine.load("http://127.0.0.1/stream/a");
-    musicAudioEngine.preload("b", "http://127.0.0.1/stream/b", null);
+    await musicAudioEngine.load("http://127.0.0.1/stream/a", null);
+    musicAudioEngine.preload("b", "http://127.0.0.1/stream/b", null, null);
     Object.defineProperty(pair[1], "readyState", { value: HTMLMediaElement.HAVE_FUTURE_DATA, configurable: true });
 
     musicAudioEngine.armCrossfade(4);
@@ -205,8 +235,8 @@ describe("crossfade without WebAudio", () => {
     engine.setCrossfadeEnabled(true);
     expect(engine.isEqAvailable()).toBe(false);
 
-    await engine.load("http://127.0.0.1/stream/a");
-    engine.preload("b", "http://127.0.0.1/stream/b", null);
+    await engine.load("http://127.0.0.1/stream/a", null);
+    engine.preload("b", "http://127.0.0.1/stream/b", null, null);
     Object.defineProperty(pair[1], "readyState", { value: HTMLMediaElement.HAVE_FUTURE_DATA, configurable: true });
     Object.defineProperty(pair[0], "paused", { value: false, configurable: true });
     engine.armCrossfade(2);

src/lib/audio/musicAudioEngine.test.ts

@@ -28,8 +28,8 @@ beforeEach(() => {
 
 describe("gapless standby element", () => {
   it("buffers the next track on the element that is not playing", async () => {
-    await musicAudioEngine.load("http://127.0.0.1/stream/current");
-    musicAudioEngine.preload("next", "http://127.0.0.1/stream/next", -4);
+    await musicAudioEngine.load("http://127.0.0.1/stream/current", null);
+    musicAudioEngine.preload("next", "http://127.0.0.1/stream/next", -4, null);
 
     expect(musicAudioEngine.getActiveElement()).toBe(pair[0]);
     expect(pair[1].getAttribute("src")).toBe("http://127.0.0.1/stream/next");
@@ -38,8 +38,8 @@ describe("gapless standby element", () => {
   });
 
   it("swaps only for the track it holds, and empties the one that played", async () => {
-    await musicAudioEngine.load("http://127.0.0.1/stream/current");
-    musicAudioEngine.preload("next", "http://127.0.0.1/stream/next", -4);
+    await musicAudioEngine.load("http://127.0.0.1/stream/current", null);
+    musicAudioEngine.preload("next", "http://127.0.0.1/stream/next", -4, null);
     bufferedEnough(pair[1]);
 
     expect(musicAudioEngine.activatePreloaded("other")).toBe(false);
@@ -53,15 +53,15 @@ describe("gapless standby element", () => {
   });
 
   it("never swaps in a preload that has no audio to play yet", async () => {
-    await musicAudioEngine.load("http://127.0.0.1/stream/current");
-    musicAudioEngine.preload("next", "http://127.0.0.1/stream/next", null);
+    await musicAudioEngine.load("http://127.0.0.1/stream/current", null);
+    musicAudioEngine.preload("next", "http://127.0.0.1/stream/next", null, null);
 
     expect(musicAudioEngine.activatePreloaded("next")).toBe(false);
     expect(musicAudioEngine.getActiveElement()).toBe(pair[0]);
   });
 
   it("reports and drops what the standby element held", () => {
-    musicAudioEngine.preload("next", "http://127.0.0.1/stream/next", null);
+    musicAudioEngine.preload("next", "http://127.0.0.1/stream/next", null, null);
     expect(musicAudioEngine.clearPreload()).toBe("next");
     expect(pair[1].getAttribute("src")).toBeNull();
     expect(musicAudioEngine.clearPreload()).toBeNull();

src/lib/audio/musicAudioEngine.ts

@@ -1,10 +1,11 @@
 import { loudnessFactor, MUSIC_MAX_CUT_DB } from "../loudness";
-import { IS_LINUX_RUNTIME } from "../platform";
+import { IS_LINUX_RUNTIME, IS_MACOS_RUNTIME } from "../platform";
 import { createAudioContext, keepRunning, resumeWithin, webAudioSupported } from "./audioContext";
 import { fadeInGain, fadeOutGain, resetFade, scheduleFade, steppedFade } from "./crossfadeRamp";
 import { EQ_GRAPH_OFF, type EqGraphSpec } from "./eq/graphPlan";
 import { EqChain, RAMP_TIME_CONSTANT } from "./eqChain";
 import { volumeGain } from "./volumeCurve";
+import { webAudioHears } from "./webAudioFormats";
 
 /**
  * One of the two <audio> elements. While one plays, the other can hold the next
@@ -20,6 +21,8 @@ interface Slot {
   /** The same share when WebAudio is not in use, applied to the element's volume. */
   fade: number;
   loudnessDb: number | null;
+  /** The loaded file's MIME type, which decides whether WebAudio can hear it. */
+  mime: string | null;
   /** The track a standby slot holds, so a swap only ever plays the right one. */
   key: string | null;
 }
@@ -31,6 +34,7 @@ const emptySlot = (): Slot => ({
   fadeGain: null,
   fade: 1,
   loudnessDb: null,
+  mime: null,
   key: null,
 });
 
@@ -59,6 +63,7 @@ class MusicAudioEngine {
   private graphReady = false;
   private webAudioFailed = false;
   private availabilityListeners = new Set<(available: boolean) => void>();
+  private graphListeners = new Set<() => void>();
 
   private volume = 1;
   private muted = false;
@@ -145,6 +150,7 @@ class MusicAudioEngine {
       this.chain.apply(this.eqSpec);
       // Turned on mid-song: the context has to run for the element to be heard.
       if (this.active.el && !this.active.el.paused) void resumeWithin(ctx);
+      for (const listener of this.graphListeners) listener();
     } catch (err) {
       console.warn("[musicAudioEngine] WebAudio graph unavailable, playing without the equalizer", err);
       this.webAudioFailed = true;
@@ -155,10 +161,12 @@ class MusicAudioEngine {
 
   // --- transport ----------------------------------------------------------
 
-  async load(url: string): Promise<void> {
+  async load(url: string, mime: string | null): Promise<void> {
     const el = this.active.el;
     if (!el) return;
     this.active.key = null;
+    this.active.mime = mime;
+    this.applyGain();
     el.src = url;
     try {
       el.load();
@@ -171,11 +179,12 @@ class MusicAudioEngine {
    * take over the moment the current track ends. A no-op when it already holds
    * that track, or while it is still fading out the last one.
    */
-  preload(key: string, url: string, loudnessDb: number | null): void {
+  preload(key: string, url: string, loudnessDb: number | null, mime: string | null): void {
     const slot = this.standby;
     if (!slot.el || slot.key === key || this.fading) return;
     slot.key = key;
     slot.loudnessDb = loudnessDb;
+    slot.mime = mime;
     slot.el.src = url;
     try {
       slot.el.load();
@@ -235,14 +244,15 @@ class MusicAudioEngine {
     if (slot.el) unload(slot.el);
     slot.key = null;
     slot.loudnessDb = null;
+    slot.mime = null;
   }
 
   // The two elements play together while their gains move in opposite directions; the one
   // fading out is emptied when the fade ends.
   private startFade(previous: Slot, seconds: number): void {
     const next = this.active;
     let cancelSteps: (() => void) | null = null;
-    if (this.graphReady && this.ctx && previous.fadeGain && next.fadeGain) {
+    if (this.ctx && this.routes(previous) && this.routes(next) && previous.fadeGain && next.fadeGain) {
       const t = this.ctx.currentTime;
       scheduleFade(previous.fadeGain.gain, "out", t, seconds);
       scheduleFade(next.fadeGain.gain, "in", t, seconds);
@@ -436,26 +446,30 @@ class MusicAudioEngine {
   private applyGain(): void {
     const vol = (this.muted ? 0 : volumeGain(this.volume)) * this.fadeLevel;
 
-    if (this.graphReady && this.ctx && this.volGain) {
-      const t = this.ctx.currentTime;
-      this.volGain.gain.setTargetAtTime(vol, t, RAMP_TIME_CONSTANT);
-      for (const slot of this.slots) {
-        slot.normGain?.gain.setTargetAtTime(this.normalizationFactor(slot.loudnessDb), t, RAMP_TIME_CONSTANT);
-        if (slot.el) {
-          slot.el.volume = 1;
-          slot.el.muted = false;
-        }
-      }
-    } else {
-      for (const slot of this.slots) {
-        if (!slot.el) continue;
-        const norm = this.normalizationFactor(slot.loudnessDb);
+    const t = this.ctx?.currentTime ?? 0;
+    if (this.graphReady && this.volGain) this.volGain.gain.setTargetAtTime(vol, t, RAMP_TIME_CONSTANT);
+    // A scheduled crossfade owns the fade gains until it ends; a stepped one moves `slot.fade`.
+    const scheduledFade = !!this.fading && !this.fading.cancelSteps;
+    for (const slot of this.slots) {
+      if (!slot.el) continue;
+      const norm = this.normalizationFactor(slot.loudnessDb);
+      if (this.routes(slot)) {
+        slot.normGain?.gain.setTargetAtTime(norm, t, RAMP_TIME_CONSTANT);
+        if (!scheduledFade) slot.fadeGain?.gain.setTargetAtTime(slot.fade, t, RAMP_TIME_CONSTANT);
+        slot.el.volume = 1;
+        slot.el.muted = false;
+      } else {
         slot.el.volume = Math.max(0, Math.min(1, vol * norm * slot.fade));
         slot.el.muted = this.muted;
       }
     }
   }
 
+  /** Whether `slot` plays through the graph, rather than past it on its own volume. */
+  private routes(slot: Slot): boolean {
+    return this.graphReady && !!slot.source && webAudioHears(slot.mime);
+  }
+
   // --- crossfade ----------------------------------------------------------
 
   /** Crossfades are timed on the audio clock, so turning them on brings WebAudio in. */
@@ -489,6 +503,25 @@ class MusicAudioEngine {
     if (this.needsGraph() && this.active.el && !this.active.el.paused) this.ensureGraph();
   }
 
+  /**
+   * Whether streams should come as AAC: on macOS only MP4 audio reaches the graph, and once the
+   * graph is in use every song plays through it.
+   */
+  wantsAac(): boolean {
+    return IS_MACOS_RUNTIME && (this.graphReady || this.needsGraph());
+  }
+
+  /** Whether the song playing bypasses the graph, so the equalizer cannot be heard on it. */
+  activeBypassesGraph(): boolean {
+    return this.graphReady && !!this.active.el?.getAttribute("src") && !this.routes(this.active);
+  }
+
+  /** Called once the graph is built, when songs loaded before it may need loading again. */
+  onGraphReady(listener: () => void): () => void {
+    this.graphListeners.add(listener);
+    return () => this.graphListeners.delete(listener);
+  }
+
   /** False once WebAudio failed here, so the equalizer cannot be heard. */
   isEqAvailable(): boolean {
     return !this.webAudioFailed && webAudioSupported();

src/lib/audio/musicAudioSettings.ts

@@ -63,6 +63,7 @@ export function bindMusicAudioSettings(): void {
       (preview !== null && graphChangesSound(graphSpec(state, false, preview), REFERENCE_SAMPLE_RATE));
     musicAudioEngine.setEq(spec, engaged);
   };
+  musicAudioEngine.onGraphReady(() => useMusicPlayerStore.getState()._reloadForWebAudio());
   applyEq();
   useEqStore.subscribe((state, previous) => {
     if (state.state !== previous.state || state.bypass !== previous.bypass || state.preview !== previous.preview) {

src/lib/audio/webAudioFormats.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, it } from "vitest";
+
+import { webAudioHears } from "./webAudioFormats";
+
+describe("webAudioHears", () => {
+  it("hears MP4 audio on macOS", () => {
+    expect(webAudioHears('audio/mp4; codecs="mp4a.40.2"', true)).toBe(true);
+    expect(webAudioHears("audio/mpeg", true)).toBe(true);
+  });
+
+  it("does not hear WebM or Ogg on macOS", () => {
+    expect(webAudioHears('audio/webm; codecs="opus"', true)).toBe(false);
+    expect(webAudioHears("audio/ogg", true)).toBe(false);
+  });
+
+  it("hears every format elsewhere", () => {
+    expect(webAudioHears('audio/webm; codecs="opus"', false)).toBe(true);
+  });
+
+  it("assumes an unknown format is heard", () => {
+    expect(webAudioHears(null, true)).toBe(true);
+  });
+});

src/lib/audio/webAudioFormats.ts

@@ -0,0 +1,11 @@
+import { IS_MACOS_RUNTIME } from "../platform";
+
+/**
+ * Whether WebAudio hears an element playing `mime`. macOS WebKit plays WebM and Ogg through its
+ * own player, which never feeds a MediaElementAudioSourceNode: the node gets silence while the
+ * element keeps playing straight to the speakers, so filters and gains in the graph do nothing.
+ */
+export function webAudioHears(mime: string | null, macos = IS_MACOS_RUNTIME): boolean {
+  return !macos || mime === null || !/webm|ogg|opus/i.test(mime);
+}
+

src/lib/musicPreload.ts

@@ -34,8 +34,8 @@ export async function preloadTrack(
   preloading = videoId;
   recordPlayerEvent(`music preload start: ${videoId}`);
   try {
-    const info = await resolveMusicStream(videoId, quality);
-    if (stillUpcoming(videoId)) musicAudioEngine.preload(videoId, info.audioUrl, info.loudnessDb);
+    const info = await resolveMusicStream(videoId, quality, { preferAac: musicAudioEngine.wantsAac() });
+    if (stillUpcoming(videoId)) musicAudioEngine.preload(videoId, info.audioUrl, info.loudnessDb, info.mimeType);
   } catch {
     // Speculative: if it fails, the track resolves again when it is played.
   } finally {

src/lib/musicStreamResolution.ts

@@ -29,7 +29,7 @@ const MUSIC_STREAM_TIMEOUT_MS = 15_000;
 const MAX_CACHED_TRACKS = 8;
 
 export interface MusicStreamOptions {
-  /** Ask for AAC over Opus, after the element could not decode the first answer. */
+  /** Ask for AAC over Opus: after the element could not decode the first answer, or for WebAudio. */
   preferAac?: boolean;
 }
 
@@ -120,11 +120,12 @@ export function resolveMusicStream(
 export function prefetchMusicStream(
   videoId: string | null | undefined,
   quality: MusicAudioQuality,
+  options?: MusicStreamOptions,
 ) {
   if (!videoId) return;
-  const key = keyFor(videoId, quality);
+  const key = keyFor(videoId, quality, options);
   if (readCache(key) || walks.has(key)) return;
-  void request(videoId, quality).catch(() => {
+  void request(videoId, quality, options).catch(() => {
     // A speculative resolve that fails costs nothing — playing the track will
     // resolve it again and surface the real error then.
   });