feat(player): apply the equalizer to videos

changelog/v0.1.0-beta3.txt

@@ -62,6 +62,8 @@ AUDIO
 - The music volume slider follows how loud it sounds, so each step changes the level by a similar amount. Your saved volume plays at the same level after the update.
 - Music loudness is measured the way Android does it, from the perceptual loudness first, and loud songs are never turned down by more than 20 dB.
 - An optional crossfade, off by default, fades the next song in over 1 to 12 seconds. Consecutive tracks from the same album and repeat-one keep their gapless joins.
+- The equalizer now also applies to videos, including the pop-out player, and opens from the video settings menu. Shorts are left as they are.
+- Videos only go through audio processing once the equalizer changes the sound, and only after a short silent self-test shows it works. Where it does not, as on some Linux setups, videos keep playing normally and the menu says the equalizer is not available.
 
 MACOS
 - Flow uses the native macOS window controls and titlebar again. Closing the window hides it and keeps playback and downloads running until you quit, and clicking the Dock icon brings it back.

src/components/music/eq/EqualizerDialog.tsx

@@ -35,6 +35,11 @@ export function EqualizerDialog() {
   const setPreview = useEqStore((s) => s.setPreview);
   const setBypass = useEqStore((s) => s.setBypass);
 
+  useEffect(() => {
+    useEqStore.setState({ editorHosted: true });
+    return () => useEqStore.setState({ editorHosted: false });
+  }, []);
+
   useEffect(() => {
     if (!open) return;
     const onKey = (event: KeyboardEvent) => {

src/components/player/FlowPlayerControls.tsx

@@ -22,6 +22,7 @@ import {
   Volume2,
   VolumeX,
   Sliders,
+  SlidersHorizontal,
 } from "lucide-react";
 import { usePlayerStore, type PlaybackRate } from "../../store/usePlayerStore";
 import { useSettingsStore, type SponsorBlockCategory } from "../../store/useSettingsStore";
@@ -51,6 +52,9 @@ import { sponsorBlockCategoryLabel } from "../../lib/sponsorBlockCategories";
 import { getString } from "../../lib/i18n/index";
 import { SleepTimerLabel } from "../sleep/SleepTimerLabel";
 import { SleepTimerPanel } from "../sleep/SleepTimerPanel";
+import { useEqActive } from "../../lib/audio/eq/useEqualizer";
+import { videoEqAvailable } from "../../lib/audio/mediaEqEngine";
+import { useEqStore } from "../../store/useEqStore";
 import { Slider } from "../ui/Slider";
 import { videoCodecLabel } from "../../lib/settings/playerRuntime";
 
@@ -478,6 +482,10 @@ export const FlowPlayerControls: React.FC<FlowPlayerControlsProps> = ({
     updateHoverPreview(event.clientX, event.currentTarget);
   };
 
+  const eqActive = useEqActive();
+  const eqEditorHosted = useEqStore((s) => s.editorHosted);
+  const openEqEditor = useEqStore((s) => s.setEditorOpen);
+
   const renderSettingRow = (
     label: string,
     value: React.ReactNode,
@@ -952,6 +960,17 @@ export const FlowPlayerControls: React.FC<FlowPlayerControlsProps> = ({
               <Pause size={18} />,
               () => setSettingsPane("sleep")
             )}
+            {eqEditorHosted &&
+              renderSettingRow(
+                getString("music_equalizer"),
+                // Read when the menu draws: the self-test only settles once the equalizer is used.
+                getString(!videoEqAvailable() ? "eq_state_unavailable" : eqActive ? "eq_state_on" : "eq_state_off"),
+                <SlidersHorizontal size={18} />,
+                () => {
+                  setSettingsOpen(false);
+                  openEqEditor(true);
+                }
+              )}
           </div>
         )}
 

src/components/player/Player.tsx

@@ -28,6 +28,7 @@ import {
 import { SubtitleOverlay } from "./SubtitleOverlay";
 import { SETTINGS } from "../../lib/settings/schema";
 import { loudnessFactor } from "../../lib/loudness";
+import { useVideoEq } from "../../lib/audio/useVideoEq";
 import { IS_LINUX_RUNTIME } from "../../lib/platform";
 import {
   AUDIO_DRIFT_TOLERANCE_SECONDS,
@@ -595,6 +596,8 @@ export const Player: React.FC<PlayerProps> = ({
     !isHlsPlayback &&
     (hasSelectedAlternateAudio || (!!selectedQuality && !selectedQuality.hasAudio));
 
+  useVideoEq(videoRef, audioRef, usesExternalAudio ? selectedAudioTrack?.localUrl ?? null : null);
+
   const isPipMode = videoPlayerMode === "pip";
   const isTheaterSurface = isTheaterMode && !isPipMode;
   const shouldShowControls = controlsVisible || !isPlaying || settingsOpen || isScrubbing;
@@ -2510,6 +2513,8 @@ export const Player: React.FC<PlayerProps> = ({
 
       <video
         ref={videoRef}
+        // Set before src: the equalizer can only read cross-origin audio fetched with CORS.
+        crossOrigin="anonymous"
         src={isDashPlayback || isHlsPlayback ? undefined : src || undefined}
         poster={effectivePoster}
         loop={videoLoopEnabled && !isLive}
@@ -2581,6 +2586,7 @@ export const Player: React.FC<PlayerProps> = ({
       {usesExternalAudio && selectedAudioTrack?.localUrl && (
         <audio
           ref={audioRef}
+          crossOrigin="anonymous"
           src={selectedAudioTrack.localUrl}
           preload="auto"
           onLoadedMetadata={handleAudioLoadedMetadata}

src/lib/audio/mediaEqEngine.test.ts

@@ -0,0 +1,159 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../diagnostics", () => ({ logToBackend: vi.fn(async () => {}) }));
+
+import { graphSpec } from "./eq/graphPlan";
+import { defaultEqState, selectPreset } from "./eq/stateOps";
+
+class FakeParam {
+  value = 1;
+  setTargetAtTime = vi.fn((v: number) => {
+    this.value = v;
+  });
+}
+class FakeNode {
+  connect = vi.fn((node: unknown) => node);
+  disconnect = vi.fn();
+}
+let probeSignal = 0.2;
+const contexts: FakeContext[] = [];
+class FakeContext extends EventTarget {
+  state: AudioContextState = "running";
+  currentTime = 0;
+  sampleRate = 48_000;
+  baseLatency = 0.005;
+  outputLatency = 0.02;
+  destination = new FakeNode();
+  sources: HTMLMediaElement[] = [];
+  resume = vi.fn(async () => {});
+  constructor() {
+    super();
+    contexts.push(this);
+  }
+  createGain() {
+    return Object.assign(new FakeNode(), { gain: new FakeParam() });
+  }
+  createBiquadFilter() {
+    return Object.assign(new FakeNode(), {
+      type: "peaking",
+      frequency: new FakeParam(),
+      Q: new FakeParam(),
+      gain: new FakeParam(),
+    });
+  }
+  createWaveShaper() {
+    return Object.assign(new FakeNode(), { curve: null });
+  }
+  createAnalyser() {
+    return Object.assign(new FakeNode(), {
+      fftSize: 64,
+      getFloatTimeDomainData: (data: Float32Array) => data.fill(probeSignal),
+    });
+  }
+  createMediaElementSource(el: HTMLMediaElement) {
+    this.sources.push(el);
+    return new FakeNode();
+  }
+}
+
+const media = <T extends HTMLMediaElement>(tag: "video" | "audio", cors = true): T => {
+  const el = document.createElement(tag) as T;
+  if (cors) el.crossOrigin = "anonymous";
+  return el;
+};
+
+const rock = graphSpec(selectPreset(defaultEqState(), "builtin:rock"), false);
+
+async function freshEngine() {
+  vi.resetModules();
+  const module = await import("./mediaEqEngine");
+  return { engine: new module.MediaEqEngine(), available: module.videoEqAvailable };
+}
+
+const settle = () => vi.advanceTimersByTimeAsync(2_000);
+
+beforeEach(() => {
+  vi.useFakeTimers();
+  contexts.length = 0;
+  probeSignal = 0.2;
+  vi.stubGlobal("AudioContext", FakeContext);
+  vi.stubGlobal("URL", Object.assign(URL, { createObjectURL: () => "blob:tone", revokeObjectURL: () => {} }));
+  vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(undefined);
+  vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});
+});
+
+afterEach(() => {
+  vi.useRealTimers();
+  vi.unstubAllGlobals();
+  vi.restoreAllMocks();
+});
+
+describe("MediaEqEngine", () => {
+  it("leaves the video alone until the equalizer changes the sound", async () => {
+    const { engine } = await freshEngine();
+    engine.setElements([media("video"), null]);
+    engine.setEq(rock, false);
+    await settle();
+    expect(contexts).toHaveLength(0);
+  });
+
+  it("routes the video and its separate audio once each, and follows a new audio element", async () => {
+    const { engine } = await freshEngine();
+    const video = media<HTMLVideoElement>("video");
+    const audio = media<HTMLAudioElement>("audio");
+    engine.setElements([video, audio]);
+    engine.setEq(rock, true);
+    await settle();
+
+    const ctx = contexts[0]!;
+    // The first source is the self-test's own element.
+    expect(ctx.sources.slice(1)).toEqual([video, audio]);
+
+    engine.setElements([video, audio]);
+    engine.setEq(rock, true);
+    await settle();
+    expect(ctx.sources).toHaveLength(3);
+
+    const nextAudio = media<HTMLAudioElement>("audio");
+    engine.setElements([video, nextAudio]);
+    await settle();
+    expect(ctx.sources.slice(1)).toEqual([video, audio, nextAudio]);
+  });
+
+  it("falls back to plain playback when WebAudio returns silence", async () => {
+    probeSignal = 0;
+    const { engine, available } = await freshEngine();
+    const video = media<HTMLVideoElement>("video");
+    engine.setElements([video, null]);
+    engine.setEq(rock, true);
+    await settle();
+
+    expect(contexts[0]!.sources).not.toContain(video);
+    expect(available()).toBe(false);
+  });
+
+  it("tries the self-test again when the video plays, if it could not run before", async () => {
+    const play = vi.spyOn(HTMLMediaElement.prototype, "play").mockRejectedValueOnce(new Error("NotAllowedError"));
+    const { engine, available } = await freshEngine();
+    const video = media<HTMLVideoElement>("video");
+    engine.setElements([video, null]);
+    engine.setEq(rock, true);
+    await settle();
+    expect(contexts[0]!.sources).not.toContain(video);
+    expect(available()).toBe(true);
+
+    play.mockResolvedValue(undefined);
+    video.dispatchEvent(new Event("play"));
+    await settle();
+    expect(contexts[0]!.sources).toContain(video);
+  });
+
+  it("never routes an element fetched without CORS, which would play silent", async () => {
+    const { engine } = await freshEngine();
+    const video = media<HTMLVideoElement>("video", false);
+    engine.setElements([video, null]);
+    engine.setEq(rock, true);
+    await settle();
+    expect(contexts[0]!.sources).not.toContain(video);
+  });
+});

src/lib/audio/mediaEqEngine.ts

@@ -0,0 +1,129 @@
+import { logToBackend } from "../diagnostics";
+import { createAudioContext, keepRunning, resumeWithin } from "./audioContext";
+import { EQ_GRAPH_OFF, type EqGraphSpec } from "./eq/graphPlan";
+import { EqChain } from "./eqChain";
+import { probeMediaElementRouting } from "./webAudioProbe";
+
+// One context for every video player in this window; each player keeps its own chain.
+let sharedContext: AudioContext | null = null;
+let routingWorks: Promise<boolean | null> | null = null;
+// WebAudio failed or delivered silence in this window: video plays from its elements alone.
+let unavailable = false;
+const routedElements = new Set<HTMLMediaElement>();
+
+function videoContext(): AudioContext | null {
+  if (!sharedContext) {
+    sharedContext = createAudioContext();
+    if (sharedContext) keepRunning(sharedContext, () => [...routedElements].some((el) => !el.paused));
+  }
+  return sharedContext;
+}
+
+/** Runs the routing self-test once per window; a test that could not run is tried again later. */
+function checkRouting(ctx: AudioContext): Promise<boolean | null> {
+  routingWorks ??= probeMediaElementRouting(ctx).then((works) => {
+    if (works === null) routingWorks = null;
+    if (works === false) {
+      unavailable = true;
+      void logToBackend("warn", "video equalizer off: WebAudio returned silence from a media element");
+    }
+    return works;
+  });
+  return routingWorks;
+}
+
+/**
+ * The equalizer for one video player: its <video> element and the separate <audio> element
+ * adaptive playback uses. Elements are routed through WebAudio only once the equalizer changes
+ * the sound, and each gets its source node exactly once in its life, as WebAudio requires.
+ */
+export class MediaEqEngine {
+  private chain: EqChain | null = null;
+  private elements: HTMLMediaElement[] = [];
+  private sources = new WeakMap<HTMLMediaElement, MediaElementAudioSourceNode>();
+  private watched = new WeakSet<HTMLMediaElement>();
+  private spec: EqGraphSpec = EQ_GRAPH_OFF;
+  private engaged = false;
+  private routing = false;
+
+  /** The player's current media elements; an element that left is let go. */
+  setElements(elements: (HTMLMediaElement | null)[]): void {
+    const next = elements.filter((el): el is HTMLMediaElement => el !== null);
+    for (const el of this.elements) {
+      if (next.includes(el)) continue;
+      this.sources.get(el)?.disconnect();
+      routedElements.delete(el);
+    }
+    this.elements = next;
+    for (const el of next) {
+      if (this.watched.has(el)) continue;
+      this.watched.add(el);
+      // A routed element needs its context running; one not routed yet gets another chance,
+      // since the self-test needs a context the webview lets run.
+      el.addEventListener("play", () => {
+        if (routedElements.has(el) && sharedContext) void resumeWithin(sharedContext);
+        else if (this.engaged) void this.route();
+      });
+    }
+    if (this.engaged) void this.route();
+  }
+
+  setEq(spec: EqGraphSpec, engaged: boolean): void {
+    this.spec = spec;
+    this.engaged = engaged;
+    if (engaged) void this.route();
+    this.chain?.apply(spec);
+  }
+
+  dispose(): void {
+    this.setElements([]);
+    this.chain?.output.disconnect();
+    this.chain = null;
+  }
+
+  private async route(): Promise<void> {
+    if (unavailable || this.routing) return;
+    const ctx = videoContext();
+    if (!ctx) {
+      unavailable = true;
+      return;
+    }
+    this.routing = true;
+    try {
+      if ((await checkRouting(ctx)) !== true || !this.engaged) return;
+      if (!this.chain) {
+        this.chain = new EqChain(ctx);
+        this.chain.output.connect(ctx.destination);
+        this.chain.apply(this.spec);
+      }
+      for (const el of this.elements) this.connect(ctx, this.chain, el);
+    } catch (error) {
+      unavailable = true;
+      console.warn("[mediaEqEngine] WebAudio unavailable, video plays without the equalizer", error);
+    } finally {
+      this.routing = false;
+    }
+  }
+
+  private connect(ctx: AudioContext, chain: EqChain, el: HTMLMediaElement): void {
+    if (this.sources.has(el) && routedElements.has(el)) return;
+    // Without CORS from the start the samples reach WebAudio as silence.
+    if (el.crossOrigin !== "anonymous") return;
+    let source = this.sources.get(el);
+    if (!source) {
+      source = ctx.createMediaElementSource(el);
+      this.sources.set(el, source);
+      void logToBackend("info", "video equalizer routed", {
+        element: el.tagName.toLowerCase(),
+        baseLatencyMs: Math.round(ctx.baseLatency * 1_000),
+        outputLatencyMs: Math.round((ctx.outputLatency ?? 0) * 1_000),
+      });
+    }
+    source.connect(chain.input);
+    routedElements.add(el);
+    if (!el.paused) void resumeWithin(ctx);
+  }
+}
+
+/** False once WebAudio failed or proved silent for video in this window. */
+export const videoEqAvailable = (): boolean => !unavailable;