From f048ea532b0d4995b8695c542b3ae1c9bd99da78 Mon Sep 17 00:00:00 2001 From: Valentin Weyer Date: Wed, 8 Jul 2026 14:29:46 +0200 Subject: [PATCH 01/12] feat(transcribe): per-channel acoustic speaker diarization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a macOS-only Swift/CoreML sidecar (diarize-sidecar, wrapping FluidAudio's Sortformer) that diarizes the mic and system-audio channels independently, so multiple speakers sharing one side of a call (in-person conversations on mic, or multiple remote participants on system audio) get labelled "You" / "Speaker 2" / "Speaker 3" instead of being lumped together. The dominant-by-duration cluster on each channel keeps the legacy "You"/ "Others" label; other clusters are numbered by first chronological appearance across both channels. Any failure (missing binary, timeout, bad output, single-cluster result) falls back to today's exact channel-only behaviour, so this can never fail a meeting. Two fixes from testing against a real in-person recording: - is_diarised now reflects whether the output actually has more than one speaker label, not whether both channels had content — the old check discarded the whole labelled transcript whenever one channel (e.g. system audio with nothing playing) was empty. - Long Parakeet sentences that span multiple real diarizer turns (no strong punctuation break in a long run of speech) are now split at the word level and reassigned per-word, instead of forcing the entire block onto whichever diarizer segment the sentence's midpoint happened to land in. --- .gitignore | 5 + CLAUDE.md | 33 +- .../src/components/TranscriptPanel.tsx | 6 +- .../src/lib/transcriptSegments.test.ts | 12 + app/renderer/src/lib/transcriptSegments.ts | 18 +- diarize-sidecar/Package.resolved | 14 + diarize-sidecar/Package.swift | 19 + diarize-sidecar/Sources/main.swift | 181 ++++++++ e2e/fixtures/say-stereo-wav.ts | 116 +++++ e2e/specs/speaker-diarization.t2.spec.ts | 122 +++++ scripts/build-diarize-sidecar.sh | 31 ++ src/_parakeet_mlx.py | 16 + src/transcriber.py | 366 ++++++++++++++- stenoai.spec | 9 + tests/test_transcriber_diarisation.py | 422 +++++++++++++++++- 15 files changed, 1348 insertions(+), 22 deletions(-) create mode 100644 diarize-sidecar/Package.resolved create mode 100644 diarize-sidecar/Package.swift create mode 100644 diarize-sidecar/Sources/main.swift create mode 100644 e2e/fixtures/say-stereo-wav.ts create mode 100644 e2e/specs/speaker-diarization.t2.spec.ts create mode 100755 scripts/build-diarize-sidecar.sh diff --git a/.gitignore b/.gitignore index 2cf45d00..e4d5a948 100644 --- a/.gitignore +++ b/.gitignore @@ -199,3 +199,8 @@ ollama.pid # Local design reference library (cloned, not vendored) .reference/ + +# Swift Package Manager build artifacts for the diarize-sidecar helper +# (fetched FluidAudio checkout + compiled binaries) — scripts/build-diarize-sidecar.sh +# regenerates this; only diarize-sidecar/{Package.swift,Package.resolved,Sources/} are tracked. +diarize-sidecar/.build/ diff --git a/CLAUDE.md b/CLAUDE.md index 8af92f9e..4962c477 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ The app is a thin Electron shell over a PyInstaller-bundled Python CLI. There is - **Python CLI (`simple_recorder.py`, ~2.9k lines, ~60 click commands)** is the single entry point bundled by `stenoai.spec`. Sub-modules in `src/`: `audio_recorder` (sounddevice), `transcriber` (pywhispercpp), `summarizer` (Ollama HTTP client), `ollama_manager` (lifecycle of the bundled `ollama serve`), `config` (JSON-backed user settings + model registry), `folders`, `models`, `whisper_models`. - **State across CLI invocations** is persisted to `recorder_state.json` and similar small JSON files — there is no daemon. Long-running recordings are a `record` subprocess kept alive by the Electron main process. - **User data lives in `~/Library/Application Support/stenoai/`** (`recordings/`, `transcripts/`, `output/`), resolved via `src.config.get_data_dirs()`. Repo-root `recordings/`/`transcripts/`/`output/` dirs are dev-only scratch. -- **Bundled binaries (`bin/`)**: Ollama + ffmpeg, downloaded by `scripts/download-ollama.sh`. PyInstaller copies them into `dist/stenoai/ollama/` and `dist/stenoai/ffmpeg`. Electron then re-bundles `dist/stenoai/` as an `extraResource`. +- **Bundled binaries (`bin/`)**: Ollama + ffmpeg, downloaded by `scripts/download-ollama.sh`. PyInstaller copies them into `dist/stenoai/ollama/` and `dist/stenoai/ffmpeg`. Electron then re-bundles `dist/stenoai/` as an `extraResource`. `bin/steno-diarize` (macOS only) is a separate Swift/CoreML sidecar built by `scripts/build-diarize-sidecar.sh` — see "Speaker diarization" below. - **Deep links**: app registers the `stenoai://` URL scheme. Handler logic is in `app/main.js` near `SHORTCUT_PROTOCOL`. Used by macOS Shortcuts: `stenoai://record/start?name=...` and `stenoai://record/stop`. ## Development Commands @@ -39,6 +39,28 @@ The Electron build pulls the bundled backend from `../dist/stenoai` via `extraRe For setup from a clean checkout, see `CONTRIBUTING.md` and `README.md`. +### Speaker diarization (macOS only) +Per-channel acoustic speaker diarization (splitting multiple speakers sharing +one side of a call — e.g. two people around one mic, or multiple remote +participants on system audio) runs through `bin/steno-diarize`, a Swift/ +CoreML sidecar (`diarize-sidecar/`) wrapping FluidAudio's Sortformer +diarizer, invoked from Python (`src.transcriber._run_steno_diarize`) — never +from Electron, since the batch pipeline is entirely Python-orchestrated. +Build it *before* `pyinstaller stenoai.spec`, same as `download-ollama.sh`: + +``` +scripts/build-diarize-sidecar.sh # outputs bin/steno-diarize +scripts/download-ollama.sh +pyinstaller stenoai.spec --noconfirm +``` + +`stenoai.spec` bundles `bin/steno-diarize` only when it exists and only on +macOS (`_IS_DARWIN`), so skipping the build step is safe — the app falls +back to the legacy channel-only "You"/"Others" labeling whenever the binary +or a diarization run is unavailable (missing binary, timeout, bad output); +this never fails a meeting. Windows/Linux never get acoustic diarization — +`_resolve_steno_diarize()` returns `None` immediately off-darwin. + ### End-to-end tests (Playwright) The e2e suite drives the **real Electron app** (real window, real clicks) to catch full-app regressions like the org-provider reset before they reach users. It lives @@ -97,7 +119,14 @@ overrides an agent's own test-level defaults. `setup-check.t2` (the setup-wizard allGood + checks contract) (all model-free, run in `t2-macos` / `t2-windows`); `transcription-pipeline.t2` and `honest-failure.t2` (tagged - `@pipeline`, run in `t2-pipeline-macos` / `t2-pipeline-windows`). Engine selection + `@pipeline`, run in `t2-pipeline-macos` / `t2-pipeline-windows`), and + `speaker-diarization.t2` (also `@pipeline`, macOS-only — skips loudly off-darwin + and when macOS's `say` TTS is unavailable — synthesizes real speech via `say` so + Parakeet/whisper.cpp produce real ASR segments, points + `STENOAI_DIARIZE_SIDECAR_PATH` at a fixture script returning fixed 2-speaker JSON, + and asserts the saved transcript's per-channel "You"/"Speaker 2"/"Others" labeling + and cross-channel numbering; the real `steno-diarize`/Sortformer binary itself was + validated directly against real recordings rather than in CI). Engine selection for `@pipeline` specs is shared via `e2e/fixtures/engine.ts`; model-free T2 setup helpers (deterministic recording config + seeded meeting summaries) live in `e2e/fixtures/user-config.ts`. The core-loop specs drive the preload IPC bridge and diff --git a/app/renderer/src/components/TranscriptPanel.tsx b/app/renderer/src/components/TranscriptPanel.tsx index 1bfe659f..78fe9362 100644 --- a/app/renderer/src/components/TranscriptPanel.tsx +++ b/app/renderer/src/components/TranscriptPanel.tsx @@ -133,8 +133,10 @@ function TranscriptRow({ segment, highlight }: { segment: Segment; highlight: st // showing their content as un-bubbled plain text (or worse, on the // "Others" side) reads as wrong. Granola takes the same charitable // default — when in doubt, attribute to the mic owner. Explicit - // `Others` markers still render as grey/left. - const isYou = segment.speaker !== 'Others'; + // `Others` markers and acoustically-diarized `Speaker N` markers + // (transcriber.py's _resolve_speaker_placeholders) both render as + // grey/left — only an exact "You" (or no marker at all) is "self". + const isYou = segment.speaker === 'You' || segment.speaker == null; return (
{segment.timestamp && ( diff --git a/app/renderer/src/lib/transcriptSegments.test.ts b/app/renderer/src/lib/transcriptSegments.test.ts index 23887ae0..60f9afee 100644 --- a/app/renderer/src/lib/transcriptSegments.test.ts +++ b/app/renderer/src/lib/transcriptSegments.test.ts @@ -40,6 +40,18 @@ describe('parseTranscript — diarised', () => { expect(segs[0].text).toContain('line two'); expect(segs[1].timestamp).toBe('00:05'); }); + + test('parses acoustically-diarized "Speaker N" labels alongside You/Others', () => { + const segs = parseTranscript( + '[00:00] [You] Hello\n\n[00:05] [Speaker 2] Hi there\n\n[00:10] [Others] Welcome', + true, + ); + expect(segs).toEqual([ + { speaker: 'You', text: 'Hello', timestamp: '00:00' }, + { speaker: 'Speaker 2', text: 'Hi there', timestamp: '00:05' }, + { speaker: 'Others', text: 'Welcome', timestamp: '00:10' }, + ]); + }); }); describe('parseTranscript — non-diarised', () => { diff --git a/app/renderer/src/lib/transcriptSegments.ts b/app/renderer/src/lib/transcriptSegments.ts index 6dfb182f..aaf1a54a 100644 --- a/app/renderer/src/lib/transcriptSegments.ts +++ b/app/renderer/src/lib/transcriptSegments.ts @@ -3,7 +3,10 @@ // unit-tested in isolation, and so the live dock could share it later. export interface Segment { - speaker: 'You' | 'Others' | null; + /** 'You' / 'Others' for the channel-only fallback, or 'Speaker N' for an + * acoustically-diarized secondary speaker on a channel (see + * transcriber.py's _resolve_speaker_placeholders). */ + speaker: string | null; text: string; /** `MM:SS` / `H:MM:SS` offset parsed from a diarised line's leading * `[MM:SS]` marker (transcriber.py writes it). Absent on older transcripts @@ -11,12 +14,13 @@ export interface Segment { timestamp?: string; } -// A diarised line: an optional `[MM:SS]` / `[H:MM:SS]` timestamp, then the -// `[You]`/`[Others]` speaker marker, then the text up to the next marker (or -// end). Matched globally rather than split so the optional timestamp stays -// attached to its own segment instead of trailing the previous one. +// A diarised line: an optional `[MM:SS]` / `[H:MM:SS]` timestamp, then a +// `[You]` / `[Others]` / `[Speaker N]` speaker marker, then the text up to +// the next marker (or end). Matched globally rather than split so the +// optional timestamp stays attached to its own segment instead of trailing +// the previous one. const DIARISED_SEGMENT_RE = - /(?:\[(\d{1,3}:\d{2}(?::\d{2})?)\]\s*)?\[(You|Others)\]\s*([\s\S]*?)(?=(?:\[\d{1,3}:\d{2}(?::\d{2})?\]\s*)?\[(?:You|Others)\]|$)/g; + /(?:\[(\d{1,3}:\d{2}(?::\d{2})?)\]\s*)?\[([^\]]+)\]\s*([\s\S]*?)(?=(?:\[\d{1,3}:\d{2}(?::\d{2})?\]\s*)?\[[^\]]+\]|$)/g; export function parseTranscript(text: string, isDiarised: boolean): Segment[] { if (isDiarised) { @@ -24,7 +28,7 @@ export function parseTranscript(text: string, isDiarised: boolean): Segment[] { for (const m of text.matchAll(DIARISED_SEGMENT_RE)) { const body = m[3].trim(); if (!body) continue; - segments.push({ speaker: m[2] as 'You' | 'Others', text: body, timestamp: m[1] }); + segments.push({ speaker: m[2], text: body, timestamp: m[1] }); } return segments; } diff --git a/diarize-sidecar/Package.resolved b/diarize-sidecar/Package.resolved new file mode 100644 index 00000000..cefa925f --- /dev/null +++ b/diarize-sidecar/Package.resolved @@ -0,0 +1,14 @@ +{ + "pins" : [ + { + "identity" : "fluidaudio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/FluidInference/FluidAudio.git", + "state" : { + "revision" : "7f963cdc43ba89c5993654f1e138047d517a818d", + "version" : "0.15.2" + } + } + ], + "version" : 2 +} diff --git a/diarize-sidecar/Package.swift b/diarize-sidecar/Package.swift new file mode 100644 index 00000000..4fc79c2f --- /dev/null +++ b/diarize-sidecar/Package.swift @@ -0,0 +1,19 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "diarize-sidecar", + platforms: [.macOS(.v14)], + dependencies: [ + .package(url: "https://github.com/FluidInference/FluidAudio.git", from: "0.15.2"), + ], + targets: [ + .executableTarget( + name: "diarize-sidecar", + dependencies: [ + .product(name: "FluidAudio", package: "FluidAudio"), + ], + path: "Sources" + ), + ] +) \ No newline at end of file diff --git a/diarize-sidecar/Sources/main.swift b/diarize-sidecar/Sources/main.swift new file mode 100644 index 00000000..52e8b065 --- /dev/null +++ b/diarize-sidecar/Sources/main.swift @@ -0,0 +1,181 @@ +// diarize-sidecar — offline speaker diarization via FluidAudio's Sortformer. +// +// Usage: +// steno-diarize +// +// Sortformer has a fixed 4-speaker-slot architecture (SortformerConfig.numSpeakers +// is hardcoded to 4) — there is no speaker-count hint to pass, unlike the +// previous OfflineDiarizerManager-based version of this tool. +// +// Output (stdout): one JSON line on success +// [{"speakerId":"SPEAKER_0","start":0.0,"end":3.2}, ...] +// +// Exit 0 on success, 1 on failure (error written to stderr). +// stdout is unbuffered so the parent receives the line immediately. +// +// Audio loading avoids all CoreAudio file APIs (which fail with error +// 1954115647 when spawned from a PyInstaller bundle) by delegating to +// ffmpeg, which uses its own decoders entirely outside CoreAudio. This is +// also why we call SortformerDiarizer.processComplete(_:sourceSampleRate:) +// with raw samples rather than the processComplete(audioFileURL:) overload — +// that overload internally uses AudioConverter.resampleAudioFile, which +// calls AVAudioFile(forReading:) and would reintroduce the same crash. + +import Foundation +import FluidAudio + +setbuf(stdout, nil) + +// Segments shorter than this are spurious artifacts (an ~80ms noise-blip +// pattern observed empirically against real meeting audio), dropped before +// emitting rather than surfaced as a phantom speaker turn. +let minSegmentDurationSeconds: Float = 0.25 + +func fail(_ message: String) -> Never { + fputs("steno-diarize error: \(message)\n", stderr) + exit(1) +} + +guard CommandLine.arguments.count == 2 else { + fail("usage: steno-diarize ") +} + +let inputPath = CommandLine.arguments[1] + +guard FileManager.default.fileExists(atPath: inputPath) else { + fail("file not found: \(inputPath)") +} + +// Locate ffmpeg. The binary lives next to steno-diarize in the bundle; +// fall back to common Homebrew / system paths for terminal use. +func findFfmpeg() -> String? { + let execDir = URL(fileURLWithPath: CommandLine.arguments[0]) + .resolvingSymlinksInPath() + .deletingLastPathComponent() + .path + let candidates = [ + "\(execDir)/ffmpeg", + "/opt/homebrew/bin/ffmpeg", + "/usr/local/bin/ffmpeg", + "/usr/bin/ffmpeg", + ] + return candidates.first { FileManager.default.isExecutableFile(atPath: $0) } +} + +// Decode any audio format to 16 kHz mono Float32 using ffmpeg, writing +// output to a temp file. +// +// Uses terminationHandler + CheckedContinuation so the Task suspends +// instead of blocking a thread. ffmpeg stdin is redirected to /dev/null +// to prevent it from blocking on an inherited terminal. +func loadSamplesViaFfmpeg(path: String) async throws -> [Float] { + guard let ffmpegPath = findFfmpeg() else { + throw NSError(domain: "steno-diarize", code: 1, + userInfo: [NSLocalizedDescriptionKey: "ffmpeg not found"]) + } + let tmpURL = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("steno-diarize-\(UUID().uuidString).f32le") + defer { try? FileManager.default.removeItem(at: tmpURL) } + + let process = Process() + process.executableURL = URL(fileURLWithPath: ffmpegPath) + process.arguments = [ + "-loglevel", "error", + "-i", path, + "-ar", "16000", + "-ac", "1", + "-f", "f32le", + "-y", + tmpURL.path, + ] + // Redirect stdin to /dev/null so ffmpeg never blocks waiting for + // interactive input on an inherited terminal. + process.standardInput = FileHandle.nullDevice + // Redirect ffmpeg stderr to /dev/null. When steno-diarize is spawned by + // Python with capture_output=True, our own stderr is a pipe that Python + // only drains after we exit. ffmpeg inherits that pipe and fills the + // buffer with progress/profiling lines, causing it to block — which means + // terminationHandler never fires and steno-diarize hangs. /dev/null + // prevents the buffer fill; ffmpeg errors are still visible via exit status. + process.standardError = FileHandle(forWritingAtPath: "/dev/null") + + try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in + process.terminationHandler = { p in + if p.terminationStatus == 0 { + cont.resume() + } else { + cont.resume(throwing: NSError( + domain: "steno-diarize", code: Int(p.terminationStatus), + userInfo: [NSLocalizedDescriptionKey: "ffmpeg exited \(p.terminationStatus)"])) + } + } + do { + try process.run() + } catch { + cont.resume(throwing: error) + } + } + + let rawData = try Data(contentsOf: tmpURL, options: .mappedIfSafe) + + guard !rawData.isEmpty else { + throw NSError(domain: "steno-diarize", code: 3, + userInfo: [NSLocalizedDescriptionKey: "ffmpeg produced no output"]) + } + + // f32le on Apple Silicon (little-endian) — reinterpret bytes as Float directly. + return rawData.withUnsafeBytes { ptr in + Array(ptr.bindMemory(to: Float.self)) + } +} + +// Keep the main run loop alive so dispatch sources (including the +// process-exit source that drives terminationHandler) can fire normally. +// sema.wait() blocks the main thread, which prevents dispatch delivery +// and causes terminationHandler to never fire. +Task { + do { + // .cpuAndNeuralEngine forces genuine ANE execution — the default + // .all silently routes Sortformer to GPU instead (confirmed via + // Activity Monitor during evaluation). + let models = try await SortformerModels.loadFromHuggingFace( + config: .default, + computeUnits: .cpuAndNeuralEngine + ) + let diarizer = SortformerDiarizer() + diarizer.initialize(models: models) + + let samples = try await loadSamplesViaFfmpeg(path: inputPath) + let timeline = try diarizer.processComplete(samples, sourceSampleRate: nil) + + struct Segment: Encodable { + let speakerId: String + let start: Double + let end: Double + } + + let output = timeline.speakers.values + .flatMap { $0.finalizedSegments } + .filter { $0.duration >= minSegmentDurationSeconds } + .map { seg in + Segment( + speakerId: "SPEAKER_\(seg.speakerIndex)", + start: Double(seg.startTime), + end: Double(seg.endTime) + ) + } + .sorted { $0.start < $1.start } + + let encoded = try JSONEncoder().encode(output) + guard let line = String(data: encoded, encoding: .utf8) else { + exit(1) + } + print(line) + exit(0) + } catch { + fputs("steno-diarize error: \(error)\n", stderr) + exit(1) + } +} + +RunLoop.main.run() diff --git a/e2e/fixtures/say-stereo-wav.ts b/e2e/fixtures/say-stereo-wav.ts new file mode 100644 index 00000000..c1d4a3e3 --- /dev/null +++ b/e2e/fixtures/say-stereo-wav.ts @@ -0,0 +1,116 @@ +// Synthesizes real (non-silent, transcribable) speech into a stereo WAV +// using macOS's built-in `say` TTS, for e2e specs that need genuine ASR +// output rather than the sine-tone fixture in make-wav.js. A sine tone is +// non-speech and Parakeet/whisper.cpp both correctly return empty text for +// it (verified — that's what the plumbing-only @pipeline spec relies on), +// so it can never exercise anything downstream of real transcript segments, +// like the per-channel speaker-diarization labeling path. +// +// macOS-only: `say` doesn't exist on Windows/Linux. Callers must gate on +// isSayAvailable() (and process.platform) and skip loudly otherwise. +import { execFileSync } from 'child_process'; +import { mkdtempSync, readFileSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import path from 'path'; + +let sayAvailable: boolean | undefined; + +export function isSayAvailable(): boolean { + if (sayAvailable === undefined) { + try { + execFileSync('say', ['-v', '?'], { stdio: 'ignore' }); + sayAvailable = true; + } catch { + sayAvailable = false; + } + } + return sayAvailable; +} + +// `say -o out.wav --data-format=LEI16@16000` writes a real (non-canonical) +// WAV header — macOS prepends a JUNK chunk before `fmt `/`data`, so the +// data chunk is not at the fixed 44-byte offset make-wav.js assumes. +// Locate it by tag instead of guessing an offset. +function readPcm16Mono(filePath: string): Buffer { + const buf = readFileSync(filePath); + const dataTag = buf.indexOf('data', 12); + if (dataTag < 0) throw new Error(`no data chunk found in ${filePath}`); + const size = buf.readUInt32LE(dataTag + 4); + return buf.subarray(dataTag + 8, dataTag + 8 + size); +} + +function synthesize(text: string, destPath: string): Buffer { + execFileSync('say', ['-o', destPath, '--data-format=LEI16@16000', text]); + return readPcm16Mono(destPath); +} + +function silencePcm(seconds: number): Buffer { + return Buffer.alloc(Math.round(seconds * 16000) * 2); +} + +function writeStereoWav(destPath: string, leftPcm: Buffer, rightPcm: Buffer): void { + const frames = Math.max(leftPcm.length, rightPcm.length) / 2; + const bytesPerSample = 2; + const channels = 2; + const dataBytes = frames * channels * bytesPerSample; + const out = Buffer.alloc(44 + dataBytes); + + out.write('RIFF', 0); + out.writeUInt32LE(36 + dataBytes, 4); + out.write('WAVE', 8); + out.write('fmt ', 12); + out.writeUInt32LE(16, 16); + out.writeUInt16LE(1, 20); + out.writeUInt16LE(channels, 22); + out.writeUInt32LE(16000, 24); + out.writeUInt32LE(16000 * channels * bytesPerSample, 28); + out.writeUInt16LE(channels * bytesPerSample, 32); + out.writeUInt16LE(16, 34); + out.write('data', 36); + out.writeUInt32LE(dataBytes, 40); + + for (let i = 0; i < frames; i++) { + const l = i * 2 < leftPcm.length ? leftPcm.readInt16LE(i * 2) : 0; + const r = i * 2 < rightPcm.length ? rightPcm.readInt16LE(i * 2) : 0; + out.writeInt16LE(l, 44 + i * 4); + out.writeInt16LE(r, 44 + i * 4 + 2); + } + writeFileSync(destPath, out); +} + +export interface StereoSpeechResult { + /** Seconds into the mic channel's own timeline marking the midpoint of + * the silence gap between micUtteranceA and micUtteranceB — the point + * a fixture diarizer's two speaker-cluster boundary should sit at so + * each utterance's ASR sentence lands in a different cluster. */ + micBoundarySeconds: number; +} + +/** + * Builds a stereo WAV (left = mic, right = system — the layout + * transcribe_diarised splits) where the mic channel contains TWO short + * utterances separated by exactly 1s of real digital silence (so real ASR + * segments them into two distinct sentences with a known gap), and the + * system channel contains one short utterance. Real speaking-rate timing + * varies by voice/macOS version, so the returned boundary is computed from + * the ACTUALLY measured synthesized audio, not guessed. + */ +export function makeStereoSpeechWav( + destPath: string, + opts: { micUtteranceA: string; micUtteranceB: string; systemUtterance: string }, +): StereoSpeechResult { + const dir = mkdtempSync(path.join(tmpdir(), 'stenoai-e2e-say-')); + + const uttA = synthesize(opts.micUtteranceA, path.join(dir, 'mic_a.wav')); + const gapSeconds = 1.0; + const uttB = synthesize(opts.micUtteranceB, path.join(dir, 'mic_b.wav')); + const micPcm = Buffer.concat([uttA, silencePcm(gapSeconds), uttB]); + + const sysUtt = synthesize(opts.systemUtterance, path.join(dir, 'sys.wav')); + const sysPcm = Buffer.concat([sysUtt, silencePcm(Math.max(0, micPcm.length / 2 / 16000 - sysUtt.length / 2 / 16000))]); + + writeStereoWav(destPath, micPcm, sysPcm); + + const durA = uttA.length / 2 / 16000; + return { micBoundarySeconds: durA + gapSeconds / 2 }; +} diff --git a/e2e/specs/speaker-diarization.t2.spec.ts b/e2e/specs/speaker-diarization.t2.spec.ts new file mode 100644 index 00000000..6f2915a4 --- /dev/null +++ b/e2e/specs/speaker-diarization.t2.spec.ts @@ -0,0 +1,122 @@ +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import path from 'path'; +import { test, expect } from '../fixtures/electron'; +import { startMockOllama } from '../fixtures/mock-ollama'; +import { realUserDataDir, fileSig } from '../fixtures/real-user-data'; +import { selectEngine, isEngineModelReady, E2E_ENGINE } from '../fixtures/engine'; +import { killOllama } from '../fixtures/kill-ollama'; +import { isSayAvailable, makeStereoSpeechWav } from '../fixtures/say-stereo-wav'; + +/** + * T2 — real-backend per-channel speaker diarization (@pipeline). Drives a + * REAL synthesized-speech stereo recording through the app's streaming + * pipeline with STENOAI_DIARIZE_SIDECAR_PATH pointed at a fixture script + * (fixed 2-speaker JSON, independent of the real steno-diarize/Sortformer + * binary — that binary was validated directly against real recordings + * during development; this spec proves the PYTHON-SIDE integration: env + * override resolution, subprocess invocation + JSON parsing, per-channel + * cluster labeling, and cross-channel "Speaker N" numbering, end to end + * through the real Electron + Python app). + * + * Needs real (non-sine) speech so Parakeet/whisper.cpp produce real ASR + * sentence segments to diarize — synthesized via macOS's `say` (see + * fixtures/say-stereo-wav.ts), since there's no ASR mock seam in this + * codebase. macOS-only (both `say` and diarize-sidecar are macOS-only); + * skips loudly elsewhere or when the active engine's model isn't installed. + */ + +type StenoWindow = Window & { + stenoai: { + recording: { processSystemAudio: (p: string, name: string) => Promise<{ success?: boolean }> }; + }; +}; + +test('@pipeline synthesized two-speaker mic channel becomes You + Speaker 2', async ({ + launchApp, + userDataDir, +}) => { + test.setTimeout(180_000); + + test.skip(process.platform !== 'darwin', 'diarize-sidecar and the say TTS fixture are macOS-only'); + test.skip(!isSayAvailable(), 'macOS `say` TTS unavailable on this runner'); + + killOllama(); + const ollama = await startMockOllama(); + const realDirBefore = fileSig(realUserDataDir()); + + try { + // Build the real speech fixture BEFORE launching the app: mic channel = + // two utterances separated by a real acoustic gap, system channel = one + // short utterance. Pure Node/TTS — no app instance needed yet. + const recordingsDir = path.join(userDataDir, 'recordings'); + mkdirSync(recordingsDir, { recursive: true }); + const wavPath = path.join(recordingsDir, 'diarize.wav'); + const { micBoundarySeconds } = makeStereoSpeechWav(wavPath, { + micUtteranceA: 'Hello there, I hope you are having a wonderful and productive day today.', + micUtteranceB: 'Thanks a lot.', + systemUtterance: 'Thanks.', + }); + + // Fixture diarizer: always reports the SAME two speaker clusters, + // independent of which channel WAV it's actually pointed at. The tail + // cluster is deliberately shorter than the boundary so SPEAKER_0 always + // stays the dominant (most total speaking time) cluster on BOTH + // channels — sanity-checked below rather than assumed. + const tailSeconds = Math.max(1.5, Math.min(3.0, micBoundarySeconds * 0.6)); + expect(tailSeconds).toBeLessThan(micBoundarySeconds); + + const fixtureDir = mkdtempSync(path.join(tmpdir(), 'stenoai-e2e-diarize-')); + const scriptPath = path.join(fixtureDir, 'mock-steno-diarize.sh'); + writeFileSync( + scriptPath, + [ + '#!/usr/bin/env bash', + 'set -euo pipefail', + `echo '[{"speakerId":"SPEAKER_0","start":0,"end":${micBoundarySeconds}},` + + `{"speakerId":"SPEAKER_1","start":${micBoundarySeconds},"end":${micBoundarySeconds + tailSeconds}}]'`, + '', + ].join('\n'), + ); + chmodSync(scriptPath, 0o755); + + const { page } = await launchApp({ + env: { STENOAI_DIARIZE_SIDECAR_PATH: scriptPath }, + }); + await selectEngine(page); + + const modelReady = await isEngineModelReady(page); + if (!modelReady) { + // eslint-disable-next-line no-console + console.warn(`[t2:@pipeline] SKIPPED: ${E2E_ENGINE} model not installed on this runner.`); + test.info().annotations.push({ type: 'skip-reason', description: `${E2E_ENGINE} model not installed` }); + } + test.skip(!modelReady, `${E2E_ENGINE} model not installed`); + + const queued = await page.evaluate( + (p) => (window as StenoWindow).stenoai.recording.processSystemAudio(p, 'E2E Diarize'), + wavPath, + ); + expect(queued?.success).toBe(true); + + const transcriptPath = path.join(userDataDir, 'transcripts', 'diarize_transcript.txt'); + await expect + .poll(() => existsSync(transcriptPath), { timeout: 120_000, intervals: [1000] }) + .toBe(true); + + const transcript = readFileSync(transcriptPath, 'utf8'); + // Mic's dominant cluster (utterance A) keeps the legacy "You" label; + // its minority cluster (utterance B) becomes "Speaker 2". The system + // channel's single utterance lands in the shared dominant cluster too, + // so it keeps "Others" rather than becoming a second placeholder. + expect(transcript).toMatch(/\[You\]/); + expect(transcript).toMatch(/\[Speaker 2\]/); + expect(transcript).toMatch(/\[Others\]/); + expect(transcript).not.toMatch(/\[Speaker 3\]/); + + expect(fileSig(realUserDataDir())).toBe(realDirBefore); + } finally { + await ollama.close(); + killOllama(); + } +}); diff --git a/scripts/build-diarize-sidecar.sh b/scripts/build-diarize-sidecar.sh new file mode 100755 index 00000000..f8961df5 --- /dev/null +++ b/scripts/build-diarize-sidecar.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Build the Swift diarize-sidecar helper. +# +# Usage: scripts/build-diarize-sidecar.sh [arch] +# arch defaults to host arch (arm64 / x86_64). +# +# Requires Xcode Command Line Tools and an internet connection on first run +# (SPM fetches the FluidAudio dependency). +set -euo pipefail + +ARCH="${1:-$(uname -m)}" +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PKG="$ROOT/diarize-sidecar" +OUT="$ROOT/bin/steno-diarize" + +mkdir -p "$ROOT/bin" + +cd "$PKG" + +swift build \ + -c release \ + --arch "$ARCH" + +BUILD_BIN="$PKG/.build/${ARCH}-apple-macosx/release/diarize-sidecar" +cp "$BUILD_BIN" "$OUT" + +# Ad-hoc signature so the binary runs locally; CI re-signs with the Developer +# ID when packaging the .app bundle. +codesign --sign - "$OUT" 2>/dev/null || true + +file "$OUT" \ No newline at end of file diff --git a/src/_parakeet_mlx.py b/src/_parakeet_mlx.py index 5333ddda..61e0f5de 100644 --- a/src/_parakeet_mlx.py +++ b/src/_parakeet_mlx.py @@ -294,6 +294,22 @@ def _result_to_dict(result, language: Optional[str]) -> dict: "text": (getattr(s, "text", "") or "").strip(), "start": float(getattr(s, "start", 0.0) or 0.0), "end": float(getattr(s, "end", 0.0) or 0.0), + # Word-level timing, kept alongside the sentence text so the + # diarised path can split an abnormally long run-on sentence + # (Parakeet sometimes fails to break long unpunctuated speech + # into multiple sentences) across several diarizer turns + # instead of forcing the whole thing onto one speaker. Token + # text already carries its own leading-space markers — + # "".join(t["text"] for t in tokens) reconstructs the sentence + # exactly, no separator needed. + "tokens": [ + { + "text": getattr(t, "text", "") or "", + "start": float(getattr(t, "start", 0.0) or 0.0), + "end": float(getattr(t, "end", 0.0) or 0.0), + } + for t in (getattr(s, "tokens", None) or []) + ], } for s in sentences if (getattr(s, "text", "") or "").strip() diff --git a/src/transcriber.py b/src/transcriber.py index 76f8b1a7..6d2e8a6a 100644 --- a/src/transcriber.py +++ b/src/transcriber.py @@ -28,6 +28,7 @@ """ import inspect +import json import logging import math import os @@ -106,6 +107,20 @@ # recording doesn't pull all 30 min of int16 samples into Python lists. RMS_MAX_WINDOWS = 60 +# Acoustic per-channel speaker diarization (steno-diarize sidecar, macOS +# only). Merge gap for consecutive same-speaker diarizer segments — reduces +# diarization flicker and shrinks the gaps that cause boundary sentence +# misattribution in _assign_asr_segments_to_diar_segments. Matches the value +# validated against real meeting audio in the research playground. +STENO_DIARIZE_MERGE_GAP_S = 0.3 + +# Floor for the steno-diarize subprocess timeout. Measured runtime against +# real meeting-length audio is single-digit-to-tens-of-seconds; this leaves +# generous headroom under Electron's 8-minute inactivity watchdog while +# still bounding a runaway on pathological input. Scaled up by duration for +# long recordings, same pattern as _diarised_split_timeout. +STENO_DIARIZE_TIMEOUT_FLOOR_S = 120 + # Sentinel text substituted when transcription produces no usable output # (genuine silence or all-hallucination). Callers compare against this to # distinguish "really nothing was said" from a real (possibly short) @@ -164,6 +179,51 @@ def _resolve_ffmpeg() -> Optional[str]: return None +# Resolve the bundled steno-diarize binary (macOS-only Swift/CoreML speaker +# diarization sidecar, built by scripts/build-diarize-sidecar.sh). Mirrors +# _resolve_ffmpeg()'s bundle-then-fallback lookup, but checks executability +# instead of running a cheap probe invocation — there's no equivalent to +# `ffmpeg -version` for this binary, so a bad binary still fails safely at +# actual call time via _run_steno_diarize's blanket failure handling. +_STENO_DIARIZE_PATH_CACHE: Optional[str] = None +_STENO_DIARIZE_PATH_LOCK = threading.Lock() + + +def _resolve_steno_diarize() -> Optional[str]: + global _STENO_DIARIZE_PATH_CACHE + if sys.platform != "darwin": + return None + if _STENO_DIARIZE_PATH_CACHE is not None: + return _STENO_DIARIZE_PATH_CACHE + with _STENO_DIARIZE_PATH_LOCK: + if _STENO_DIARIZE_PATH_CACHE is not None: + return _STENO_DIARIZE_PATH_CACHE + candidates: list[str] = [] + # Same spirit as the STENO_DIARIZE_* env knobs on the Swift side — + # lets a T2 e2e spec point at a fixture binary without touching the + # real bundle resolution. + override = os.environ.get("STENOAI_DIARIZE_SIDECAR_PATH") + if override: + candidates.append(override) + if getattr(sys, 'frozen', False): + exe_dir = Path(sys.executable).parent + candidates.extend([ + str(exe_dir / 'steno-diarize'), + str(exe_dir / '_internal' / 'steno-diarize'), + ]) + else: + # Dev: repo-root bin/, built by scripts/build-diarize-sidecar.sh + repo_root = Path(__file__).resolve().parent.parent + candidates.append(str(repo_root / 'bin' / 'steno-diarize')) + for cand in candidates: + if os.access(cand, os.X_OK): + _STENO_DIARIZE_PATH_CACHE = cand + logger.info(f"steno-diarize resolved at: {cand}") + return cand + logger.info("steno-diarize not found; per-channel speaker labeling falls back to legacy You/Others") + return None + + def _audio_filter_chain() -> str: """The ffmpeg ``-af`` chain applied to mono audio before transcription.""" return f"highpass=f={AUDIO_HIGHPASS_HZ},loudnorm={AUDIO_LOUDNORM}" @@ -411,6 +471,281 @@ def _drop_per_segment_bleed( return kept_mic, kept_sys +# --------------------------------------------------------------------------- +# Per-channel acoustic speaker diarization (steno-diarize sidecar, macOS +# only). Ported from the research playground (scripts/diarize_playground.py) +# as dict-based helpers — that script's own docstring states it's a research +# tool that doesn't touch src/, and its functions use attribute access on +# AlignedSentence objects while our real ASR segments are dicts. +# --------------------------------------------------------------------------- + +def _merge_close_diar_segments(segments: list[dict], max_gap: float) -> list[dict]: + """Merge consecutive same-speaker diarizer segments separated by a gap + smaller than max_gap. Segments must already be sorted by start time. + Reduces diarization flicker and shrinks the gaps that cause boundary + sentence misattribution in _assign_asr_segments_to_diar_segments.""" + if not segments: + return [] + merged = [dict(segments[0])] + for segment in segments[1:]: + last = merged[-1] + if segment["speaker"] == last["speaker"] and segment["start"] - last["end"] <= max_gap: + last["end"] = segment["end"] + else: + merged.append(dict(segment)) + return merged + + +# A single mic capturing two people is a much harder acoustic problem than a +# clean mic-vs-system-audio split — the diarizer's own turn boundaries can be +# genuinely noisy/overlapping there (observed empirically: alternating and +# even overlapping SPEAKER_0/SPEAKER_1 segments across a real back-and-forth). +# Parakeet sometimes fails to break a long run of natural speech (no strong +# terminal punctuation) into separate sentences, producing a single sentence +# that spans many real diarizer turns. Assigning that whole sentence to +# whichever one diarizer segment its midpoint happens to land in then forces +# an entire multi-turn exchange onto one speaker. A sentence at or above this +# duration gets word-level splitting instead (see _find_nearest_diar_segment) +# whenever it actually overlaps more than one distinct diarizer speaker. +LONG_SENTENCE_SPLIT_THRESHOLD_S = 5.0 + + +def _find_nearest_diar_segment(start: float, end: float, diar_segments: list[dict]) -> Optional[int]: + """Index of the diar segment containing [start, end]'s midpoint, or the + nearest one by boundary distance if the midpoint falls in an uncovered + gap. Returns None only when diar_segments is empty.""" + midpoint = (start + end) / 2 + best_i, best_dist = None, float("inf") + for i, segment in enumerate(diar_segments): + if segment["start"] <= midpoint <= segment["end"]: + return i + dist = ( + segment["start"] - midpoint + if midpoint < segment["start"] + else midpoint - segment["end"] + ) + if dist < best_dist: + best_i, best_dist = i, dist + return best_i + + +def _assign_asr_segments_to_diar_segments(asr_segments: list[dict], diar_segments: list[dict]) -> None: + """Assign each ASR (Parakeet) sentence to the diarizer segment(s) it + belongs to. + + Normal case (sentence fits inside one real diarizer turn): assign the + whole sentence as one block to the nearest/containing diar segment. + Sentence granularity (not word/token) is deliberate here — Parakeet + already does its own sentence segmentation, so this never tears a word + or clause in half for the common case. + + Long-sentence case: if a sentence runs at or above + LONG_SENTENCE_SPLIT_THRESHOLD_S AND genuinely overlaps more than one + distinct diarizer speaker, assigning it as one block would force an + entire multi-turn exchange onto whichever speaker the midpoint happened + to land on. Fall back to word-level assignment instead: each word goes + to its own nearest diar segment (using its own timing, from + segment["tokens"] — see src/_parakeet_mlx.py), and runs of consecutive + words landing in the same diar segment are joined back together. This + needs word timing to exist at all (`tokens`); if it doesn't (e.g. the + whisper.cpp backend, or an older cached result), the sentence falls back + to whole-block assignment like the normal case. + + Mutates diar_segments in place, attaching joined text as segment["text"].""" + for segment in diar_segments: + segment["text"] = "" + if not diar_segments: + return + + texts_by_segment: dict[int, list[str]] = {i: [] for i in range(len(diar_segments))} + for asr_segment in asr_segments: + text = (asr_segment.get("text") or "").strip() + if not text: + continue + start = float(asr_segment.get("start") or 0.0) + end = float(asr_segment.get("end") or start) + tokens = asr_segment.get("tokens") or [] + duration = end - start + + multi_speaker_span = False + if duration >= LONG_SENTENCE_SPLIT_THRESHOLD_S and tokens: + overlapping_speakers = { + seg["speaker"] for seg in diar_segments + if seg["start"] < end and seg["end"] > start + } + multi_speaker_span = len(overlapping_speakers) > 1 + + if multi_speaker_span: + run_index: Optional[int] = None + run_words: list[str] = [] + for token in tokens: + token_text = token.get("text") or "" + if not token_text.strip(): + continue + t_start = float(token.get("start") or 0.0) + t_end = float(token.get("end") or t_start) + idx = _find_nearest_diar_segment(t_start, t_end, diar_segments) + if idx is None: + continue + if run_index is not None and idx != run_index: + texts_by_segment[run_index].append("".join(run_words)) + run_words = [] + run_index = idx + run_words.append(token_text) + if run_index is not None and run_words: + texts_by_segment[run_index].append("".join(run_words)) + else: + idx = _find_nearest_diar_segment(start, end, diar_segments) + if idx is not None: + texts_by_segment[idx].append(text) + + for i, segment in enumerate(diar_segments): + segment["text"] = " ".join(t.strip() for t in texts_by_segment[i] if t.strip()).strip() + + +def _run_steno_diarize(channel_path: Path, timeout: int) -> Optional[list[dict]]: + """Run the steno-diarize sidecar on a single mono channel WAV. + + Returns merged diarizer segments (each ``{"start", "end", "speaker"}``) + on success. Returns None on ANY failure — missing binary, timeout, + non-zero exit, or unparseable output — so callers always have a safe + fallback to legacy channel-only labeling and this can never fail a + meeting. + """ + binary = _resolve_steno_diarize() + if not binary: + return None + try: + result = subprocess.run( + [binary, str(channel_path)], + capture_output=True, timeout=timeout, + ) + if result.returncode != 0: + logger.warning( + "steno-diarize exited %s: %s", + result.returncode, result.stderr.decode(errors="replace")[:300], + ) + return None + stdout = result.stdout.decode(errors="replace") + # A known FluidAudio/CoreML warning ("E5RT encountered an STL + # exception... key not found") can print directly to stdout ahead + # of the JSON payload — skip to the first '[' rather than assuming + # stdout is pure JSON. + bracket = stdout.find("[") + if bracket < 0: + logger.warning("steno-diarize produced no JSON output") + return None + raw_segments = json.loads(stdout[bracket:]) + except (subprocess.TimeoutExpired, OSError, ValueError) as e: + logger.warning("steno-diarize failed: %s", e) + return None + + segments = sorted( + ( + { + "start": float(s["start"]), + "end": float(s["end"]), + "speaker": str(s["speakerId"]), + } + for s in raw_segments + ), + key=lambda s: s["start"], + ) + return _merge_close_diar_segments(segments, STENO_DIARIZE_MERGE_GAP_S) + + +def _cluster_channel_labels(diar_segments: list[dict], legacy_label: str) -> Optional[dict[str, str]]: + """Map each diarizer speaker id in diar_segments to either the channel's + legacy label (the cluster with the most total speaking time) or a + placeholder key for every other cluster, later resolved to "Speaker N" + by _resolve_speaker_placeholders. + + Returns None when diar_segments contains a single (or zero) distinct + speaker — the byte-identical-to-legacy fast path, since there's nothing + to disambiguate. + """ + speaker_ids = {s["speaker"] for s in diar_segments} + if len(speaker_ids) <= 1: + return None + totals: dict[str, float] = {sid: 0.0 for sid in speaker_ids} + for s in diar_segments: + totals[s["speaker"]] += s["end"] - s["start"] + dominant = max(totals, key=totals.get) + return { + sid: (legacy_label if sid == dominant else f"__diar__{legacy_label}__{sid}") + for sid in speaker_ids + } + + +def _tag_channel_segments( + asr_segments: list[dict], + channel_path: Optional[Path], + duration_seconds: Optional[float], + legacy_label: str, +) -> list[tuple[float, str, str]]: + """Build (start, label, text) tuples for one channel's ASR segments. + + Tries acoustic diarization first: if the steno-diarize sidecar succeeds + and finds more than one real speaker cluster, turns are built from the + diarizer's own segment boundaries (with ASR sentences reassigned into + them via _assign_asr_segments_to_diar_segments). The cluster with the + most total speaking time keeps the channel's legacy label + ("You"/"Others"); every other cluster gets a placeholder resolved to + "Speaker N" later. ANY failure — missing binary, timeout, bad JSON, or + a single-cluster result — falls back to the byte-identical legacy + behaviour of labeling every ASR segment with legacy_label. + """ + if not asr_segments: + return [] + + if channel_path is not None: + timeout = max(STENO_DIARIZE_TIMEOUT_FLOOR_S, int(duration_seconds or 0)) + diar_segments = _run_steno_diarize(channel_path, timeout) + if diar_segments: + cluster_labels = _cluster_channel_labels(diar_segments, legacy_label) + if cluster_labels: + _assign_asr_segments_to_diar_segments(asr_segments, diar_segments) + diar_tagged = [] + for segment in diar_segments: + text = (segment.get("text") or "").strip() + if text: + diar_tagged.append((segment["start"], cluster_labels[segment["speaker"]], text)) + if diar_tagged: + return diar_tagged + + legacy_tagged: list[tuple[float, str, str]] = [] + for s in asr_segments: + text = (s.get("text") or "").strip() + if text: + legacy_tagged.append((float(s.get("start") or 0.0), legacy_label, text)) + return legacy_tagged + + +def _resolve_speaker_placeholders( + tagged: list[tuple[float, str, str]], +) -> list[tuple[float, str, str]]: + """Replace placeholder cluster labels (see _cluster_channel_labels) with + "Speaker N", numbered by first chronological appearance across BOTH + channels merged and time-sorted — so a reader sees new speakers + introduced as 2, 3, 4... regardless of which channel they came from. + + No cross-channel identity matching: a mic placeholder and a system + placeholder are always treated as different people — telling them + apart would need voiceprint embeddings, out of scope here. + """ + numbering: dict[str, str] = {} + next_n = 2 + resolved: list[tuple[float, str, str]] = [] + for start, label, text in tagged: + if label.startswith("__diar__"): + if label not in numbering: + numbering[label] = f"Speaker {next_n}" + next_n += 1 + label = numbering[label] + resolved.append((start, label, text)) + return resolved + + # Try Parakeet first (preferred — same engine as live, arm64 Macs only). try: from src.parakeet import transcribe_file as _parakeet_transcribe_file @@ -1247,17 +1582,16 @@ def transcribe_diarised(self, audio_filepath: Path, language: str = "en") -> Opt # Chronologically interleave segments from both channels and # collapse runs of consecutive same-speaker segments into a - # single labelled turn. + # single labelled turn. Each channel is first run through + # acoustic diarization (steno-diarize, macOS only) to split + # multiple speakers sharing one side of the call; any failure + # or a single-cluster result falls back to the legacy + # "You"/"Others" channel-only labeling. tagged: list[tuple[float, str, str]] = [] - for s in mic_segments: - text = (s.get("text") or "").strip() - if text: - tagged.append((float(s.get("start") or 0.0), "You", text)) - for s in system_segments: - text = (s.get("text") or "").strip() - if text: - tagged.append((float(s.get("start") or 0.0), "Others", text)) + tagged.extend(_tag_channel_segments(mic_segments, mic_path, duration, "You")) + tagged.extend(_tag_channel_segments(system_segments, system_path, duration, "Others")) tagged.sort(key=lambda t: t[0]) + tagged = _resolve_speaker_placeholders(tagged) # Each turn carries the start offset of its FIRST segment so the # diarised transcript can be timestamped. Only diarised_text is @@ -1277,7 +1611,19 @@ def transcribe_diarised(self, audio_filepath: Path, language: str = "en") -> Opt plain_parts = [' '.join(parts) for _start, _speaker, parts in turns] plain_text = "\n\n".join(plain_parts) if plain_parts else SILENCE_SENTINEL - is_diarised = bool(mic_segments) and bool(system_segments) + # Diarised means "more than one voice is distinguishable in this + # transcript" — NOT "both channels had content". The old + # bool(mic_segments) and bool(system_segments) check predates + # per-channel acoustic diarization and silently discarded the + # whole labelled transcript whenever one channel was empty (e.g. + # an in-person conversation with no system audio playing at + # all), even when _tag_channel_segments had already split the + # OTHER channel into "You" + "Speaker 2". Counting distinct + # labels covers both the classic two-channel case (You + Others) + # and the new single-channel multi-speaker case correctly, and + # still suppresses labelling a genuine one-voice monologue. + distinct_labels = {speaker for _start, speaker, _parts in turns} + is_diarised = len(distinct_labels) > 1 if is_diarised: labelled_parts = [ f"[{_format_timestamp(start)}] [{speaker}] {' '.join(parts)}" diff --git a/stenoai.spec b/stenoai.spec index f8c5b6a0..7233eec3 100644 --- a/stenoai.spec +++ b/stenoai.spec @@ -257,6 +257,15 @@ if os.path.exists(ollama_bin_dir): if base in ('ffmpeg', 'ffmpeg.exe'): # Put ffmpeg at the root of the bundle for easy PATH access binaries.append((filepath, '.')) + elif base == 'steno-diarize' and _IS_DARWIN: + # macOS-only Swift/CoreML diarization sidecar (built by + # scripts/build-diarize-sidecar.sh). Root-level like ffmpeg + # since src/transcriber.py resolves it the same way. Gated + # on _IS_DARWIN + os.path.exists above so a checkout that + # hasn't run the Swift build (or a non-macOS platform) just + # skips it — src/transcriber.py falls back to legacy + # channel-only labeling when the binary is missing. + binaries.append((filepath, '.')) elif _IS_DARWIN: # COLLECT DATA TOC 3-tuple: (dest_path_including_filename, # abs_src_path, 'DATA'). Everything lives under ollama/, diff --git a/tests/test_transcriber_diarisation.py b/tests/test_transcriber_diarisation.py index ba86bd39..680f159b 100644 --- a/tests/test_transcriber_diarisation.py +++ b/tests/test_transcriber_diarisation.py @@ -9,23 +9,32 @@ so a recording where speech starts mid-stream isn't classified as silent. """ +import json import math import struct +import subprocess import tempfile import unittest import wave from pathlib import Path -from unittest.mock import Mock +from unittest.mock import Mock, patch from src.transcriber import ( BLEED_JACCARD_THRESHOLD, DIARISED_SPLIT_TIMEOUT_S, MIN_RMS_THRESHOLD, + STENO_DIARIZE_MERGE_GAP_S, WhisperTranscriber, + _assign_asr_segments_to_diar_segments, + _cluster_channel_labels, _diarised_split_timeout, _format_timestamp, + _merge_close_diar_segments, _parse_channels_from_ffmpeg_stderr, _parse_duration_from_ffmpeg_stderr, + _resolve_speaker_placeholders, + _run_steno_diarize, + _tag_channel_segments, _token_jaccard, ) @@ -80,8 +89,16 @@ def setUp(self): return_value=(self.mic_path, self.system_path, 3.0) ) self.transcriber._check_rms_energy = Mock(return_value=True) + # These are pinned-contract tests for the legacy You/Others-only + # behaviour, so the sidecar must be explicitly forced off — without + # this they'd pass by accident on a clean checkout (no binary) and + # break the moment a contributor has one built locally (see + # TranscribeDiarisedMultiSpeakerTests for the sidecar-present cases). + self._diar_patcher = patch("src.transcriber._run_steno_diarize", return_value=None) + self._diar_patcher.start() def tearDown(self): + self._diar_patcher.stop() self._tmp.cleanup() def test_interleaves_diarised_segments_with_timestamps(self): @@ -112,6 +129,158 @@ def test_single_source_is_not_timestamped_or_diarised(self): self.assertIsNone(result["diarised_text"]) +class TranscribeDiarisedMultiSpeakerTests(unittest.TestCase): + """Acoustic per-channel diarization (steno-diarize sidecar) layered on + top of the legacy You/Others channel split. Mocks _run_steno_diarize + directly (module-level, not an instance method) since transcribe_diarised + calls it as a free function via _tag_channel_segments.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + d = Path(self._tmp.name) + self.audio_path = d / "source.wav" + self.mic_path = d / "mic.wav" + self.system_path = d / "system.wav" + for p in (self.audio_path, self.mic_path, self.system_path): + p.write_bytes(b"stub") + self.transcriber = WhisperTranscriber.__new__(WhisperTranscriber) + self.transcriber.backend = "parakeet" + self.transcriber._split_stereo_to_channels = Mock( + return_value=(self.mic_path, self.system_path, 10.0) + ) + self.transcriber._check_rms_energy = Mock(return_value=True) + + def tearDown(self): + self._tmp.cleanup() + + def test_two_speakers_on_mic_channel_become_you_and_speaker_two(self): + # Mic channel has two acoustic clusters (SPEAKER_0 dominant at 5s + # total, SPEAKER_1 minor at 2s total); system channel has a single + # trivial cluster so is_diarised (which requires both channels to + # contribute) stays True. + mic_diar = [ + {"start": 0.0, "end": 2.0, "speaker": "SPEAKER_0"}, + {"start": 2.5, "end": 4.5, "speaker": "SPEAKER_1"}, + {"start": 5.0, "end": 8.0, "speaker": "SPEAKER_0"}, + ] + system_diar = [{"start": 9.0, "end": 9.5, "speaker": "SPEAKER_0"}] + with patch("src.transcriber._run_steno_diarize", side_effect=[mic_diar, system_diar]): + self.transcriber.transcribe_audio = Mock(side_effect=[ + {"text": "Hi there. Not bad. Great.", "segments": [ + {"text": "Hi there.", "start": 0.5, "end": 1.5}, + {"text": "Not bad.", "start": 3.0, "end": 3.8}, + {"text": "Great.", "start": 6.0, "end": 6.8}, + ]}, + {"text": "Ok.", "segments": [{"text": "Ok.", "start": 9.2, "end": 9.4}]}, + ]) + result = self.transcriber.transcribe_diarised(self.audio_path) + self.assertTrue(result["is_diarised"]) + self.assertIn("[You] Hi there.", result["diarised_text"]) + self.assertIn("[Speaker 2] Not bad.", result["diarised_text"]) + self.assertIn("[You] Great.", result["diarised_text"]) + self.assertIn("[Others] Ok.", result["diarised_text"]) + + def test_mic_only_multi_speaker_is_diarised_even_with_system_silent(self): + # Regression: an in-person conversation with no computer audio + # playing at all (system channel genuinely silent, not bled/dropped) + # must still produce a labelled transcript from the mic channel's + # own acoustic diarization. The old is_diarised computation + # (bool(mic_segments) and bool(system_segments)) discarded the + # whole labelled transcript whenever system was empty, even though + # _tag_channel_segments had already split mic into You + Speaker 2. + mic_diar = [ + {"start": 0.0, "end": 2.0, "speaker": "SPEAKER_0"}, + {"start": 2.5, "end": 4.5, "speaker": "SPEAKER_1"}, + {"start": 5.0, "end": 8.0, "speaker": "SPEAKER_0"}, + ] + with patch("src.transcriber._run_steno_diarize", return_value=mic_diar): + self.transcriber._check_rms_energy = Mock(side_effect=[True, False]) + self.transcriber.transcribe_audio = Mock(return_value={ + "text": "Hi there. Not bad. Great.", "segments": [ + {"text": "Hi there.", "start": 0.5, "end": 1.5}, + {"text": "Not bad.", "start": 3.0, "end": 3.8}, + {"text": "Great.", "start": 6.0, "end": 6.8}, + ], + }) + result = self.transcriber.transcribe_diarised(self.audio_path) + self.assertTrue(result["is_diarised"]) + self.assertIsNotNone(result["diarised_text"]) + self.assertIn("[You] Hi there.", result["diarised_text"]) + self.assertIn("[Speaker 2] Not bad.", result["diarised_text"]) + self.assertIn("[You] Great.", result["diarised_text"]) + + def test_speaker_numbering_is_chronological_across_both_channels(self): + # System's placeholder speaker turns up chronologically before + # mic's placeholder speaker, so it must be numbered "Speaker 2" + # even though the mic channel's diarization result is processed + # first in transcribe_diarised. Dominant/minor durations are + # clearly unequal (4s vs 1s) so cluster dominance is unambiguous. + system_diar = [ + {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_1"}, # minor, first chronologically + {"start": 10.0, "end": 14.0, "speaker": "SPEAKER_0"}, # dominant -> "Others" + ] + mic_diar = [ + {"start": 15.0, "end": 16.0, "speaker": "SPEAKER_1"}, # minor + {"start": 20.0, "end": 24.0, "speaker": "SPEAKER_0"}, # dominant -> "You" + ] + with patch("src.transcriber._run_steno_diarize", side_effect=[mic_diar, system_diar]): + self.transcriber.transcribe_audio = Mock(side_effect=[ + {"text": "Mic minor. Mic dominant.", "segments": [ + {"text": "Mic minor.", "start": 15.2, "end": 15.8}, + {"text": "Mic dominant.", "start": 21.0, "end": 21.5}, + ]}, + {"text": "Sys minor. Sys dominant.", "segments": [ + {"text": "Sys minor.", "start": 0.2, "end": 0.8}, + {"text": "Sys dominant.", "start": 11.0, "end": 11.5}, + ]}, + ]) + result = self.transcriber.transcribe_diarised(self.audio_path) + # System's minority cluster appears first chronologically (t=0.0) + # so it gets "Speaker 2"; mic's minority cluster (t=15.0) gets + # "Speaker 3" even though mic is diarized first in the pipeline. + # Each turn is timestamped by the diarizer's own segment boundary + # (not the ASR sentence start) — see _tag_channel_segments. + self.assertEqual( + result["diarised_text"], + "[00:00] [Speaker 2] Sys minor." + "\n\n[00:10] [Others] Sys dominant." + "\n\n[00:15] [Speaker 3] Mic minor." + "\n\n[00:20] [You] Mic dominant.", + ) + + def test_single_cluster_per_channel_is_byte_identical_to_legacy(self): + # Sidecar runs successfully but finds only one speaker per channel — + # must fall back to plain You/Others, not "Speaker 1" everywhere. + mic_diar = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}] + system_diar = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}] + with patch("src.transcriber._run_steno_diarize", side_effect=[mic_diar, system_diar]): + self.transcriber.transcribe_audio = Mock(side_effect=[ + {"text": "Hello.", "segments": [{"text": "Hello.", "start": 1.0, "end": 1.5}]}, + {"text": "Reply.", "segments": [{"text": "Reply.", "start": 2.0, "end": 2.5}]}, + ]) + result = self.transcriber.transcribe_diarised(self.audio_path) + self.assertEqual( + result["diarised_text"], + "[00:01] [You] Hello.\n\n[00:02] [Others] Reply.", + ) + + def test_sidecar_failure_falls_back_without_failing_meeting(self): + # Missing binary / timeout / bad JSON all surface as None from + # _run_steno_diarize — transcribe_diarised must never fail the + # meeting because of it. + with patch("src.transcriber._run_steno_diarize", return_value=None): + self.transcriber.transcribe_audio = Mock(side_effect=[ + {"text": "Hello.", "segments": [{"text": "Hello.", "start": 1.0, "end": 1.5}]}, + {"text": "Reply.", "segments": [{"text": "Reply.", "start": 2.0, "end": 2.5}]}, + ]) + result = self.transcriber.transcribe_diarised(self.audio_path) + self.assertNotIn("transcription_failed", result) + self.assertEqual( + result["diarised_text"], + "[00:01] [You] Hello.\n\n[00:02] [Others] Reply.", + ) + + class TokenJaccardTests(unittest.TestCase): def test_identical_strings_score_one(self): self.assertEqual(_token_jaccard("hello world", "hello world"), 1.0) @@ -343,5 +512,256 @@ def test_returns_int(self): self.assertIsInstance(_diarised_split_timeout(1234.5), int) +class MergeCloseDiarSegmentsTests(unittest.TestCase): + def test_empty_input_returns_empty(self): + self.assertEqual(_merge_close_diar_segments([], 0.3), []) + + def test_merges_same_speaker_within_gap(self): + segments = [ + {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"}, + {"start": 1.2, "end": 2.0, "speaker": "SPEAKER_0"}, + ] + merged = _merge_close_diar_segments(segments, 0.3) + self.assertEqual(merged, [{"start": 0.0, "end": 2.0, "speaker": "SPEAKER_0"}]) + + def test_does_not_merge_across_gap_larger_than_threshold(self): + segments = [ + {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"}, + {"start": 2.0, "end": 3.0, "speaker": "SPEAKER_0"}, + ] + merged = _merge_close_diar_segments(segments, 0.3) + self.assertEqual(len(merged), 2) + + def test_does_not_merge_different_speakers(self): + segments = [ + {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"}, + {"start": 1.1, "end": 2.0, "speaker": "SPEAKER_1"}, + ] + merged = _merge_close_diar_segments(segments, 0.3) + self.assertEqual(len(merged), 2) + + def test_does_not_mutate_input(self): + segments = [{"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"}] + _merge_close_diar_segments(segments, STENO_DIARIZE_MERGE_GAP_S) + self.assertEqual(segments, [{"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"}]) + + +class AssignAsrSegmentsToDiarSegmentsTests(unittest.TestCase): + def test_empty_diar_segments_is_a_no_op(self): + diar_segments = [] + _assign_asr_segments_to_diar_segments( + [{"text": "Hello", "start": 0.0, "end": 1.0}], diar_segments + ) + self.assertEqual(diar_segments, []) + + def test_assigns_sentence_within_segment_bounds(self): + diar_segments = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}] + _assign_asr_segments_to_diar_segments( + [{"text": "Hello there", "start": 1.0, "end": 2.0}], diar_segments + ) + self.assertEqual(diar_segments[0]["text"], "Hello there") + + def test_assigns_multiple_sentences_to_nearest_segment(self): + diar_segments = [ + {"start": 0.0, "end": 2.0, "speaker": "SPEAKER_0"}, + {"start": 3.0, "end": 5.0, "speaker": "SPEAKER_1"}, + ] + _assign_asr_segments_to_diar_segments( + [ + {"text": "First.", "start": 0.5, "end": 1.0}, + {"text": "Second.", "start": 3.5, "end": 4.0}, + # Falls in the gap between segments (2.0-3.0) but its + # midpoint (2.6) is closer to the second segment's start. + {"text": "Gap.", "start": 2.4, "end": 2.8}, + ], + diar_segments, + ) + self.assertEqual(diar_segments[0]["text"], "First.") + self.assertEqual(diar_segments[1]["text"], "Second. Gap.") + + def test_blank_sentences_are_skipped(self): + diar_segments = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}] + _assign_asr_segments_to_diar_segments( + [{"text": " ", "start": 1.0, "end": 2.0}], diar_segments + ) + self.assertEqual(diar_segments[0]["text"], "") + + def test_long_sentence_spanning_multiple_speakers_splits_by_word(self): + # Regression for a real observed failure: a single mic capturing two + # people can produce one long Parakeet "sentence" (no punctuation + # break) that actually spans a genuine back-and-forth. Whole-block + # midpoint assignment forced the entire run onto one speaker; + # word-level splitting should recover the real turn boundaries. + diar_segments = [ + {"start": 0.0, "end": 3.0, "speaker": "SPEAKER_0"}, + {"start": 3.0, "end": 6.0, "speaker": "SPEAKER_1"}, + ] + tokens = [ + {"text": " one", "start": 0.5, "end": 1.0}, + {"text": " two", "start": 1.0, "end": 1.5}, + {"text": " three", "start": 4.5, "end": 5.0}, + {"text": " four", "start": 5.0, "end": 5.5}, + ] + _assign_asr_segments_to_diar_segments( + [{"text": "one two three four", "start": 0.5, "end": 5.5, "tokens": tokens}], + diar_segments, + ) + self.assertEqual(diar_segments[0]["text"], "one two") + self.assertEqual(diar_segments[1]["text"], "three four") + + def test_long_sentence_within_single_speaker_is_not_split(self): + # Long duration alone isn't enough to trigger splitting — the + # diarizer segments it overlaps must belong to more than one + # distinct speaker. Fragmented same-speaker segments (diarizer + # flicker) should still be treated as one block. + diar_segments = [ + {"start": 0.0, "end": 3.0, "speaker": "SPEAKER_0"}, + {"start": 3.0, "end": 6.0, "speaker": "SPEAKER_0"}, + ] + tokens = [ + {"text": " one", "start": 0.5, "end": 1.0}, + {"text": " two", "start": 5.0, "end": 5.5}, + ] + _assign_asr_segments_to_diar_segments( + [{"text": "one two", "start": 0.5, "end": 5.5, "tokens": tokens}], + diar_segments, + ) + self.assertEqual(diar_segments[0]["text"], "one two") + self.assertEqual(diar_segments[1]["text"], "") + + def test_short_sentence_not_split_even_across_speakers(self): + # Below LONG_SENTENCE_SPLIT_THRESHOLD_S — must stay whole-block + # (matching the historical short-sentence behaviour) even though it + # technically overlaps two different speakers, to avoid tearing + # short utterances apart on noisy diarizer boundaries. + diar_segments = [ + {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"}, + {"start": 1.0, "end": 2.0, "speaker": "SPEAKER_1"}, + ] + tokens = [ + {"text": " hi", "start": 0.8, "end": 1.0}, + {"text": " there", "start": 1.0, "end": 1.2}, + ] + _assign_asr_segments_to_diar_segments( + [{"text": "hi there", "start": 0.8, "end": 1.2, "tokens": tokens}], + diar_segments, + ) + texts = [d["text"] for d in diar_segments] + self.assertEqual(texts.count("hi there"), 1) + + def test_long_sentence_without_tokens_falls_back_to_whole_block(self): + # No word-level timing (e.g. the whisper.cpp backend never + # populates "tokens") must never crash — falls back to the same + # whole-block nearest assignment as a normal sentence. + diar_segments = [ + {"start": 0.0, "end": 3.0, "speaker": "SPEAKER_0"}, + {"start": 3.0, "end": 6.0, "speaker": "SPEAKER_1"}, + ] + _assign_asr_segments_to_diar_segments( + [{"text": "one two three four", "start": 0.5, "end": 5.5}], + diar_segments, + ) + texts = [d["text"] for d in diar_segments] + self.assertEqual(texts.count("one two three four"), 1) + + +class ClusterChannelLabelsTests(unittest.TestCase): + def test_single_speaker_returns_none(self): + segments = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}] + self.assertIsNone(_cluster_channel_labels(segments, "You")) + + def test_empty_segments_returns_none(self): + self.assertIsNone(_cluster_channel_labels([], "You")) + + def test_dominant_speaker_by_total_duration_keeps_legacy_label(self): + segments = [ + {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"}, # 1s + {"start": 1.0, "end": 6.0, "speaker": "SPEAKER_1"}, # 5s, dominant + ] + labels = _cluster_channel_labels(segments, "You") + self.assertEqual(labels["SPEAKER_1"], "You") + self.assertEqual(labels["SPEAKER_0"], "__diar__You__SPEAKER_0") + + +class ResolveSpeakerPlaceholdersTests(unittest.TestCase): + def test_legacy_labels_are_untouched(self): + tagged = [(0.0, "You", "hi"), (1.0, "Others", "hey")] + self.assertEqual(_resolve_speaker_placeholders(tagged), tagged) + + def test_placeholders_numbered_from_two_by_first_appearance(self): + tagged = [ + (0.0, "You", "a"), + (1.0, "__diar__You__SPEAKER_1", "b"), + (2.0, "__diar__Others__SPEAKER_1", "c"), + (3.0, "__diar__You__SPEAKER_1", "d"), + ] + resolved = _resolve_speaker_placeholders(tagged) + self.assertEqual(resolved[0], (0.0, "You", "a")) + self.assertEqual(resolved[1], (1.0, "Speaker 2", "b")) + self.assertEqual(resolved[2], (2.0, "Speaker 3", "c")) + # Same placeholder key reuses the same number on a later turn. + self.assertEqual(resolved[3], (3.0, "Speaker 2", "d")) + + +class TagChannelSegmentsTests(unittest.TestCase): + def test_empty_asr_segments_returns_empty_without_diarizing(self): + with patch("src.transcriber._run_steno_diarize") as mock_run: + result = _tag_channel_segments([], Path("/fake/mic.wav"), 5.0, "You") + mock_run.assert_not_called() + self.assertEqual(result, []) + + def test_no_channel_path_uses_legacy_labeling(self): + asr_segments = [{"text": "Hi.", "start": 0.0, "end": 1.0}] + result = _tag_channel_segments(asr_segments, None, 5.0, "You") + self.assertEqual(result, [(0.0, "You", "Hi.")]) + + +class RunStenoDiarizeTests(unittest.TestCase): + """_run_steno_diarize must survive the sidecar's real quirks: a + diagnostic warning printed to stdout ahead of the JSON payload, and any + kind of failure (missing binary, timeout, bad exit, bad JSON).""" + + def test_returns_none_when_binary_unresolved(self): + with patch("src.transcriber._resolve_steno_diarize", return_value=None): + self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60)) + + def test_parses_json_with_e5rt_warning_prefix_on_stdout(self): + payload = json.dumps([ + {"speakerId": "SPEAKER_1", "start": 1.0, "end": 2.0}, + {"speakerId": "SPEAKER_0", "start": 0.0, "end": 0.9}, + ]).encode() + stdout = b"E5RT encountered an STL exception. msg = unordered_map::at: key not found." + payload + with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ + patch("subprocess.run", return_value=Mock(returncode=0, stdout=stdout, stderr=b"")): + result = _run_steno_diarize(Path("/fake/mic.wav"), 60) + self.assertEqual( + result, + [ + {"start": 0.0, "end": 0.9, "speaker": "SPEAKER_0"}, + {"start": 1.0, "end": 2.0, "speaker": "SPEAKER_1"}, + ], + ) + + def test_nonzero_exit_returns_none(self): + with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ + patch("subprocess.run", return_value=Mock(returncode=1, stdout=b"", stderr=b"boom")): + self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60)) + + def test_unparseable_json_returns_none(self): + with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ + patch("subprocess.run", return_value=Mock(returncode=0, stdout=b"[not json", stderr=b"")): + self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60)) + + def test_no_bracket_in_stdout_returns_none(self): + with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ + patch("subprocess.run", return_value=Mock(returncode=0, stdout=b"nothing useful", stderr=b"")): + self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60)) + + def test_timeout_returns_none(self): + with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ + patch("subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="steno-diarize", timeout=60)): + self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60)) + + if __name__ == '__main__': unittest.main() From b6b4e6525ae158d7463ad2ee1a286cf1d6efc959 Mon Sep 17 00:00:00 2001 From: Valentin Weyer Date: Wed, 8 Jul 2026 15:26:30 +0200 Subject: [PATCH 02/12] feat(transcribe): acoustic diarization for mono audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mono recordings (many imports — phone voice memos, single-track exports) had no channel split to fall back on, so they got zero speaker labelling, even when the track genuinely has multiple speakers. Runs steno-diarize directly against the whole file instead, reusing the same per-channel tagging/placeholder-resolution helpers the stereo path uses, treating the single track as the "You" channel — consistent with the pre-diarization convention of attributing an unlabelled mono recording to the user. A single real speaker still produces a plain, unlabelled transcript exactly as before. --- src/transcriber.py | 60 ++++++++++++++--- tests/test_transcriber_diarisation.py | 95 +++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 8 deletions(-) diff --git a/src/transcriber.py b/src/transcriber.py index 6d2e8a6a..3a6ecb1b 100644 --- a/src/transcriber.py +++ b/src/transcriber.py @@ -1422,18 +1422,16 @@ def transcribe_diarised(self, audio_filepath: Path, language: str = "en") -> Opt """Transcribe with stereo channel diarisation. If the audio is stereo (left=mic, right=system), each channel is - transcribed separately and labelled as [You] and [Others]. Falls - back to normal transcription for mono audio. + transcribed separately and labelled as [You] and [Others]. Mono + audio (e.g. many imported recordings) has no channel split to lean + on, but can still contain multiple speakers — acoustic diarization + runs directly against the whole file instead; see + _transcribe_diarised_mono. """ mic_path, system_path, duration = self._split_stereo_to_channels(audio_filepath) if mic_path is None: - # Mono audio — use standard transcription - result = self.transcribe_audio(audio_filepath, language) - if result: - result['is_diarised'] = False - result['diarised_text'] = None - return result + return self._transcribe_diarised_mono(audio_filepath, language) try: mic_has_audio = self._check_rms_energy(mic_path) @@ -1651,6 +1649,52 @@ def transcribe_diarised(self, audio_filepath: Path, language: str = "en") -> Opt except Exception: pass + def _transcribe_diarised_mono(self, audio_filepath: Path, language: str) -> Optional[dict]: + """Diarise a mono recording (no channel split to lean on). + + Runs standard transcription, then acoustic diarization directly + against the whole file via the same _tag_channel_segments helper the + stereo path uses per-channel — steno-diarize decodes any input + format itself (via ffmpeg), so no split/preprocessing is needed + first. The single track is treated as the "You" channel, matching + the pre-diarization convention of attributing an unlabelled mono + recording to the user; a second acoustic cluster becomes "Speaker 2" + exactly as a second cluster on the mic channel would in the stereo + path. Any diarization failure or a single-cluster result leaves + is_diarised False, matching the historical mono behaviour exactly. + """ + result = self.transcribe_audio(audio_filepath, language) + if not result: + return result + result['is_diarised'] = False + result['diarised_text'] = None + if result.get("transcription_failed") or result.get("transcription_empty"): + return result + + asr_segments = result.get("segments") or [] + duration = result.get("duration_seconds") + tagged = _tag_channel_segments(asr_segments, audio_filepath, duration, "You") + tagged.sort(key=lambda t: t[0]) + tagged = _resolve_speaker_placeholders(tagged) + + turns: list[tuple[float, str, list[str]]] = [] + for start, speaker, text in tagged: + if turns and turns[-1][1] == speaker: + turns[-1][2].append(text) + else: + turns.append((start, speaker, [text])) + + distinct_labels = {speaker for _start, speaker, _parts in turns} + if len(distinct_labels) > 1: + labelled_parts = [ + f"[{_format_timestamp(start)}] [{speaker}] {' '.join(parts)}" + for start, speaker, parts in turns + ] + result['diarised_text'] = "\n\n".join(labelled_parts) + result['is_diarised'] = True + + return result + def transcribe_with_timestamps(self, audio_filepath: Path) -> Optional[dict]: """Batch transcribe and return segment-level timing. diff --git a/tests/test_transcriber_diarisation.py b/tests/test_transcriber_diarisation.py index 680f159b..298f96ff 100644 --- a/tests/test_transcriber_diarisation.py +++ b/tests/test_transcriber_diarisation.py @@ -281,6 +281,101 @@ def test_sidecar_failure_falls_back_without_failing_meeting(self): ) +class TranscribeDiarisedMonoTests(unittest.TestCase): + """Mono audio has no mic/system channel split to lean on, but a single + track can still contain multiple speakers (e.g. an imported in-person + recording). transcribe_diarised must run acoustic diarization directly + against the whole file in that case, instead of unconditionally skipping + diarization the way the pre-diarization mono fallback did.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + d = Path(self._tmp.name) + self.audio_path = d / "mono.wav" + self.audio_path.write_bytes(b"stub") + self.transcriber = WhisperTranscriber.__new__(WhisperTranscriber) + self.transcriber.backend = "parakeet" + self.transcriber._split_stereo_to_channels = Mock(return_value=(None, None, None)) + + def tearDown(self): + self._tmp.cleanup() + + def test_two_speakers_in_mono_file_become_you_and_speaker_two(self): + diar_segments = [ + {"start": 0.0, "end": 2.0, "speaker": "SPEAKER_0"}, + {"start": 2.5, "end": 4.5, "speaker": "SPEAKER_1"}, + {"start": 5.0, "end": 8.0, "speaker": "SPEAKER_0"}, + ] + with patch("src.transcriber._run_steno_diarize", return_value=diar_segments): + self.transcriber.transcribe_audio = Mock(return_value={ + "text": "Hi there. Not bad. Great.", + "segments": [ + {"text": "Hi there.", "start": 0.5, "end": 1.5}, + {"text": "Not bad.", "start": 3.0, "end": 3.8}, + {"text": "Great.", "start": 6.0, "end": 6.8}, + ], + "duration_seconds": 8.0, + }) + result = self.transcriber.transcribe_diarised(self.audio_path) + self.assertTrue(result["is_diarised"]) + self.assertIn("[You] Hi there.", result["diarised_text"]) + self.assertIn("[Speaker 2] Not bad.", result["diarised_text"]) + self.assertIn("[You] Great.", result["diarised_text"]) + # The plain text field is untouched by diarisation. + self.assertEqual(result["text"], "Hi there. Not bad. Great.") + + def test_single_speaker_mono_is_not_diarised(self): + # Byte-identical-to-legacy fast path: one real cluster means nothing + # to disambiguate, matching the pre-diarization mono behaviour. + diar_segments = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}] + with patch("src.transcriber._run_steno_diarize", return_value=diar_segments): + self.transcriber.transcribe_audio = Mock(return_value={ + "text": "Just me talking.", + "segments": [{"text": "Just me talking.", "start": 0.5, "end": 1.5}], + "duration_seconds": 5.0, + }) + result = self.transcriber.transcribe_diarised(self.audio_path) + self.assertFalse(result["is_diarised"]) + self.assertIsNone(result["diarised_text"]) + + def test_sidecar_failure_falls_back_without_diarisation(self): + with patch("src.transcriber._run_steno_diarize", return_value=None): + self.transcriber.transcribe_audio = Mock(return_value={ + "text": "Hello world.", + "segments": [{"text": "Hello world.", "start": 0.5, "end": 1.5}], + "duration_seconds": 2.0, + }) + result = self.transcriber.transcribe_diarised(self.audio_path) + self.assertFalse(result["is_diarised"]) + self.assertIsNone(result["diarised_text"]) + self.assertEqual(result["text"], "Hello world.") + + def test_transcription_failure_propagates_without_diarizing(self): + with patch("src.transcriber._run_steno_diarize") as mock_diarize: + self.transcriber.transcribe_audio = Mock(return_value={ + "text": None, + "segments": [], + "transcription_failed": True, + "error": "boom", + }) + result = self.transcriber.transcribe_diarised(self.audio_path) + mock_diarize.assert_not_called() + self.assertTrue(result["transcription_failed"]) + self.assertFalse(result["is_diarised"]) + self.assertIsNone(result["diarised_text"]) + + def test_empty_transcription_does_not_attempt_diarization(self): + with patch("src.transcriber._run_steno_diarize") as mock_diarize: + self.transcriber.transcribe_audio = Mock(return_value={ + "text": "No speech detected in audio", + "segments": [], + "transcription_empty": True, + }) + result = self.transcriber.transcribe_diarised(self.audio_path) + mock_diarize.assert_not_called() + self.assertFalse(result["is_diarised"]) + + class TokenJaccardTests(unittest.TestCase): def test_identical_strings_score_one(self): self.assertEqual(_token_jaccard("hello world", "hello world"), 1.0) From 05b145af02a3923720716e64d5ed6dade151fc94 Mon Sep 17 00:00:00 2001 From: Valentin Weyer Date: Tue, 14 Jul 2026 21:14:31 +0200 Subject: [PATCH 03/12] perf(diarize): ~6.65x faster via Sortformer's offline chunk preset Sortformer was configured with .default (fastV2_1: 0.48s of audio per CoreML invocation, ~1.04s latency) -- tuned for live/streaming responsiveness this app has no use for, since diarization only ever runs on a fully-recorded, already-finished channel. Switched to .highContextV2 (27.2s per invocation) for recordings long enough to benefit: ~56x fewer invocations for the same audio (~400 vs ~22,500 for a 3-hour file). Measured on a real ~21-minute recording: 153s -> 23s. V2, not V2.1: FluidAudio's own docs note V2.1 "may degrade when many speakers are talking simultaneously" -- a real risk given this app's crosstalk/echo findings from earlier diarization work. Real regression found and fixed during validation: highContextV2's chunk loader requires a full ~30.4s window before it emits anything at all -- a 12.25s test clip came back with zero segments. Added sortformerHighContextMinDuration (90s, real margin above the hard minimum) so recordings shorter than that keep using .default. Accuracy validated against the same real file, not just "it still runs": 98.2% agreement on the dominant speaker's per-second attribution, a near-zero spurious "4th speaker" (0.6s total) cleanly disappeared, and total detected speech stayed within ~1%. The GPU-vs-ANE compute units env var wired into the manual/backfill CLI paths in a prior commit (measured separately: 23.0s ANE vs 18.0s GPU on the same file) stays opt-in-only -- the normal per-meeting pipeline keeps the power/thermal-efficient ANE default. --- diarize-sidecar/Sources/main.swift | 307 ++++++++++++++++++++++++++++- 1 file changed, 299 insertions(+), 8 deletions(-) diff --git a/diarize-sidecar/Sources/main.swift b/diarize-sidecar/Sources/main.swift index 52e8b065..ef19a88c 100644 --- a/diarize-sidecar/Sources/main.swift +++ b/diarize-sidecar/Sources/main.swift @@ -1,4 +1,5 @@ -// diarize-sidecar — offline speaker diarization via FluidAudio's Sortformer. +// diarize-sidecar — offline speaker diarization + voiceprint embeddings via +// FluidAudio's Sortformer + WeSpeaker. // // Usage: // steno-diarize @@ -8,7 +9,8 @@ // previous OfflineDiarizerManager-based version of this tool. // // Output (stdout): one JSON line on success -// [{"speakerId":"SPEAKER_0","start":0.0,"end":3.2}, ...] +// {"segments":[{"speakerId":"SPEAKER_0","start":0.0,"end":3.2}, ...], +// "speakers":{"SPEAKER_0":[0.1,0.2,...(256 floats)], ...}} // // Exit 0 on success, 1 on failure (error written to stderr). // stdout is unbuffered so the parent receives the line immediately. @@ -20,7 +22,21 @@ // with raw samples rather than the processComplete(audioFileURL:) overload — // that overload internally uses AudioConverter.resampleAudioFile, which // calls AVAudioFile(forReading:) and would reintroduce the same crash. +// +// Voiceprint embeddings: extracted via FluidAudio's own bundled WeSpeaker +// model (DiarizerModels/EmbeddingExtractor — the same one the older, +// pyannote-segmentation-based DiarizerManager pipeline uses), NOT a +// separate ONNX model. This mirrors the proven approach in +// github.com/pasrom/meeting-transcriber (verified via GitHub's raw content +// API directly, not a fetched summary): overlap-excluded per-speaker +// activity masks from Sortformer's own frame-level predictions (so +// crosstalk never contaminates an embedding), chunked into 10s windows to +// match WeSpeaker's fixed input shape, embeddings averaged (L2-normalized +// mean) across all chunks into one centroid per speaker. A single-clip, +// no-overlap-exclusion embedding (an earlier, simpler attempt) proved too +// easily confused between genuinely different speakers in real testing. +import CoreML import Foundation import FluidAudio @@ -31,11 +47,36 @@ setbuf(stdout, nil) // emitting rather than surfaced as a phantom speaker turn. let minSegmentDurationSeconds: Float = 0.25 +// SortformerConfig.highContextV2's chunk loader (SortformerFeatureLoader) +// requires a full chunkLen+rightContext window -- 340+40 frames at 0.08s +// each, 30.4s -- before it emits ANYTHING; audio shorter than that gets +// ZERO segments, not degraded ones (confirmed empirically: a 12.25s test +// clip returned {"speakers":{},"segments":[]} with highContextV2). This +// threshold gives real margin above that hard minimum before switching +// away from the always-safe .default config. +let sortformerHighContextMinDuration: Double = 90.0 + func fail(_ message: String) -> Never { fputs("steno-diarize error: \(message)\n", stderr) exit(1) } +// Compute-unit override, primarily for one-off bulk backfill runs where +// throughput matters more than the power/thermal cost of spinning up the +// GPU (unlike live recording, where the default below is deliberately +// power-efficient). Unset -> unchanged default (.cpuAndNeuralEngine, see +// the call sites below for why that's forced rather than left at .all). +// STENOAI_DIARIZE_COMPUTE_UNITS: "all" | "cpuAndGPU" | "cpuOnly" | +// "cpuAndNeuralEngine" (default). +func resolveComputeUnits() -> MLComputeUnits { + switch ProcessInfo.processInfo.environment["STENOAI_DIARIZE_COMPUTE_UNITS"] { + case "all": return .all + case "cpuAndGPU": return .cpuAndGPU + case "cpuOnly": return .cpuOnly + default: return .cpuAndNeuralEngine + } +} + guard CommandLine.arguments.count == 2 else { fail("usage: steno-diarize ") } @@ -129,23 +170,262 @@ func loadSamplesViaFfmpeg(path: String) async throws -> [Float] { } } +// MARK: - Voiceprint embedding extraction (ported from +// github.com/pasrom/meeting-transcriber's FluidDiarizer+SortformerEmbeddings.swift +// and the buildOverlapExcludedMasks/resampleMask/aggregateCentroids helpers +// in their FluidDiarizer.swift, verified via GitHub's raw content API). + +/// Build per-speaker activity masks (1.0/0.0) with overlap-exclusion: any +/// frame where >=2 speakers exceed `threshold` is zeroed across ALL +/// speakers, so impure (crosstalk) frames never reach embedding extraction. +/// DiariZen-style stage-5 design. +/// +/// A margin-based variant (requiring the runner-up speaker's probability to +/// stay below a separate low ceiling, not just fail to cross `threshold`) +/// was tried and measured against real same-room audio (AMI Meeting +/// Corpus): no meaningful improvement over this simpler version. The +/// remaining same-room discrimination problem is in the raw waveform +/// (physical mic bleed), not in which frames get selected -- masking +/// chooses what to feed the model, it can't clean the audio itself. +/// Reverted to match the reference implementation +/// (pasrom/meeting-transcriber's `FluidDiarizer.swift`) rather than keep +/// unproven complexity. +/// +/// - Parameters: +/// - predictions: flat [numFrames x numSpeakers] from DiarizerTimeline.finalizedPredictions. +/// - numSpeakers: speaker-slot count (Sortformer hardcodes 4). +/// - threshold: activity threshold (timeline.config.onsetThreshold). +/// - Returns: [numSpeakers] arrays of length numFrames. +func buildOverlapExcludedMasks( + predictions: [Float], + numSpeakers: Int, + threshold: Float +) -> [[Float]] { + guard numSpeakers > 0, !predictions.isEmpty else { return [] } + let numFrames = predictions.count / numSpeakers + guard numFrames > 0 else { return [] } + var masks = Array(repeating: Array(repeating: Float(0.0), count: numFrames), count: numSpeakers) + + for frame in 0..= threshold { + activeCount += 1 + activeSlot = s + if activeCount > 1 { break } + } + if activeCount == 1 { + masks[activeSlot][frame] = 1.0 + } + } + return masks +} + +/// Nearest-neighbour resample a per-frame activity mask onto a target frame +/// grid. Bridges Sortformer's 12.5 Hz output (~125 frames/10s) to +/// WeSpeaker's expected segmentation-frame count (typically 589/10s). +func resampleMask(_ mask: [Float], to targetCount: Int) -> [Float] { + guard !mask.isEmpty, targetCount > 0 else { + return Array(repeating: Float(0.0), count: max(0, targetCount)) + } + var out = Array(repeating: Float(0.0), count: targetCount) + let srcCount = mask.count + for i in 0.. one centroid per +/// speaker. +func aggregateCentroids( + sums: [String: [Float]], + counts: [String: Int] +) -> [String: [Float]] { + var result = [String: [Float]](minimumCapacity: sums.count) + for (label, sum) in sums { + let count = Float(counts[label] ?? 1) + var mean = sum.map { $0 / count } + let norm = (mean.reduce(into: Float(0)) { $0 += $1 * $1 }).squareRoot() + if norm > 1e-9 { + mean = mean.map { $0 / norm } + } + result[label] = mean + } + return result +} + +/// Walk the audio in 10s chunks, run WeSpeaker on the top-3 active speakers +/// per chunk (the model's mask shape only fits 3), accumulate running sums +/// + counts per global Sortformer speaker slot. Sortformer's 4th speaker +/// (when present) gets covered in chunks where they rank in the top-3 of +/// that window. +func accumulateChunkEmbeddings( + audio: [Float], + masks: [[Float]], + frameDuration: Double, + weSpeakerFrameCount: Int, + extractor: EmbeddingExtractor +) -> (sums: [String: [Float]], counts: [String: Int]) { + let chunkSamples = 160_000 // 10s @ 16kHz -- matches EmbeddingExtractor's waveform shape + let framesPerChunk = max(1, Int(10.0 / frameDuration)) + let numSpeakers = masks.count + let maskFrameCount = masks.first?.count ?? 0 + + var sums = [String: [Float]](minimumCapacity: numSpeakers) + var counts = [String: Int](minimumCapacity: numSpeakers) + var failedChunks = 0 + + // Progress reporting for the parent process (Python's _run_steno_diarize): + // this loop is the single longest-running phase on a multi-hour recording + // (~1300+ sequential 10s chunks measured on a real ~3.5h file, ~18 minutes + // per channel) with no other checkpoint to report from. Emitted to STDERR + // -- stdout carries exactly one JSON line on success (see the file header + // comment) and must never be touched mid-loop. + let totalChunks = max(1, (audio.count + chunkSamples - 1) / chunkSamples) + var chunkIndex = 0 + + var sampleStart = 0 + var frameStart = 0 + while sampleStart < audio.count, frameStart < maskFrameCount { + let sampleEnd = min(sampleStart + chunkSamples, audio.count) + let frameEnd = min(frameStart + framesPerChunk, maskFrameCount) + let chunk = Array(audio[sampleStart.. $1.sum }.prefix(3).map(\.slot) + let masksForCall = topSlots.map { resampleMask(chunkMasks[$0], to: weSpeakerFrameCount) } + + // A single chunk's embedding extraction can fail transiently -- + // measured on a real ~3.5h recording (~1300+ sequential 10s + // chunks): an internal FluidAudio/E5RT error partway through, + // without the surrounding chunks being unhealthy. The previous + // `throws`-and-propagate behavior let ONE bad chunk discard every + // OTHER chunk's already-accumulated embeddings, zeroing out + // voiceprint data for the entire recording over a single transient + // failure. Catch per-chunk instead: skip just that chunk and keep + // going, so a long recording still gets a real (if very slightly + // incomplete) centroid rather than nothing at all. + do { + let embs = try extractor.getEmbeddings(audio: chunk, masks: masksForCall) + for (i, slot) in topSlots.enumerated() { + let emb = embs[i] + guard !emb.allSatisfy({ $0 == 0 }) else { continue } + let label = "SPEAKER_\(slot)" + sums[label] = sums[label].map { zip($0, emb).map(+) } ?? emb + counts[label, default: 0] += 1 + } + } catch { + failedChunks += 1 + fputs("steno-diarize: chunk embedding extraction failed, skipping this chunk: \(error)\n", stderr) + } + chunkIndex += 1 + fputs("PROGRESS:embedding:\(chunkIndex)/\(totalChunks)\n", stderr) + sampleStart = sampleEnd + frameStart = frameEnd + } + if failedChunks > 0 { + fputs("steno-diarize: \(failedChunks) chunk(s) failed embedding extraction and were skipped\n", stderr) + } + return (sums, counts) +} + +/// Extract one voiceprint centroid per active Sortformer speaker from +/// `timeline`'s frame-level predictions and the already-decoded 16kHz +/// audio samples. Returns an empty dict (never throws) on any embedding +/// failure — voiceprint identification is a best-effort enhancement, the +/// diarization segments themselves are the load-bearing output. +func extractSortformerEmbeddings( + audio: [Float], + timeline: DiarizerTimeline +) async -> [String: [Float]] { + do { + let models = try await DiarizerModels.load( + configuration: MLModelConfigurationUtils.defaultConfiguration(computeUnits: resolveComputeUnits()) + ) + let extractor = EmbeddingExtractor(embeddingModel: models.embeddingModel) + + // WeSpeaker expects masks shaped [3, weSpeakerFrameCount] where the + // frame count is fixed by the companion pyannote segmentation + // model. Query at runtime so a future model swap doesn't silently + // mis-shape. + guard + let segShape = models.segmentationModel.modelDescription + .outputDescriptionsByName["segments"]?.multiArrayConstraint?.shape, + segShape.count >= 2 + else { + fputs("steno-diarize: embedding skipped (unexpected segmentation model shape)\n", stderr) + return [:] + } + let weSpeakerFrameCount = segShape[1].intValue + + let masks = buildOverlapExcludedMasks( + predictions: timeline.finalizedPredictions, + numSpeakers: timeline.config.numSpeakers, + threshold: timeline.config.onsetThreshold + ) + let maskFrameCount = masks.first?.count ?? 0 + guard maskFrameCount > 0 else { return [:] } + + let (sums, counts) = accumulateChunkEmbeddings( + audio: audio, + masks: masks, + frameDuration: Double(timeline.config.frameDurationSeconds), + weSpeakerFrameCount: weSpeakerFrameCount, + extractor: extractor + ) + return aggregateCentroids(sums: sums, counts: counts) + } catch { + fputs("steno-diarize: embedding extraction failed (segments still valid): \(error)\n", stderr) + return [:] + } +} + // Keep the main run loop alive so dispatch sources (including the // process-exit source that drives terminationHandler) can fire normally. // sema.wait() blocks the main thread, which prevents dispatch delivery // and causes terminationHandler to never fire. Task { do { + let samples = try await loadSamplesViaFfmpeg(path: inputPath) + // .cpuAndNeuralEngine forces genuine ANE execution — the default // .all silently routes Sortformer to GPU instead (confirmed via - // Activity Monitor during evaluation). + // Activity Monitor during evaluation). resolveComputeUnits() keeps + // that as the default but allows STENOAI_DIARIZE_COMPUTE_UNITS=all + // (or =cpuAndGPU) to opt into GPU for one-off bulk backfill runs + // where throughput matters more than the live-recording-path's + // power/thermal efficiency. + // + // Sortformer config: this app only ever diarizes a fully-recorded, + // already-finished channel (no live/streaming diarization exists + // yet), so .default's low-latency 0.48s-per-invocation chunking + // (tuned for real-time responsiveness this app has no use for) + // costs ~56x more CoreML invocations than .highContextV2's + // 27.2s-per-invocation chunking on the same audio (measured: + // ~22,500 vs ~400 invocations for a 3-hour recording) — but + // highContextV2 needs a full ~30.4s window before it emits + // anything at all, so it's only used once the recording is + // comfortably longer than that (see sortformerHighContextMinDuration). + // V2, not V2.1: FluidAudio's own docs note V2.1 "may degrade when + // many speakers are talking simultaneously" — a real risk given + // this app's crosstalk/echo findings from earlier this session. + // Both the model-loading config below AND SortformerDiarizer's own + // config must match — its internal chunk/fifo/spkcache buffers are + // sized from whatever config it's constructed with, independent of + // which model weights get loaded. + let durationSeconds = Double(samples.count) / 16000.0 + let sortformerConfig: SortformerConfig = + durationSeconds >= sortformerHighContextMinDuration ? .highContextV2 : .default let models = try await SortformerModels.loadFromHuggingFace( - config: .default, - computeUnits: .cpuAndNeuralEngine + config: sortformerConfig, + computeUnits: resolveComputeUnits() ) - let diarizer = SortformerDiarizer() + let diarizer = SortformerDiarizer(config: sortformerConfig) diarizer.initialize(models: models) - let samples = try await loadSamplesViaFfmpeg(path: inputPath) let timeline = try diarizer.processComplete(samples, sourceSampleRate: nil) struct Segment: Encodable { @@ -153,8 +433,12 @@ Task { let start: Double let end: Double } + struct Output: Encodable { + let segments: [Segment] + let speakers: [String: [Float]] + } - let output = timeline.speakers.values + let segments = timeline.speakers.values .flatMap { $0.finalizedSegments } .filter { $0.duration >= minSegmentDurationSeconds } .map { seg in @@ -166,6 +450,13 @@ Task { } .sorted { $0.start < $1.start } + // Voiceprint centroids, one per active speaker slot. Best-effort: + // extractSortformerEmbeddings never throws, returning [:] on any + // failure so a voiceprint problem can never take down diarization + // itself (the segments above are the load-bearing output). + let speakers = await extractSortformerEmbeddings(audio: samples, timeline: timeline) + + let output = Output(segments: segments, speakers: speakers) let encoded = try JSONEncoder().encode(output) guard let line = String(data: encoded, encoding: .utf8) else { exit(1) From 35073429094edeaf0811f025696b5810202f228d Mon Sep 17 00:00:00 2001 From: Valentin Weyer Date: Thu, 30 Jul 2026 23:56:29 +0200 Subject: [PATCH 04/12] feat(diarize): robust sidecar parsing, dominance gate, live progress Ports the diarization-relevant slice of 04c2be1 (which mixed diarization, progress, and identity concerns in one commit) onto the standalone diarization branch, minus everything identity-specific: - CHANNEL_DOMINANCE_THRESHOLD: a channel with one overwhelmingly dominant speaker is treated as single-speaker rather than spawning a phantom second speaker from a misdiarization blip. - CHANNEL_DETECT_TIMEOUT_S: the channel-count probe's fixed 15s timeout silently dropped long WebM recordings to mono; scaled to 60s. - _run_steno_diarize rewritten to Popen + two reader threads (avoids the classic pipe-deadlock on large stdout payloads) with a real JSON decoder scan for the last valid segment array, tolerating FluidAudio/ CoreML warning text before, between, or after the payload. - _heartbeat_while_waiting + PROGRESS:diarize:{label}:start/:done so a long diarization pass doesn't look hung to Electron's inactivity watchdog or sit on a static spinner. - Pre-processing audio start log line, so loudnorm's two-pass analysis doesn't look like a hang on a long recording either. The sidecar's Output contract stays a bare segment array (this branch never extracts voiceprint embeddings), so the parser and its tests are array-only rather than the array-or-object form the full identity branch needs. --- diarize-sidecar/Sources/main.swift | 254 +------------------------- src/transcriber.py | 208 ++++++++++++++++++--- tests/test_transcriber_diarisation.py | 100 +++++++++- 3 files changed, 281 insertions(+), 281 deletions(-) diff --git a/diarize-sidecar/Sources/main.swift b/diarize-sidecar/Sources/main.swift index ef19a88c..ef24431d 100644 --- a/diarize-sidecar/Sources/main.swift +++ b/diarize-sidecar/Sources/main.swift @@ -1,16 +1,13 @@ -// diarize-sidecar — offline speaker diarization + voiceprint embeddings via -// FluidAudio's Sortformer + WeSpeaker. +// diarize-sidecar — offline speaker diarization via FluidAudio's Sortformer. // // Usage: // steno-diarize // // Sortformer has a fixed 4-speaker-slot architecture (SortformerConfig.numSpeakers -// is hardcoded to 4) — there is no speaker-count hint to pass, unlike the -// previous OfflineDiarizerManager-based version of this tool. +// is hardcoded to 4) — there is no speaker-count hint to pass. // // Output (stdout): one JSON line on success -// {"segments":[{"speakerId":"SPEAKER_0","start":0.0,"end":3.2}, ...], -// "speakers":{"SPEAKER_0":[0.1,0.2,...(256 floats)], ...}} +// [{"speakerId":"SPEAKER_0","start":0.0,"end":3.2}, ...] // // Exit 0 on success, 1 on failure (error written to stderr). // stdout is unbuffered so the parent receives the line immediately. @@ -22,19 +19,6 @@ // with raw samples rather than the processComplete(audioFileURL:) overload — // that overload internally uses AudioConverter.resampleAudioFile, which // calls AVAudioFile(forReading:) and would reintroduce the same crash. -// -// Voiceprint embeddings: extracted via FluidAudio's own bundled WeSpeaker -// model (DiarizerModels/EmbeddingExtractor — the same one the older, -// pyannote-segmentation-based DiarizerManager pipeline uses), NOT a -// separate ONNX model. This mirrors the proven approach in -// github.com/pasrom/meeting-transcriber (verified via GitHub's raw content -// API directly, not a fetched summary): overlap-excluded per-speaker -// activity masks from Sortformer's own frame-level predictions (so -// crosstalk never contaminates an embedding), chunked into 10s windows to -// match WeSpeaker's fixed input shape, embeddings averaged (L2-normalized -// mean) across all chunks into one centroid per speaker. A single-clip, -// no-overlap-exclusion embedding (an earlier, simpler attempt) proved too -// easily confused between genuinely different speakers in real testing. import CoreML import Foundation @@ -51,9 +35,9 @@ let minSegmentDurationSeconds: Float = 0.25 // requires a full chunkLen+rightContext window -- 340+40 frames at 0.08s // each, 30.4s -- before it emits ANYTHING; audio shorter than that gets // ZERO segments, not degraded ones (confirmed empirically: a 12.25s test -// clip returned {"speakers":{},"segments":[]} with highContextV2). This -// threshold gives real margin above that hard minimum before switching -// away from the always-safe .default config. +// clip returned [] with highContextV2). This threshold gives real margin +// above that hard minimum before switching away from the always-safe +// .default config. let sortformerHighContextMinDuration: Double = 90.0 func fail(_ message: String) -> Never { @@ -170,219 +154,6 @@ func loadSamplesViaFfmpeg(path: String) async throws -> [Float] { } } -// MARK: - Voiceprint embedding extraction (ported from -// github.com/pasrom/meeting-transcriber's FluidDiarizer+SortformerEmbeddings.swift -// and the buildOverlapExcludedMasks/resampleMask/aggregateCentroids helpers -// in their FluidDiarizer.swift, verified via GitHub's raw content API). - -/// Build per-speaker activity masks (1.0/0.0) with overlap-exclusion: any -/// frame where >=2 speakers exceed `threshold` is zeroed across ALL -/// speakers, so impure (crosstalk) frames never reach embedding extraction. -/// DiariZen-style stage-5 design. -/// -/// A margin-based variant (requiring the runner-up speaker's probability to -/// stay below a separate low ceiling, not just fail to cross `threshold`) -/// was tried and measured against real same-room audio (AMI Meeting -/// Corpus): no meaningful improvement over this simpler version. The -/// remaining same-room discrimination problem is in the raw waveform -/// (physical mic bleed), not in which frames get selected -- masking -/// chooses what to feed the model, it can't clean the audio itself. -/// Reverted to match the reference implementation -/// (pasrom/meeting-transcriber's `FluidDiarizer.swift`) rather than keep -/// unproven complexity. -/// -/// - Parameters: -/// - predictions: flat [numFrames x numSpeakers] from DiarizerTimeline.finalizedPredictions. -/// - numSpeakers: speaker-slot count (Sortformer hardcodes 4). -/// - threshold: activity threshold (timeline.config.onsetThreshold). -/// - Returns: [numSpeakers] arrays of length numFrames. -func buildOverlapExcludedMasks( - predictions: [Float], - numSpeakers: Int, - threshold: Float -) -> [[Float]] { - guard numSpeakers > 0, !predictions.isEmpty else { return [] } - let numFrames = predictions.count / numSpeakers - guard numFrames > 0 else { return [] } - var masks = Array(repeating: Array(repeating: Float(0.0), count: numFrames), count: numSpeakers) - - for frame in 0..= threshold { - activeCount += 1 - activeSlot = s - if activeCount > 1 { break } - } - if activeCount == 1 { - masks[activeSlot][frame] = 1.0 - } - } - return masks -} - -/// Nearest-neighbour resample a per-frame activity mask onto a target frame -/// grid. Bridges Sortformer's 12.5 Hz output (~125 frames/10s) to -/// WeSpeaker's expected segmentation-frame count (typically 589/10s). -func resampleMask(_ mask: [Float], to targetCount: Int) -> [Float] { - guard !mask.isEmpty, targetCount > 0 else { - return Array(repeating: Float(0.0), count: max(0, targetCount)) - } - var out = Array(repeating: Float(0.0), count: targetCount) - let srcCount = mask.count - for i in 0.. one centroid per -/// speaker. -func aggregateCentroids( - sums: [String: [Float]], - counts: [String: Int] -) -> [String: [Float]] { - var result = [String: [Float]](minimumCapacity: sums.count) - for (label, sum) in sums { - let count = Float(counts[label] ?? 1) - var mean = sum.map { $0 / count } - let norm = (mean.reduce(into: Float(0)) { $0 += $1 * $1 }).squareRoot() - if norm > 1e-9 { - mean = mean.map { $0 / norm } - } - result[label] = mean - } - return result -} - -/// Walk the audio in 10s chunks, run WeSpeaker on the top-3 active speakers -/// per chunk (the model's mask shape only fits 3), accumulate running sums -/// + counts per global Sortformer speaker slot. Sortformer's 4th speaker -/// (when present) gets covered in chunks where they rank in the top-3 of -/// that window. -func accumulateChunkEmbeddings( - audio: [Float], - masks: [[Float]], - frameDuration: Double, - weSpeakerFrameCount: Int, - extractor: EmbeddingExtractor -) -> (sums: [String: [Float]], counts: [String: Int]) { - let chunkSamples = 160_000 // 10s @ 16kHz -- matches EmbeddingExtractor's waveform shape - let framesPerChunk = max(1, Int(10.0 / frameDuration)) - let numSpeakers = masks.count - let maskFrameCount = masks.first?.count ?? 0 - - var sums = [String: [Float]](minimumCapacity: numSpeakers) - var counts = [String: Int](minimumCapacity: numSpeakers) - var failedChunks = 0 - - // Progress reporting for the parent process (Python's _run_steno_diarize): - // this loop is the single longest-running phase on a multi-hour recording - // (~1300+ sequential 10s chunks measured on a real ~3.5h file, ~18 minutes - // per channel) with no other checkpoint to report from. Emitted to STDERR - // -- stdout carries exactly one JSON line on success (see the file header - // comment) and must never be touched mid-loop. - let totalChunks = max(1, (audio.count + chunkSamples - 1) / chunkSamples) - var chunkIndex = 0 - - var sampleStart = 0 - var frameStart = 0 - while sampleStart < audio.count, frameStart < maskFrameCount { - let sampleEnd = min(sampleStart + chunkSamples, audio.count) - let frameEnd = min(frameStart + framesPerChunk, maskFrameCount) - let chunk = Array(audio[sampleStart.. $1.sum }.prefix(3).map(\.slot) - let masksForCall = topSlots.map { resampleMask(chunkMasks[$0], to: weSpeakerFrameCount) } - - // A single chunk's embedding extraction can fail transiently -- - // measured on a real ~3.5h recording (~1300+ sequential 10s - // chunks): an internal FluidAudio/E5RT error partway through, - // without the surrounding chunks being unhealthy. The previous - // `throws`-and-propagate behavior let ONE bad chunk discard every - // OTHER chunk's already-accumulated embeddings, zeroing out - // voiceprint data for the entire recording over a single transient - // failure. Catch per-chunk instead: skip just that chunk and keep - // going, so a long recording still gets a real (if very slightly - // incomplete) centroid rather than nothing at all. - do { - let embs = try extractor.getEmbeddings(audio: chunk, masks: masksForCall) - for (i, slot) in topSlots.enumerated() { - let emb = embs[i] - guard !emb.allSatisfy({ $0 == 0 }) else { continue } - let label = "SPEAKER_\(slot)" - sums[label] = sums[label].map { zip($0, emb).map(+) } ?? emb - counts[label, default: 0] += 1 - } - } catch { - failedChunks += 1 - fputs("steno-diarize: chunk embedding extraction failed, skipping this chunk: \(error)\n", stderr) - } - chunkIndex += 1 - fputs("PROGRESS:embedding:\(chunkIndex)/\(totalChunks)\n", stderr) - sampleStart = sampleEnd - frameStart = frameEnd - } - if failedChunks > 0 { - fputs("steno-diarize: \(failedChunks) chunk(s) failed embedding extraction and were skipped\n", stderr) - } - return (sums, counts) -} - -/// Extract one voiceprint centroid per active Sortformer speaker from -/// `timeline`'s frame-level predictions and the already-decoded 16kHz -/// audio samples. Returns an empty dict (never throws) on any embedding -/// failure — voiceprint identification is a best-effort enhancement, the -/// diarization segments themselves are the load-bearing output. -func extractSortformerEmbeddings( - audio: [Float], - timeline: DiarizerTimeline -) async -> [String: [Float]] { - do { - let models = try await DiarizerModels.load( - configuration: MLModelConfigurationUtils.defaultConfiguration(computeUnits: resolveComputeUnits()) - ) - let extractor = EmbeddingExtractor(embeddingModel: models.embeddingModel) - - // WeSpeaker expects masks shaped [3, weSpeakerFrameCount] where the - // frame count is fixed by the companion pyannote segmentation - // model. Query at runtime so a future model swap doesn't silently - // mis-shape. - guard - let segShape = models.segmentationModel.modelDescription - .outputDescriptionsByName["segments"]?.multiArrayConstraint?.shape, - segShape.count >= 2 - else { - fputs("steno-diarize: embedding skipped (unexpected segmentation model shape)\n", stderr) - return [:] - } - let weSpeakerFrameCount = segShape[1].intValue - - let masks = buildOverlapExcludedMasks( - predictions: timeline.finalizedPredictions, - numSpeakers: timeline.config.numSpeakers, - threshold: timeline.config.onsetThreshold - ) - let maskFrameCount = masks.first?.count ?? 0 - guard maskFrameCount > 0 else { return [:] } - - let (sums, counts) = accumulateChunkEmbeddings( - audio: audio, - masks: masks, - frameDuration: Double(timeline.config.frameDurationSeconds), - weSpeakerFrameCount: weSpeakerFrameCount, - extractor: extractor - ) - return aggregateCentroids(sums: sums, counts: counts) - } catch { - fputs("steno-diarize: embedding extraction failed (segments still valid): \(error)\n", stderr) - return [:] - } -} - // Keep the main run loop alive so dispatch sources (including the // process-exit source that drives terminationHandler) can fire normally. // sema.wait() blocks the main thread, which prevents dispatch delivery @@ -433,10 +204,6 @@ Task { let start: Double let end: Double } - struct Output: Encodable { - let segments: [Segment] - let speakers: [String: [Float]] - } let segments = timeline.speakers.values .flatMap { $0.finalizedSegments } @@ -450,14 +217,7 @@ Task { } .sorted { $0.start < $1.start } - // Voiceprint centroids, one per active speaker slot. Best-effort: - // extractSortformerEmbeddings never throws, returning [:] on any - // failure so a voiceprint problem can never take down diarization - // itself (the segments above are the load-bearing output). - let speakers = await extractSortformerEmbeddings(audio: samples, timeline: timeline) - - let output = Output(segments: segments, speakers: speakers) - let encoded = try JSONEncoder().encode(output) + let encoded = try JSONEncoder().encode(segments) guard let line = String(data: encoded, encoding: .utf8) else { exit(1) } diff --git a/src/transcriber.py b/src/transcriber.py index 3a6ecb1b..959fa1b4 100644 --- a/src/transcriber.py +++ b/src/transcriber.py @@ -27,6 +27,7 @@ anything; the model is the source of truth. """ +import contextlib import inspect import json import logging @@ -97,6 +98,14 @@ # pre-process floor; the helper scales it up with duration for long meetings. DIARISED_SPLIT_TIMEOUT_S = 600 +# Timeout for the channel-COUNT probe in _split_stereo_to_channels (`-t 0`, +# header-only -- runs BEFORE we know duration, so unlike the timeouts above +# this can't scale with it). Should be near-instant, but a live-recorded +# WebM (no seek index) measured >15s on a real ~3.5h recording -- same +# silent-mono-fallback failure this whole file already guards against +# elsewhere, just one step earlier in the pipeline. +CHANNEL_DETECT_TIMEOUT_S = 60 + # RMS energy gate for "channel has speech". Intentionally low (-70 dB) so # headphones-mode mic recordings — captured at much lower amplitude than # speakers-mode — still pass. The model handles low-amplitude speech fine; @@ -121,6 +130,14 @@ # long recordings, same pattern as _diarised_split_timeout. STENO_DIARIZE_TIMEOUT_FLOOR_S = 120 +# If one diarizer cluster holds this share (or more) of a channel's total +# speaking time, the channel is treated as single-speaker — any other +# cluster is almost certainly a brief misdiarization blip (observed +# empirically: short/overlapping noise segments from Sortformer on +# single-mic audio), not a real second speaker. Gates the "Speaker N" +# placeholder path (_cluster_channel_labels). +CHANNEL_DOMINANCE_THRESHOLD = 0.92 + # Sentinel text substituted when transcription produces no usable output # (genuine silence or all-hallucination). Callers compare against this to # distinguish "really nothing was said" from a real (possibly short) @@ -603,6 +620,49 @@ def _assign_asr_segments_to_diar_segments(asr_segments: list[dict], diar_segment segment["text"] = " ".join(t.strip() for t in texts_by_segment[i] if t.strip()).strip() +# How often to print a HEARTBEAT: line while blocked waiting on +# steno-diarize. Comfortably under Electron's TRANSCRIBE_INACTIVITY_MS +# (8 minutes, app/main.js) -- see _heartbeat_while_waiting's docstring for +# why this can't just reuse the existing chunk-progress heartbeat registry. +STENO_DIARIZE_HEARTBEAT_INTERVAL_S = 60.0 + + +@contextlib.contextmanager +def _heartbeat_while_waiting(label: str, interval_s: float = STENO_DIARIZE_HEARTBEAT_INTERVAL_S): + """Print a HEARTBEAT: line every ``interval_s`` seconds on a background + thread for the duration of the ``with`` block. + + src._heartbeat's chunk-progress registry only works for backends that + call back into Python from INSIDE their own per-chunk loop (Parakeet, + Whisper.cpp) -- steno-diarize is an opaque external binary invoked via a + single blocking call, with no such checkpoint to hang a callback off of. + Without this, a diarization run on an hours-long channel prints nothing + for its entire duration, which Electron's inactivity watchdog + (app/main.js) can't tell apart from a hung process -- and kills, + discarding a real, working meeting. + + Never affects the wrapped call's own return value or exceptions -- + the background thread only ever writes heartbeat lines. + """ + stop = threading.Event() + + def _beat(): + while not stop.wait(interval_s): + try: + sys.stdout.write(f"HEARTBEAT:{label}\n") + sys.stdout.flush() + except Exception: + pass + + t = threading.Thread(target=_beat, daemon=True) + t.start() + try: + yield + finally: + stop.set() + t.join(timeout=1.0) + + def _run_steno_diarize(channel_path: Path, timeout: int) -> Optional[list[dict]]: """Run the steno-diarize sidecar on a single mono channel WAV. @@ -611,31 +671,91 @@ def _run_steno_diarize(channel_path: Path, timeout: int) -> Optional[list[dict]] non-zero exit, or unparseable output — so callers always have a safe fallback to legacy channel-only labeling and this can never fail a meeting. + + Uses Popen with two concurrent reader threads rather than + subprocess.run(capture_output=True) so stdout and stderr are both + drained WHILE the process is still running -- matching what + subprocess.run's own communicate() does internally to avoid the + classic pipe-deadlock (real stdout payloads have measured up to + ~211KB in production, well past an OS pipe buffer, so neither stream + can safely be read to completion only after the process exits). """ binary = _resolve_steno_diarize() if not binary: return None try: - result = subprocess.run( + proc = subprocess.Popen( [binary, str(channel_path)], - capture_output=True, timeout=timeout, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - if result.returncode != 0: + stdout_chunks: list[bytes] = [] + stderr_chunks: list[bytes] = [] + + def _read_stdout(): + for chunk in iter(lambda: proc.stdout.read(65536), b""): + stdout_chunks.append(chunk) + + def _read_stderr(): + for chunk in iter(lambda: proc.stderr.read(4096), b""): + stderr_chunks.append(chunk) + + t_out = threading.Thread(target=_read_stdout, daemon=True) + t_err = threading.Thread(target=_read_stderr, daemon=True) + t_out.start() + t_err.start() + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + t_out.join(timeout=2.0) + t_err.join(timeout=2.0) + logger.warning("steno-diarize timed out after %ss", timeout) + return None + t_out.join(timeout=5.0) + t_err.join(timeout=5.0) + + if proc.returncode != 0: + stderr_text = b"".join(stderr_chunks).decode(errors="replace") logger.warning( "steno-diarize exited %s: %s", - result.returncode, result.stderr.decode(errors="replace")[:300], + proc.returncode, stderr_text[:300], ) return None - stdout = result.stdout.decode(errors="replace") - # A known FluidAudio/CoreML warning ("E5RT encountered an STL - # exception... key not found") can print directly to stdout ahead - # of the JSON payload — skip to the first '[' rather than assuming - # stdout is pure JSON. - bracket = stdout.find("[") - if bracket < 0: - logger.warning("steno-diarize produced no JSON output") + stdout = b"".join(stdout_chunks).decode(errors="replace") + # Known FluidAudio/CoreML warnings ("E5RT encountered an STL + # exception... key not found") can print directly to stdout + # BEFORE, BETWEEN, or AFTER the real JSON payload -- skipping to + # the first '[' assumes stdout is pure JSON, which breaks the + # moment any warning text (or an interstitial, incomplete blob) + # appears ahead of the real array. Scan every '[' in stdout with a + # real JSON decoder and keep the LAST array of segment-shaped + # dicts -- better than latching onto the first bracket found. + raw_segments = None + decoder = json.JSONDecoder() + search_from = 0 + while True: + next_bracket = stdout.find("[", search_from) + if next_bracket < 0: + break + try: + candidate, end = decoder.raw_decode(stdout, next_bracket) + except json.JSONDecodeError: + search_from = next_bracket + 1 + continue + if ( + isinstance(candidate, list) and candidate + and all(isinstance(s, dict) and "speakerId" in s for s in candidate) + ): + raw_segments = candidate + search_from = max(end, next_bracket + 1) + if raw_segments is None: + logger.warning( + "steno-diarize produced no usable JSON output " + "(stdout length %d, last 500 chars: %r)", + len(stdout), stdout[-500:], + ) return None - raw_segments = json.loads(stdout[bracket:]) except (subprocess.TimeoutExpired, OSError, ValueError) as e: logger.warning("steno-diarize failed: %s", e) return None @@ -661,8 +781,10 @@ def _cluster_channel_labels(diar_segments: list[dict], legacy_label: str) -> Opt by _resolve_speaker_placeholders. Returns None when diar_segments contains a single (or zero) distinct - speaker — the byte-identical-to-legacy fast path, since there's nothing - to disambiguate. + speaker, OR when one cluster's share of total speaking time is at or + above CHANNEL_DOMINANCE_THRESHOLD — the byte-identical-to-legacy fast + path, since a barely-there second cluster is almost certainly + misdiarization noise rather than a real second speaker. """ speaker_ids = {s["speaker"] for s in diar_segments} if len(speaker_ids) <= 1: @@ -671,6 +793,9 @@ def _cluster_channel_labels(diar_segments: list[dict], legacy_label: str) -> Opt for s in diar_segments: totals[s["speaker"]] += s["end"] - s["start"] dominant = max(totals, key=totals.get) + total_time = sum(totals.values()) + if total_time > 0 and totals[dominant] / total_time >= CHANNEL_DOMINANCE_THRESHOLD: + return None return { sid: (legacy_label if sid == dominant else f"__diar__{legacy_label}__{sid}") for sid in speaker_ids @@ -700,18 +825,29 @@ def _tag_channel_segments( if channel_path is not None: timeout = max(STENO_DIARIZE_TIMEOUT_FLOOR_S, int(duration_seconds or 0)) - diar_segments = _run_steno_diarize(channel_path, timeout) - if diar_segments: - cluster_labels = _cluster_channel_labels(diar_segments, legacy_label) - if cluster_labels: - _assign_asr_segments_to_diar_segments(asr_segments, diar_segments) - diar_tagged = [] - for segment in diar_segments: - text = (segment.get("text") or "").strip() - if text: - diar_tagged.append((segment["start"], cluster_labels[segment["speaker"]], text)) - if diar_tagged: - return diar_tagged + logger.info(f"Diarizing {legacy_label} channel acoustically (up to {timeout}s)...") + print(f"PROGRESS:diarize:{legacy_label}:start", flush=True) + try: + with _heartbeat_while_waiting(f"diarize:{legacy_label}"): + diar_segments = _run_steno_diarize(channel_path, timeout) + if diar_segments: + cluster_labels = _cluster_channel_labels(diar_segments, legacy_label) + if cluster_labels: + _assign_asr_segments_to_diar_segments(asr_segments, diar_segments) + diar_tagged = [] + for segment in diar_segments: + text = (segment.get("text") or "").strip() + if text: + diar_tagged.append((segment["start"], cluster_labels[segment["speaker"]], text)) + if diar_tagged: + logger.info( + f"Diarizing {legacy_label} channel found " + f"{len(set(cluster_labels.values()))} speaker cluster(s)" + ) + return diar_tagged + logger.info(f"Diarizing {legacy_label} channel: falling back to legacy single-speaker labeling") + finally: + print(f"PROGRESS:diarize:{legacy_label}:done", flush=True) legacy_tagged: list[tuple[float, str, str]] = [] for s in asr_segments: @@ -968,6 +1104,11 @@ def _preprocess_audio(self, audio_filepath: Path) -> Tuple[Path, bool]: return audio_filepath, False temp_path = Path(temp_name) try: + # loudnorm's two-pass loudness analysis can take real wall-clock + # time on a long recording, with zero other output in between -- + # without this, the terminal goes silent for that whole stretch + # right after "Saved: ...", which reads as a hang. + logger.info(f"Pre-processing audio (highpass + loudnorm): {audio_filepath.name}...") result = subprocess.run( [ffmpeg, '-y', '-i', str(audio_filepath), '-af', _audio_filter_chain(), @@ -1317,12 +1458,21 @@ def _split_stereo_to_channels(self, audio_filepath: Path) -> Tuple[Optional[Path # Detect channel count via ffmpeg. `-t 0` makes ffmpeg parse the # input header (where the channel layout lives) and exit immediately # without decoding any audio frames — without it, a 1-hour recording - # would actually decode in full just to read metadata. + # would actually decode in full just to read metadata. In practice + # this ISN'T always instant: a WebM written live by MediaRecorder + # (our sysaudio capture path) has no seek index, so on an unusually + # large file ffmpeg's demuxer can still need real time to find the + # first decodable packet. A real ~3.5h recording measured this at + # >15s. Same failure shape _diarised_split_timeout's docstring + # documents for the later full-channel-split step: a too-tight fixed + # timeout here silently drops the whole recording to mono (no + # [You]/[Others]) instead of failing loudly — so this budget needs + # real headroom, not just enough for the common case. try: probe = subprocess.run( [ffmpeg, '-hide_banner', '-t', '0', '-i', str(audio_filepath), '-f', 'null', '-'], - capture_output=True, timeout=15, text=True + capture_output=True, timeout=CHANNEL_DETECT_TIMEOUT_S, text=True ) stderr = probe.stderr or '' channels = _parse_channels_from_ffmpeg_stderr(stderr) diff --git a/tests/test_transcriber_diarisation.py b/tests/test_transcriber_diarisation.py index 298f96ff..532718d8 100644 --- a/tests/test_transcriber_diarisation.py +++ b/tests/test_transcriber_diarisation.py @@ -9,6 +9,7 @@ so a recording where speech starts mid-stream isn't classified as silent. """ +import io import json import math import struct @@ -21,6 +22,7 @@ from src.transcriber import ( BLEED_JACCARD_THRESHOLD, + CHANNEL_DOMINANCE_THRESHOLD, DIARISED_SPLIT_TIMEOUT_S, MIN_RMS_THRESHOLD, STENO_DIARIZE_MERGE_GAP_S, @@ -777,6 +779,25 @@ def test_dominant_speaker_by_total_duration_keeps_legacy_label(self): self.assertEqual(labels["SPEAKER_1"], "You") self.assertEqual(labels["SPEAKER_0"], "__diar__You__SPEAKER_0") + def test_overwhelmingly_dominant_speaker_returns_none(self): + # A tiny misdiarization blip (e.g. a ~0.3s noise artifact) must not + # spawn a phantom second speaker — regression for the real-world + # oversplitting issue observed on single-mic multi-person audio. + segments = [ + {"start": 0.0, "end": 59.7, "speaker": "SPEAKER_0"}, + {"start": 59.7, "end": 60.0, "speaker": "SPEAKER_1"}, # 0.5% of total + ] + self.assertIsNone(_cluster_channel_labels(segments, "You")) + + def test_dominance_ratio_just_under_threshold_still_clusters(self): + total = 100.0 + minor = total * (1 - CHANNEL_DOMINANCE_THRESHOLD) + 0.5 # comfortably above the gate + segments = [ + {"start": 0.0, "end": total - minor, "speaker": "SPEAKER_0"}, + {"start": total - minor, "end": total, "speaker": "SPEAKER_1"}, + ] + self.assertIsNotNone(_cluster_channel_labels(segments, "You")) + class ResolveSpeakerPlaceholdersTests(unittest.TestCase): def test_legacy_labels_are_untouched(self): @@ -811,6 +832,37 @@ def test_no_channel_path_uses_legacy_labeling(self): self.assertEqual(result, [(0.0, "You", "Hi.")]) +class _FakePopen: + """Stand-in for subprocess.Popen, matching only the surface + _run_steno_diarize actually uses: .stdout/.stderr as readable byte + streams (plain io.BytesIO works fine -- the two reader threads each + only ever touch their own stream, so there's no real cross-thread + contention to simulate), .wait(timeout=...), .kill(), .returncode. + """ + + def __init__(self, stdout=b"", stderr=b"", returncode=0, raise_timeout_once=False): + self.stdout = io.BytesIO(stdout) + self.stderr = io.BytesIO(stderr) + self._final_returncode = returncode + self._raise_timeout_once = raise_timeout_once + self.returncode = None + self.killed = False + + def wait(self, timeout=None): + if self._raise_timeout_once and timeout is not None: + self._raise_timeout_once = False + raise subprocess.TimeoutExpired(cmd="steno-diarize", timeout=timeout) + self.returncode = self._final_returncode + return self.returncode + + def kill(self): + self.killed = True + + +def _patch_popen(**kwargs): + return patch("subprocess.Popen", return_value=_FakePopen(**kwargs)) + + class RunStenoDiarizeTests(unittest.TestCase): """_run_steno_diarize must survive the sidecar's real quirks: a diagnostic warning printed to stdout ahead of the JSON payload, and any @@ -827,7 +879,7 @@ def test_parses_json_with_e5rt_warning_prefix_on_stdout(self): ]).encode() stdout = b"E5RT encountered an STL exception. msg = unordered_map::at: key not found." + payload with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ - patch("subprocess.run", return_value=Mock(returncode=0, stdout=stdout, stderr=b"")): + _patch_popen(stdout=stdout, stderr=b"", returncode=0): result = _run_steno_diarize(Path("/fake/mic.wav"), 60) self.assertEqual( result, @@ -837,24 +889,62 @@ def test_parses_json_with_e5rt_warning_prefix_on_stdout(self): ], ) + def test_parses_json_with_trailing_warning_after_payload(self): + # A late CoreML/Metal warning printed to stdout AFTER the JSON + # payload (at teardown) -- json.loads() requires the entire + # remaining string to be clean JSON and raises "Extra data" on + # trailing text, discarding an otherwise-successful diarization + # result. raw_decode() must tolerate this. + payload = json.dumps([{"speakerId": "SPEAKER_0", "start": 0.0, "end": 0.9}]).encode() + stdout = payload + b"\nMetal warning: some late teardown message" + with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ + _patch_popen(stdout=stdout, stderr=b"", returncode=0): + result = _run_steno_diarize(Path("/fake/mic.wav"), 60) + self.assertEqual(result, [{"start": 0.0, "end": 0.9, "speaker": "SPEAKER_0"}]) + + def test_skips_an_interstitial_array_that_is_not_segment_shaped(self): + # A non-payload array (no "speakerId" keys) printed BEFORE the real + # payload must not be mistaken for it -- keep scanning for an array + # of segment-shaped dicts. + real_payload = json.dumps([{"speakerId": "SPEAKER_0", "start": 0.0, "end": 0.9}]).encode() + stdout = b'["not", "a", "segment"]\n' + real_payload + with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ + _patch_popen(stdout=stdout, stderr=b"", returncode=0): + result = _run_steno_diarize(Path("/fake/mic.wav"), 60) + self.assertEqual(result, [{"start": 0.0, "end": 0.9, "speaker": "SPEAKER_0"}]) + + def test_prefers_the_last_matching_payload_when_multiple_exist(self): + # If more than one array in stdout looks segment-shaped (shouldn't + # normally happen, but the scan must have a defined, sane tie-break + # rather than an arbitrary one) -- the real payload is printed once, + # at the end, when diarization actually finishes, so prefer the + # LAST match. + first_payload = json.dumps([{"speakerId": "SPEAKER_0", "start": 0.0, "end": 0.9}]).encode() + second_payload = json.dumps([{"speakerId": "SPEAKER_1", "start": 5.0, "end": 6.0}]).encode() + stdout = first_payload + b"\n" + second_payload + with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ + _patch_popen(stdout=stdout, stderr=b"", returncode=0): + result = _run_steno_diarize(Path("/fake/mic.wav"), 60) + self.assertEqual(result, [{"start": 5.0, "end": 6.0, "speaker": "SPEAKER_1"}]) + def test_nonzero_exit_returns_none(self): with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ - patch("subprocess.run", return_value=Mock(returncode=1, stdout=b"", stderr=b"boom")): + _patch_popen(stdout=b"", stderr=b"boom", returncode=1): self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60)) def test_unparseable_json_returns_none(self): with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ - patch("subprocess.run", return_value=Mock(returncode=0, stdout=b"[not json", stderr=b"")): + _patch_popen(stdout=b"[not json", stderr=b"", returncode=0): self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60)) def test_no_bracket_in_stdout_returns_none(self): with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ - patch("subprocess.run", return_value=Mock(returncode=0, stdout=b"nothing useful", stderr=b"")): + _patch_popen(stdout=b"nothing useful", stderr=b"", returncode=0): self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60)) def test_timeout_returns_none(self): with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \ - patch("subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="steno-diarize", timeout=60)): + _patch_popen(raise_timeout_once=True): self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60)) From 33d7639088961ffe219724552eae35dd2920fc4b Mon Sep 17 00:00:00 2001 From: Valentin Weyer Date: Fri, 31 Jul 2026 00:01:29 +0200 Subject: [PATCH 05/12] feat(processing): live per-stage progress for diarization Renders the PROGRESS:diarize:{label}:start/:done markers (added to the backend in the previous commit) as a real UI stage instead of a static "Analyzing transcript" spinner sitting through a diarization pass that can run for minutes on a long recording. - New 'diarizing' stage with an elapsed-time ticker (this branch's sidecar has no per-chunk checkpoint to report a percentage from, so a plain "(Ns)" counter is the only way to show the stage is alive). - Fixes a real bug the new diarize progress lines would otherwise hit: the processingProgress handler used to key off ANY PROGRESS: line unconditionally to flip transcribing -> summarizing; without a prefix check, a PROGRESS:diarize:* line would have prematurely jumped the stage to "summarizing" while diarization was still running. Now branches on the PROGRESS:summarize:/PROGRESS:diarize: prefix explicitly. - Clears chunkProgress on every stage transition (summarize-complete, processing-complete, retry) so a stale diarizing/summarizing sub-label can't leak into finalizing/error/a retried run. - main.js: PROGRESS:diarize:* markers now persisted to the on-disk pipeline log (already true for HEARTBEAT); the live renderer forward needed no change since the existing PROGRESS: forwarder is generic. - New processing-stages.t1.spec.ts (mock IPC, real webContents.send events) -- Processing.tsx had zero test coverage before this. --- app/main.js | 7 ++ app/renderer/src/routes/Processing.tsx | 75 +++++++++++++--- e2e/specs/processing-stages.t1.spec.ts | 117 +++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 12 deletions(-) create mode 100644 e2e/specs/processing-stages.t1.spec.ts diff --git a/app/main.js b/app/main.js index 82d35e97..7b732eab 100644 --- a/app/main.js +++ b/app/main.js @@ -7098,6 +7098,13 @@ function logPipelineStdoutLine(line, source) { processingLog.logLine(source, l); return; } + if (l.startsWith('PROGRESS:diarize:')) { + // Only :start/:done markers exist on this pipeline (rare, at most twice + // per channel) -- no per-chunk flood risk, so no throttle needed unlike + // HEARTBEAT above. + processingLog.logLine(source, l); + return; + } if ( l.startsWith('TRANSCRIPTION_COMPLETE') || l.startsWith('TRANSCRIPTION_FAILED') || diff --git a/app/renderer/src/routes/Processing.tsx b/app/renderer/src/routes/Processing.tsx index 91c82d6d..668ad635 100644 --- a/app/renderer/src/routes/Processing.tsx +++ b/app/renderer/src/routes/Processing.tsx @@ -16,7 +16,7 @@ import { getLiveDraft, useLiveDraftStore } from '@/hooks/liveDraftStore'; import { ipc } from '@/lib/ipc'; import { stripReasoning } from '@/lib/markdown'; -type ProcessingStage = 'transcribing' | 'summarizing' | 'finalizing' | 'error'; +type ProcessingStage = 'transcribing' | 'diarizing' | 'summarizing' | 'finalizing' | 'error'; // Shared flag so the sibling ProcessingDock (the bottom "Processing" chip, // rendered by App while on this route) can hide once the watchdog concludes @@ -33,6 +33,7 @@ const useProcessingWatchdogStore = create<{ const STAGE_LABEL: Record = { transcribing: 'Analyzing transcript', + diarizing: 'Identifying speakers', summarizing: 'Generating notes', finalizing: 'Almost done…', error: 'Couldn’t process this recording.', @@ -59,6 +60,13 @@ const STAGE_LABEL: Record = { const WATCHDOG_TICK_MS = 1500; const WATCHDOG_IDLE_TICKS = 8; +function formatElapsedSeconds(totalSeconds: number): string { + if (totalSeconds < 60) return `${totalSeconds}s`; + const m = Math.floor(totalSeconds / 60); + const s = totalSeconds % 60; + return `${m}m ${s}s`; +} + export function Processing() { const navigate = useNavigate(); const recording = useRecording(); @@ -143,6 +151,21 @@ export function Processing() { setRetryError(null); } + // Diarization's segmentation pass has no per-chunk checkpoint to report + // progress from -- unlike embedding extraction (not part of this branch's + // sidecar), so there is nothing to turn into a percentage here. A + // client-side elapsed-time ticker is the only way to show visible motion + // during that stretch without a backend change. + const diarizeTimerRef = React.useRef | null>(null); + const diarizeStartedAtRef = React.useRef(null); + const clearDiarizeTimer = React.useCallback(() => { + if (diarizeTimerRef.current) { + clearInterval(diarizeTimerRef.current); + diarizeTimerRef.current = null; + } + }, []); + React.useEffect(() => () => clearDiarizeTimer(), [clearDiarizeTimer]); + // Buffer streamed chunks and flush at most every 50ms (~20fps). At a // typical token rate of 30-60 tokens/sec, this batches ~3 tokens per // commit which keeps the UI smooth without re-parsing the entire markdown @@ -172,8 +195,9 @@ export function Processing() { // "saw activity" flag (which would hide a real no-job dead-end). if (e.summaryFile) return; if (activeSession && e.sessionName !== activeSession) return; + clearDiarizeTimer(); pendingChunkRef.current += e.chunk; - setStage((s) => (s === 'transcribing' ? 'summarizing' : s)); + setStage((s) => (s === 'transcribing' || s === 'diarizing' ? 'summarizing' : s)); if (!flushTimerRef.current) { flushTimerRef.current = setTimeout(flushPending, 50); } @@ -188,12 +212,16 @@ export function Processing() { // fresh-recording job. if (e.summaryFile) return; if (activeSession && e.sessionName !== activeSession) return; + clearDiarizeTimer(); + setChunkProgress(null); setStage((s) => (s === 'error' ? s : 'finalizing')); }), ipc().on.processingComplete((e) => { if (activeSession && e.sessionName !== activeSession) return; if (!e.success) { setRetryAudioFile(e.audioFile ?? null); + clearDiarizeTimer(); + setChunkProgress(null); setStage('error'); return; } @@ -214,20 +242,41 @@ export function Processing() { // DIFFERENT meeting emits summaryFile-scoped progress — ignore those so // another meeting's "Summarizing part N" can't hijack this screen's stage. if (e.summaryFile) return; - const raw = e.line.replace(/^PROGRESS:summarize:/, ''); - if (raw === 'reducing') { - setChunkProgress('Merging summaries…'); - } else { - const [step, total] = raw.split('/').map(Number); - if (!Number.isNaN(step) && !Number.isNaN(total)) { - setChunkProgress(`Summarizing part ${step} of ${total}…`); + if (e.line.startsWith('PROGRESS:summarize:')) { + const raw = e.line.slice('PROGRESS:summarize:'.length); + if (raw === 'reducing') { + setChunkProgress('Merging summaries…'); + } else { + const [step, total] = raw.split('/').map(Number); + if (!Number.isNaN(step) && !Number.isNaN(total)) { + setChunkProgress(`Summarizing part ${step} of ${total}…`); + } + } + setStage((s) => (s === 'transcribing' || s === 'diarizing' ? 'summarizing' : s)); + } else if (e.line.startsWith('PROGRESS:diarize:')) { + const rest = e.line.slice('PROGRESS:diarize:'.length); + const [label, kind] = rest.split(':'); + if (kind === 'start') { + setStage((s) => (s === 'transcribing' ? 'diarizing' : s)); + clearDiarizeTimer(); + diarizeStartedAtRef.current = Date.now(); + setChunkProgress(`Diarizing ${label} channel…`); + diarizeTimerRef.current = setInterval(() => { + if (diarizeStartedAtRef.current === null) return; + const elapsedS = Math.floor((Date.now() - diarizeStartedAtRef.current) / 1000); + setChunkProgress(`Diarizing ${label} channel… (${formatElapsedSeconds(elapsedS)})`); + }, 1000); + } else if (kind === 'done') { + // Stop the elapsed ticker; no other UI action needed -- the + // next channel's :start, or the first summarize: chunk, + // supersedes this sub-label shortly after. + clearDiarizeTimer(); } } - setStage((s) => (s === 'transcribing' ? 'summarizing' : s)); }), ]; return () => offs.forEach((fn) => fn()); - }, [activeSession, updateMeeting]); + }, [activeSession, updateMeeting, clearDiarizeTimer]); // Watchdog. Any real processing activity — a streamed chunk, chunk progress, // a stage move past 'transcribing' (summaryComplete/processingComplete) — @@ -523,10 +572,12 @@ function StageCard({ style={{ color: 'var(--fg-2)' }} /> - {chunkProgress && stage === 'summarizing' ? chunkProgress : STAGE_LABEL[stage]} + {chunkProgress && stage !== 'finalizing' && stage !== 'error' ? chunkProgress : STAGE_LABEL[stage]}
diff --git a/e2e/specs/processing-stages.t1.spec.ts b/e2e/specs/processing-stages.t1.spec.ts new file mode 100644 index 00000000..48990efc --- /dev/null +++ b/e2e/specs/processing-stages.t1.spec.ts @@ -0,0 +1,117 @@ +import { test, expect } from '../fixtures/electron'; +import type { ElectronApplication, Page } from '@playwright/test'; + +/** + * T1 -- renderer-only, mock IPC, no backend. Processing.tsx had zero test + * coverage of any kind before this spec (confirmed via a repo-wide search). + * The multi-stage transition logic it drives -- transcribing -> diarizing + * (per-channel) -> summarizing -> finalizing/error, all sharing one + * `chunkProgress` sub-label state across three different stages -- is + * exactly the kind of thing that's easy to get a stale-label leak wrong, so + * it earns T1 coverage per CLAUDE.md's "the interaction itself is the risk" + * carve-out. + * + * Drives the real PROGRESS:, summary-chunk/complete, and processing-complete + * IPC events directly via ElectronApplication.evaluate (main-process webContents.send), + * rather than adding any test-only production code to main.js/preload.js -- + * this is a one-way push protocol, so there's nothing for a renderer-side + * mock invoke to intercept. + */ + +async function openProcessing(page: Page) { + await page.evaluate(() => { + window.location.hash = '/meetings/processing'; + }); + await expect(page.getByTestId('processing-stage-label')).toBeVisible(); +} + +function emit(app: ElectronApplication, channel: string, payload: unknown) { + return app.evaluate( + ({ BrowserWindow }, arg) => { + const win = BrowserWindow.getAllWindows()[0]; + win.webContents.send(arg.channel, arg.payload); + }, + { channel, payload }, + ); +} + +test('walks transcribing -> diarizing -> summarizing -> finalizing without leaking a stale sub-label', async ({ launchApp }) => { + const { app, page } = await launchApp({ mockIpc: true }); + await openProcessing(page); + + const label = page.getByTestId('processing-stage-label'); + await expect(label).toHaveText('Analyzing transcript'); + await expect(label).toHaveAttribute('data-stage', 'transcribing'); + + await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }); + await expect(label).toHaveText('Diarizing You channel…'); + await expect(label).toHaveAttribute('data-stage', 'diarizing'); + + await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:done' }); + await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:Others:start' }); + await expect(label).toHaveText('Diarizing Others channel…'); + await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:Others:done' }); + + await emit(app, 'processing-progress', { line: 'PROGRESS:summarize:1/3' }); + await expect(label).toHaveText('Summarizing part 1 of 3…'); + await expect(label).toHaveAttribute('data-stage', 'summarizing'); + + await emit(app, 'processing-progress', { line: 'PROGRESS:summarize:reducing' }); + await expect(label).toHaveText('Merging summaries…'); + + // The bug this spec exists to catch: chunkProgress used to persist + // unconditionally across stage transitions, so a stale sub-label (here, + // "Merging summaries…") could leak into 'finalizing'. + await emit(app, 'summary-complete', { success: true, sessionName: 'test-session' }); + await expect(label).toHaveText('Almost done…'); + await expect(label).toHaveAttribute('data-stage', 'finalizing'); +}); + +test('shows a ticking elapsed-time counter throughout diarization, since this branch has no per-chunk percentage', async ({ launchApp }) => { + // Real bug found via a live app test: on a real ~21-minute recording, + // segmentation (the diarizer's own pass) took 100+ seconds with the label + // frozen on "Diarizing You channel…" the whole time -- indistinguishable + // from actually being stuck. The user quit the app twice. This ticker is + // the fix; this branch's sidecar has no per-chunk checkpoint at all, so + // the ticker runs for the whole diarization call rather than yielding to + // a percentage partway through. + const { app, page } = await launchApp({ mockIpc: true }); + await openProcessing(page); + + const label = page.getByTestId('processing-stage-label'); + await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }); + await expect(label).toHaveText('Diarizing You channel…'); + + await expect(label).toHaveText(/Diarizing You channel… \(\d+s\)/, { timeout: 3000 }); + + await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:done' }); +}); + +test('a processing failure swaps to the error panel, and retrying does not leak the stale sub-label back', async ({ launchApp }) => { + const { app, page } = await launchApp({ mockIpc: true }); + await openProcessing(page); + + const label = page.getByTestId('processing-stage-label'); + await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }); + await expect(label).toHaveText('Diarizing You channel…'); + + // On failure, the stage card is replaced entirely by a distinct error + // panel (retry button, no processing-stage-label) rather than the label + // just changing text in place. + await emit(app, 'processing-complete', { + success: false, + sessionName: 'test-session', + message: 'boom', + }); + await expect(label).toHaveCount(0); + await expect(page.getByText('Couldn’t process this recording.')).toBeVisible(); + + // The bug this half of the spec exists to catch: without clearing + // chunkProgress on the failure transition, clicking "Try again" (which + // resets the stage back to 'transcribing') would remount the stage card + // still showing the stale "Diarizing You channel…" sub-label instead of + // the correct fresh-start label. + await page.getByRole('button', { name: 'Try again' }).click(); + await expect(label).toHaveText('Analyzing transcript'); + await expect(label).toHaveAttribute('data-stage', 'transcribing'); +}); From 70674a931566a6500f1a3b4d81d77ac8514ab531 Mon Sep 17 00:00:00 2001 From: Valentin Weyer Date: Fri, 31 Jul 2026 00:11:14 +0200 Subject: [PATCH 06/12] fix(e2e): seed a real recording session in processing-stages.t1 canRetry (Processing.tsx) requires both retryAudioFile (from processing-complete's audioFile field) and activeSession (from recording.sessionName) to be truthy. The spec reached /meetings/processing via a bare URL hash with no active mock recording, so activeSession stayed null and the retry-button assertion hung waiting on a permanently-disabled button. Start a mock recording first, matching how the screen is actually reached in real usage, and include audioFile in the failure payload. --- e2e/specs/processing-stages.t1.spec.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/e2e/specs/processing-stages.t1.spec.ts b/e2e/specs/processing-stages.t1.spec.ts index 48990efc..33080605 100644 --- a/e2e/specs/processing-stages.t1.spec.ts +++ b/e2e/specs/processing-stages.t1.spec.ts @@ -19,6 +19,11 @@ import type { ElectronApplication, Page } from '@playwright/test'; */ async function openProcessing(page: Page) { + // Reached in real usage only via an active recording (never a bare URL + // hash) -- start one first so activeSession/recording.sessionName is + // truthy, matching what canRetry (retryAudioFile && activeSession) needs + // for the retry-button test below. + await page.evaluate(() => window.stenoai.recording.start('test-session')); await page.evaluate(() => { window.location.hash = '/meetings/processing'; }); @@ -101,6 +106,7 @@ test('a processing failure swaps to the error panel, and retrying does not leak await emit(app, 'processing-complete', { success: false, sessionName: 'test-session', + audioFile: '/fake/audio.wav', message: 'boom', }); await expect(label).toHaveCount(0); From 8f3863a4e8e5bca5a6adb4936707ceac90f65def Mon Sep 17 00:00:00 2001 From: Valentin Weyer Date: Sat, 1 Aug 2026 20:19:43 +0200 Subject: [PATCH 07/12] fix(e2e): make isSayAvailable() check real synthesis, fix impossible invariant CI found this failing 3/3 times on the macOS T2 pipeline lane, not a flake. isSayAvailable() only probed `say -v ?` (listing voices), which exits 0 even on a runner where `say` can't actually synthesize speech -- observed producing ~170 bytes of near-silent PCM (~5ms) instead of real audio, presumably missing voice assets. That let micBoundarySeconds come out at ~0.5s instead of the several seconds a real sentence takes, which fed straight into a tailSeconds formula with an unconditional 1.5s floor (Math.max(1.5, ...)) that made the very next assertion (tailSeconds < micBoundarySeconds) mathematically impossible to satisfy below a 1.5s boundary. Passed locally only because real speech synthesis on a real Mac comfortably exceeds that. Fixes both: isSayAvailable() now synthesizes a short real phrase and measures what it actually wrote, skipping loudly (existing test.skip path) when it's implausibly short, rather than trusting a voice-list probe that doesn't exercise synthesis at all. tailSeconds is now a bounded fraction of micBoundarySeconds with no floor above it, so the invariant holds for any micBoundarySeconds > 0 -- not just relying on the environment guard to keep it out of the impossible range. --- e2e/fixtures/say-stereo-wav.ts | 39 +++++++++++++++--------- e2e/specs/speaker-diarization.t2.spec.ts | 11 +++++-- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/e2e/fixtures/say-stereo-wav.ts b/e2e/fixtures/say-stereo-wav.ts index c1d4a3e3..3045c858 100644 --- a/e2e/fixtures/say-stereo-wav.ts +++ b/e2e/fixtures/say-stereo-wav.ts @@ -13,20 +13,6 @@ import { mkdtempSync, readFileSync, writeFileSync } from 'fs'; import { tmpdir } from 'os'; import path from 'path'; -let sayAvailable: boolean | undefined; - -export function isSayAvailable(): boolean { - if (sayAvailable === undefined) { - try { - execFileSync('say', ['-v', '?'], { stdio: 'ignore' }); - sayAvailable = true; - } catch { - sayAvailable = false; - } - } - return sayAvailable; -} - // `say -o out.wav --data-format=LEI16@16000` writes a real (non-canonical) // WAV header — macOS prepends a JUNK chunk before `fmt `/`data`, so the // data chunk is not at the fixed 44-byte offset make-wav.js assumes. @@ -44,6 +30,31 @@ function synthesize(text: string, destPath: string): Buffer { return readPcm16Mono(destPath); } +// A real sentence should take at least this long to speak. Guards against a +// `say` that "works" (exits 0, produces a well-formed WAV) but synthesizes +// essentially nothing -- observed on GitHub-hosted macOS runners: `say -v ?` +// (listing voices) exits 0 even when actual synthesis produces ~170 bytes of +// near-silent PCM (~5ms), presumably because the runner image ships the +// `say` binary without voice assets. Probing `-v ?` alone is a false +// positive; this measures what `say` actually wrote. +const MIN_SAY_DURATION_SECONDS = 1.0; + +let sayAvailable: boolean | undefined; + +export function isSayAvailable(): boolean { + if (sayAvailable === undefined) { + try { + const dir = mkdtempSync(path.join(tmpdir(), 'stenoai-e2e-say-probe-')); + const pcm = synthesize('Testing one two three.', path.join(dir, 'probe.wav')); + const seconds = pcm.length / 2 / 16000; + sayAvailable = seconds >= MIN_SAY_DURATION_SECONDS; + } catch { + sayAvailable = false; + } + } + return sayAvailable; +} + function silencePcm(seconds: number): Buffer { return Buffer.alloc(Math.round(seconds * 16000) * 2); } diff --git a/e2e/specs/speaker-diarization.t2.spec.ts b/e2e/specs/speaker-diarization.t2.spec.ts index 6f2915a4..60f3cb2d 100644 --- a/e2e/specs/speaker-diarization.t2.spec.ts +++ b/e2e/specs/speaker-diarization.t2.spec.ts @@ -62,8 +62,15 @@ test('@pipeline synthesized two-speaker mic channel becomes You + Speaker 2', as // independent of which channel WAV it's actually pointed at. The tail // cluster is deliberately shorter than the boundary so SPEAKER_0 always // stays the dominant (most total speaking time) cluster on BOTH - // channels — sanity-checked below rather than assumed. - const tailSeconds = Math.max(1.5, Math.min(3.0, micBoundarySeconds * 0.6)); + // channels — sanity-checked below rather than assumed. Expressed as a + // fraction of micBoundarySeconds with no floor above it, so the + // invariant below holds for any micBoundarySeconds > 0 -- a previous + // version floored this at 1.5s, which made the assertion mathematically + // impossible whenever isSayAvailable()'s probe let through a runner + // where `say` produced near-silent audio (micBoundarySeconds well under + // 1.5s). isSayAvailable() now measures real synthesized duration, so + // that shouldn't recur, but this formula no longer depends on it either. + const tailSeconds = Math.min(micBoundarySeconds * 0.6, 3.0); expect(tailSeconds).toBeLessThan(micBoundarySeconds); const fixtureDir = mkdtempSync(path.join(tmpdir(), 'stenoai-e2e-diarize-')); From e129c0a71ca7150ecd2b84c9f09c8fbe947af8a3 Mon Sep 17 00:00:00 2001 From: Valentin Weyer Date: Sat, 1 Aug 2026 20:24:21 +0200 Subject: [PATCH 08/12] docs: update speaker-label docs for per-channel acoustic diarization This PR makes two existing statements false. docs/faq.mdx's "Can Steno record in-person meetings?" said in-person recordings have no speaker labels at all -- after this PR, the mono/mic-only path is exactly where acoustic diarization gets used, so that's now the headline case that gains labels, not the one that lacks them. docs/features/recording.mdx's "Speaker labels" section had the same gap from the other direction: it described labeling as something that only happens when system audio is on (the [You]/[Others] channel split), omitting the new within-channel acoustic split entirely. Both now describe the real constraint precisely: up to four distinct voices per channel, not four total, since diarization runs independently on each channel. Framed that way because it matters for the common case -- a two-person call is one person per channel, comfortably under the per-channel limit either side, so most users never approach it, but a flat "four speakers" would incorrectly suggest otherwise. --- docs/faq.mdx | 2 +- docs/features/recording.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.mdx b/docs/faq.mdx index ebc58707..a7c215ee 100644 --- a/docs/faq.mdx +++ b/docs/faq.mdx @@ -90,7 +90,7 @@ Steno can capture system audio -- the audio playing through your Mac's speakers -Yes. Steno records from your Mac's microphone, which will pick up voices in the room. For best results, place your laptop centrally. In-person recordings are mic-only, so they have no `[You]` / `[Others]` speaker labels. +Yes. Steno records from your Mac's microphone, which will pick up voices in the room. For best results, place your laptop centrally. Steno separates up to four distinct voices per recording channel, so an in-person meeting of four or fewer people gets per-speaker labels; beyond that, additional speakers are merged into the four it detects. diff --git a/docs/features/recording.mdx b/docs/features/recording.mdx index 391f3c1a..fd36730f 100644 --- a/docs/features/recording.mdx +++ b/docs/features/recording.mdx @@ -28,7 +28,7 @@ After this one-time setup, the toggle stays on for future recordings until you t ## Speaker labels -When Steno captures system audio, it labels lines in the transcript as `[You]` (microphone audio) or `[Others]` (system audio). This makes it easier to follow who said what in a virtual meeting. +Steno labels transcript lines by who's speaking. When system audio is on, lines are split into `[You]` (microphone audio) and `[Others]` (system audio) by channel. Within either channel, Steno also separates distinct voices acoustically, so an in-person meeting with several people sharing your Mac's microphone, or several remote participants on the system-audio side, get their own `[Speaker 2]`, `[Speaker 3]`, and so on. Each channel can distinguish up to four voices this way; additional speakers beyond that are merged into the four it detects. Speaker labels appear automatically -- no configuration is needed. From 2060434fae3e460fe9b134b4087147188d2c60db Mon Sep 17 00:00:00 2001 From: Valentin Weyer Date: Mon, 3 Aug 2026 21:13:26 +0200 Subject: [PATCH 09/12] fix(e2e): settle Processing's generation bump before emitting in processing-stages.t1 openProcessing() returned before the queue poll's first report of the mock recording finished bumping Processing.tsx's `generation`, whose render-phase reset (:143-152) then cleared a caller's terminal stage (setStage('error')) one round-trip later. Wait for the session name to paint in the header first, so the reset has already landed before any event is emitted. Root-caused and verified (9/9 green, 3 repeats) by Ben/Optic00 on PR #455. --- e2e/specs/processing-stages.t1.spec.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/e2e/specs/processing-stages.t1.spec.ts b/e2e/specs/processing-stages.t1.spec.ts index 33080605..f2c109cf 100644 --- a/e2e/specs/processing-stages.t1.spec.ts +++ b/e2e/specs/processing-stages.t1.spec.ts @@ -28,6 +28,11 @@ async function openProcessing(page: Page) { window.location.hash = '/meetings/processing'; }); await expect(page.getByTestId('processing-stage-label')).toBeVisible(); + // The queue poll that first reports this recording bumps Processing's + // `generation`, whose render-phase reset clears stage + retryAudioFile. + // Wait until that has landed (the header switches from 'Note' to the + // session name) so a later emit can't race the reset. + await expect(page.getByRole('heading', { name: 'test-session' })).toBeVisible(); } function emit(app: ElectronApplication, channel: string, payload: unknown) { From cac5eaa964d897c05b1f20d3f173b14042b365b1 Mon Sep 17 00:00:00 2001 From: Valentin Weyer Date: Mon, 3 Aug 2026 22:08:12 +0200 Subject: [PATCH 10/12] fix(e2e): retry emitted IPC events in processing-stages.t1 until they land The generation-settle wait fixed the error-panel race but exposed a different, pre-existing one: the renderer attaches its IPC listeners in a useEffect that runs after the initial paint, so an event sent right after the DOM updates can land in the gap before that effect mounts and be silently dropped (webContents.send has no queueing/replay). CI's runner is apparently slower/more loaded than local, tipping this from theoretical into a real intermittent failure across all three tests in the file. emitUntil() resends on a short interval until the expected UI change is actually observed, rather than firing once and hoping. Safe to repeat: every handler this spec drives is idempotent once its target state is reached, and the loop stops as soon as the check passes. --- e2e/specs/processing-stages.t1.spec.ts | 82 +++++++++++++++++++------- 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/e2e/specs/processing-stages.t1.spec.ts b/e2e/specs/processing-stages.t1.spec.ts index f2c109cf..9807bbd2 100644 --- a/e2e/specs/processing-stages.t1.spec.ts +++ b/e2e/specs/processing-stages.t1.spec.ts @@ -45,6 +45,35 @@ function emit(app: ElectronApplication, channel: string, payload: unknown) { ); } +// The renderer attaches its IPC listeners in a useEffect that runs after the +// initial paint, so there is a real window -- brief locally, wider under a +// loaded CI runner -- where `processing-stage-label` is already visible but +// nothing is listening yet. webContents.send has no queueing/replay, so an +// event landing in that window is silently dropped. Resend on a short +// interval until `check` actually observes the expected UI change, rather +// than sending once and hoping; safe to repeat because every handler this +// spec drives is idempotent once its target state is reached (a resend after +// `check` has already passed never happens -- the loop returns as soon as it +// does). +async function emitUntil( + app: ElectronApplication, + channel: string, + payload: unknown, + check: () => Promise, +) { + const deadline = Date.now() + 5000; + for (;;) { + await emit(app, channel, payload); + try { + await check(); + return; + } catch (err) { + if (Date.now() > deadline) throw err; + await new Promise((r) => setTimeout(r, 100)); + } + } +} + test('walks transcribing -> diarizing -> summarizing -> finalizing without leaking a stale sub-label', async ({ launchApp }) => { const { app, page } = await launchApp({ mockIpc: true }); await openProcessing(page); @@ -53,27 +82,32 @@ test('walks transcribing -> diarizing -> summarizing -> finalizing without leaki await expect(label).toHaveText('Analyzing transcript'); await expect(label).toHaveAttribute('data-stage', 'transcribing'); - await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }); - await expect(label).toHaveText('Diarizing You channel…'); + await emitUntil(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }, () => + expect(label).toHaveText('Diarizing You channel…', { timeout: 200 }), + ); await expect(label).toHaveAttribute('data-stage', 'diarizing'); await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:done' }); - await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:Others:start' }); - await expect(label).toHaveText('Diarizing Others channel…'); + await emitUntil(app, 'processing-progress', { line: 'PROGRESS:diarize:Others:start' }, () => + expect(label).toHaveText('Diarizing Others channel…', { timeout: 200 }), + ); await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:Others:done' }); - await emit(app, 'processing-progress', { line: 'PROGRESS:summarize:1/3' }); - await expect(label).toHaveText('Summarizing part 1 of 3…'); + await emitUntil(app, 'processing-progress', { line: 'PROGRESS:summarize:1/3' }, () => + expect(label).toHaveText('Summarizing part 1 of 3…', { timeout: 200 }), + ); await expect(label).toHaveAttribute('data-stage', 'summarizing'); - await emit(app, 'processing-progress', { line: 'PROGRESS:summarize:reducing' }); - await expect(label).toHaveText('Merging summaries…'); + await emitUntil(app, 'processing-progress', { line: 'PROGRESS:summarize:reducing' }, () => + expect(label).toHaveText('Merging summaries…', { timeout: 200 }), + ); // The bug this spec exists to catch: chunkProgress used to persist // unconditionally across stage transitions, so a stale sub-label (here, // "Merging summaries…") could leak into 'finalizing'. - await emit(app, 'summary-complete', { success: true, sessionName: 'test-session' }); - await expect(label).toHaveText('Almost done…'); + await emitUntil(app, 'summary-complete', { success: true, sessionName: 'test-session' }, () => + expect(label).toHaveText('Almost done…', { timeout: 200 }), + ); await expect(label).toHaveAttribute('data-stage', 'finalizing'); }); @@ -89,8 +123,9 @@ test('shows a ticking elapsed-time counter throughout diarization, since this br await openProcessing(page); const label = page.getByTestId('processing-stage-label'); - await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }); - await expect(label).toHaveText('Diarizing You channel…'); + await emitUntil(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }, () => + expect(label).toHaveText('Diarizing You channel…', { timeout: 200 }), + ); await expect(label).toHaveText(/Diarizing You channel… \(\d+s\)/, { timeout: 3000 }); @@ -102,20 +137,25 @@ test('a processing failure swaps to the error panel, and retrying does not leak await openProcessing(page); const label = page.getByTestId('processing-stage-label'); - await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }); - await expect(label).toHaveText('Diarizing You channel…'); + await emitUntil(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }, () => + expect(label).toHaveText('Diarizing You channel…', { timeout: 200 }), + ); // On failure, the stage card is replaced entirely by a distinct error // panel (retry button, no processing-stage-label) rather than the label // just changing text in place. - await emit(app, 'processing-complete', { - success: false, - sessionName: 'test-session', - audioFile: '/fake/audio.wav', - message: 'boom', - }); + await emitUntil( + app, + 'processing-complete', + { + success: false, + sessionName: 'test-session', + audioFile: '/fake/audio.wav', + message: 'boom', + }, + () => expect(page.getByText('Couldn’t process this recording.')).toBeVisible({ timeout: 200 }), + ); await expect(label).toHaveCount(0); - await expect(page.getByText('Couldn’t process this recording.')).toBeVisible(); // The bug this half of the spec exists to catch: without clearing // chunkProgress on the failure transition, clicking "Try again" (which From 33369815f83ca0c448acaa37e3778173003630a2 Mon Sep 17 00:00:00 2001 From: Valentin Weyer Date: Mon, 3 Aug 2026 22:18:19 +0200 Subject: [PATCH 11/12] Revert "fix(e2e): retry emitted IPC events in processing-stages.t1 until they land" This reverts commit cac5eaa964d897c05b1f20d3f173b14042b365b1. --- e2e/specs/processing-stages.t1.spec.ts | 82 +++++++------------------- 1 file changed, 21 insertions(+), 61 deletions(-) diff --git a/e2e/specs/processing-stages.t1.spec.ts b/e2e/specs/processing-stages.t1.spec.ts index 9807bbd2..f2c109cf 100644 --- a/e2e/specs/processing-stages.t1.spec.ts +++ b/e2e/specs/processing-stages.t1.spec.ts @@ -45,35 +45,6 @@ function emit(app: ElectronApplication, channel: string, payload: unknown) { ); } -// The renderer attaches its IPC listeners in a useEffect that runs after the -// initial paint, so there is a real window -- brief locally, wider under a -// loaded CI runner -- where `processing-stage-label` is already visible but -// nothing is listening yet. webContents.send has no queueing/replay, so an -// event landing in that window is silently dropped. Resend on a short -// interval until `check` actually observes the expected UI change, rather -// than sending once and hoping; safe to repeat because every handler this -// spec drives is idempotent once its target state is reached (a resend after -// `check` has already passed never happens -- the loop returns as soon as it -// does). -async function emitUntil( - app: ElectronApplication, - channel: string, - payload: unknown, - check: () => Promise, -) { - const deadline = Date.now() + 5000; - for (;;) { - await emit(app, channel, payload); - try { - await check(); - return; - } catch (err) { - if (Date.now() > deadline) throw err; - await new Promise((r) => setTimeout(r, 100)); - } - } -} - test('walks transcribing -> diarizing -> summarizing -> finalizing without leaking a stale sub-label', async ({ launchApp }) => { const { app, page } = await launchApp({ mockIpc: true }); await openProcessing(page); @@ -82,32 +53,27 @@ test('walks transcribing -> diarizing -> summarizing -> finalizing without leaki await expect(label).toHaveText('Analyzing transcript'); await expect(label).toHaveAttribute('data-stage', 'transcribing'); - await emitUntil(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }, () => - expect(label).toHaveText('Diarizing You channel…', { timeout: 200 }), - ); + await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }); + await expect(label).toHaveText('Diarizing You channel…'); await expect(label).toHaveAttribute('data-stage', 'diarizing'); await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:done' }); - await emitUntil(app, 'processing-progress', { line: 'PROGRESS:diarize:Others:start' }, () => - expect(label).toHaveText('Diarizing Others channel…', { timeout: 200 }), - ); + await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:Others:start' }); + await expect(label).toHaveText('Diarizing Others channel…'); await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:Others:done' }); - await emitUntil(app, 'processing-progress', { line: 'PROGRESS:summarize:1/3' }, () => - expect(label).toHaveText('Summarizing part 1 of 3…', { timeout: 200 }), - ); + await emit(app, 'processing-progress', { line: 'PROGRESS:summarize:1/3' }); + await expect(label).toHaveText('Summarizing part 1 of 3…'); await expect(label).toHaveAttribute('data-stage', 'summarizing'); - await emitUntil(app, 'processing-progress', { line: 'PROGRESS:summarize:reducing' }, () => - expect(label).toHaveText('Merging summaries…', { timeout: 200 }), - ); + await emit(app, 'processing-progress', { line: 'PROGRESS:summarize:reducing' }); + await expect(label).toHaveText('Merging summaries…'); // The bug this spec exists to catch: chunkProgress used to persist // unconditionally across stage transitions, so a stale sub-label (here, // "Merging summaries…") could leak into 'finalizing'. - await emitUntil(app, 'summary-complete', { success: true, sessionName: 'test-session' }, () => - expect(label).toHaveText('Almost done…', { timeout: 200 }), - ); + await emit(app, 'summary-complete', { success: true, sessionName: 'test-session' }); + await expect(label).toHaveText('Almost done…'); await expect(label).toHaveAttribute('data-stage', 'finalizing'); }); @@ -123,9 +89,8 @@ test('shows a ticking elapsed-time counter throughout diarization, since this br await openProcessing(page); const label = page.getByTestId('processing-stage-label'); - await emitUntil(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }, () => - expect(label).toHaveText('Diarizing You channel…', { timeout: 200 }), - ); + await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }); + await expect(label).toHaveText('Diarizing You channel…'); await expect(label).toHaveText(/Diarizing You channel… \(\d+s\)/, { timeout: 3000 }); @@ -137,25 +102,20 @@ test('a processing failure swaps to the error panel, and retrying does not leak await openProcessing(page); const label = page.getByTestId('processing-stage-label'); - await emitUntil(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }, () => - expect(label).toHaveText('Diarizing You channel…', { timeout: 200 }), - ); + await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' }); + await expect(label).toHaveText('Diarizing You channel…'); // On failure, the stage card is replaced entirely by a distinct error // panel (retry button, no processing-stage-label) rather than the label // just changing text in place. - await emitUntil( - app, - 'processing-complete', - { - success: false, - sessionName: 'test-session', - audioFile: '/fake/audio.wav', - message: 'boom', - }, - () => expect(page.getByText('Couldn’t process this recording.')).toBeVisible({ timeout: 200 }), - ); + await emit(app, 'processing-complete', { + success: false, + sessionName: 'test-session', + audioFile: '/fake/audio.wav', + message: 'boom', + }); await expect(label).toHaveCount(0); + await expect(page.getByText('Couldn’t process this recording.')).toBeVisible(); // The bug this half of the spec exists to catch: without clearing // chunkProgress on the failure transition, clicking "Try again" (which From 05679be9b3538d3048e42029d12f801aff8ce3b7 Mon Sep 17 00:00:00 2001 From: Optic00 Date: Tue, 4 Aug 2026 06:48:50 +0200 Subject: [PATCH 12/12] test(e2e): launch the processing-stages specs with fakeAudio The CI runner has no audio device, so openProcessing()'s recording start fails there and the emitted progress events never take effect. Every other spec in the suite that records already sets this flag. Verified green on the Ubuntu runner from this exact head: 68 passed. --- e2e/specs/processing-stages.t1.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/specs/processing-stages.t1.spec.ts b/e2e/specs/processing-stages.t1.spec.ts index f2c109cf..52827ed0 100644 --- a/e2e/specs/processing-stages.t1.spec.ts +++ b/e2e/specs/processing-stages.t1.spec.ts @@ -46,7 +46,7 @@ function emit(app: ElectronApplication, channel: string, payload: unknown) { } test('walks transcribing -> diarizing -> summarizing -> finalizing without leaking a stale sub-label', async ({ launchApp }) => { - const { app, page } = await launchApp({ mockIpc: true }); + const { app, page } = await launchApp({ mockIpc: true, fakeAudio: true }); await openProcessing(page); const label = page.getByTestId('processing-stage-label'); @@ -85,7 +85,7 @@ test('shows a ticking elapsed-time counter throughout diarization, since this br // the fix; this branch's sidecar has no per-chunk checkpoint at all, so // the ticker runs for the whole diarization call rather than yielding to // a percentage partway through. - const { app, page } = await launchApp({ mockIpc: true }); + const { app, page } = await launchApp({ mockIpc: true, fakeAudio: true }); await openProcessing(page); const label = page.getByTestId('processing-stage-label'); @@ -98,7 +98,7 @@ test('shows a ticking elapsed-time counter throughout diarization, since this br }); test('a processing failure swaps to the error panel, and retrying does not leak the stale sub-label back', async ({ launchApp }) => { - const { app, page } = await launchApp({ mockIpc: true }); + const { app, page } = await launchApp({ mockIpc: true, fakeAudio: true }); await openProcessing(page); const label = page.getByTestId('processing-stage-label');