feat(diarize): per-channel acoustic speaker diarization (Sortformer)

.gitignore

@@ -199,3 +199,8 @@ ollama.pid
 
 # Local design reference library (cloned, not vendored)
 .reference/
+
+# Swift Package Manager build artifacts for the diarize-sidecar helper
+# (fetched FluidAudio checkout + compiled binaries) — scripts/build-diarize-sidecar.sh
+# regenerates this; only diarize-sidecar/{Package.swift,Package.resolved,Sources/} are tracked.
+diarize-sidecar/.build/

CLAUDE.md

@@ -13,7 +13,7 @@ The app is a thin Electron shell over a PyInstaller-bundled Python CLI. There is
 - **Python CLI (`simple_recorder.py`, ~2.9k lines, ~60 click commands)** is the single entry point bundled by `stenoai.spec`. Sub-modules in `src/`: `audio_recorder` (sounddevice), `transcriber` (pywhispercpp), `summarizer` (Ollama HTTP client), `ollama_manager` (lifecycle of the bundled `ollama serve`), `config` (JSON-backed user settings + model registry), `folders`, `models`, `whisper_models`.
 - **State across CLI invocations** is persisted to `recorder_state.json` and similar small JSON files — there is no daemon. Long-running recordings are a `record` subprocess kept alive by the Electron main process.
 - **User data lives in `~/Library/Application Support/stenoai/`** (`recordings/`, `transcripts/`, `output/`), resolved via `src.config.get_data_dirs()`. Repo-root `recordings/`/`transcripts/`/`output/` dirs are dev-only scratch.
-- **Bundled binaries (`bin/`)**: Ollama + ffmpeg, downloaded by `scripts/download-ollama.sh`. PyInstaller copies them into `dist/stenoai/ollama/` and `dist/stenoai/ffmpeg`. Electron then re-bundles `dist/stenoai/` as an `extraResource`.
+- **Bundled binaries (`bin/`)**: Ollama + ffmpeg, downloaded by `scripts/download-ollama.sh`. PyInstaller copies them into `dist/stenoai/ollama/` and `dist/stenoai/ffmpeg`. Electron then re-bundles `dist/stenoai/` as an `extraResource`. `bin/steno-diarize` (macOS only) is a separate Swift/CoreML sidecar built by `scripts/build-diarize-sidecar.sh` — see "Speaker diarization" below.
 - **Deep links**: app registers the `stenoai://` URL scheme. Handler logic is in `app/main.js` near `SHORTCUT_PROTOCOL`. Used by macOS Shortcuts: `stenoai://record/start?name=...` and `stenoai://record/stop`.
 
 ## Development Commands
@@ -39,6 +39,28 @@ The Electron build pulls the bundled backend from `../dist/stenoai` via `extraRe
 
 For setup from a clean checkout, see `CONTRIBUTING.md` and `README.md`.
 
+### Speaker diarization (macOS only)
+Per-channel acoustic speaker diarization (splitting multiple speakers sharing
+one side of a call — e.g. two people around one mic, or multiple remote
+participants on system audio) runs through `bin/steno-diarize`, a Swift/
+CoreML sidecar (`diarize-sidecar/`) wrapping FluidAudio's Sortformer
+diarizer, invoked from Python (`src.transcriber._run_steno_diarize`) — never
+from Electron, since the batch pipeline is entirely Python-orchestrated.
+Build it *before* `pyinstaller stenoai.spec`, same as `download-ollama.sh`:
+
+```
+scripts/build-diarize-sidecar.sh   # outputs bin/steno-diarize
+scripts/download-ollama.sh
+pyinstaller stenoai.spec --noconfirm
+```
+
+`stenoai.spec` bundles `bin/steno-diarize` only when it exists and only on
+macOS (`_IS_DARWIN`), so skipping the build step is safe — the app falls
+back to the legacy channel-only "You"/"Others" labeling whenever the binary
+or a diarization run is unavailable (missing binary, timeout, bad output);
+this never fails a meeting. Windows/Linux never get acoustic diarization —
+`_resolve_steno_diarize()` returns `None` immediately off-darwin.
+
 ### End-to-end tests (Playwright)
 The e2e suite drives the **real Electron app** (real window, real clicks) to catch
 full-app regressions like the org-provider reset before they reach users. It lives
@@ -97,7 +119,14 @@ overrides an agent's own test-level defaults.
     `setup-check.t2` (the setup-wizard allGood + checks contract) (all model-free,
     run in `t2-macos` /
     `t2-windows`); `transcription-pipeline.t2` and `honest-failure.t2` (tagged
-    `@pipeline`, run in `t2-pipeline-macos` / `t2-pipeline-windows`). Engine selection
+    `@pipeline`, run in `t2-pipeline-macos` / `t2-pipeline-windows`), and
+    `speaker-diarization.t2` (also `@pipeline`, macOS-only — skips loudly off-darwin
+    and when macOS's `say` TTS is unavailable — synthesizes real speech via `say` so
+    Parakeet/whisper.cpp produce real ASR segments, points
+    `STENOAI_DIARIZE_SIDECAR_PATH` at a fixture script returning fixed 2-speaker JSON,
+    and asserts the saved transcript's per-channel "You"/"Speaker 2"/"Others" labeling
+    and cross-channel numbering; the real `steno-diarize`/Sortformer binary itself was
+    validated directly against real recordings rather than in CI). Engine selection
     for `@pipeline` specs is shared via `e2e/fixtures/engine.ts`; model-free T2 setup
     helpers (deterministic recording config + seeded meeting summaries) live in
     `e2e/fixtures/user-config.ts`. The core-loop specs drive the preload IPC bridge and

app/main.js

@@ -7098,6 +7098,13 @@ function logPipelineStdoutLine(line, source) {
     processingLog.logLine(source, l);
     return;
   }
+  if (l.startsWith('PROGRESS:diarize:')) {
+    // Only :start/:done markers exist on this pipeline (rare, at most twice
+    // per channel) -- no per-chunk flood risk, so no throttle needed unlike
+    // HEARTBEAT above.
+    processingLog.logLine(source, l);
+    return;
+  }
   if (
     l.startsWith('TRANSCRIPTION_COMPLETE') ||
     l.startsWith('TRANSCRIPTION_FAILED') ||

app/renderer/src/components/TranscriptPanel.tsx

@@ -133,8 +133,10 @@ function TranscriptRow({ segment, highlight }: { segment: Segment; highlight: st
   // showing their content as un-bubbled plain text (or worse, on the
   // "Others" side) reads as wrong. Granola takes the same charitable
   // default — when in doubt, attribute to the mic owner. Explicit
-  // `Others` markers still render as grey/left.
-  const isYou = segment.speaker !== 'Others';
+  // `Others` markers and acoustically-diarized `Speaker N` markers
+  // (transcriber.py's _resolve_speaker_placeholders) both render as
+  // grey/left — only an exact "You" (or no marker at all) is "self".
+  const isYou = segment.speaker === 'You' || segment.speaker == null;
   return (
     <div className={cn('flex flex-col gap-0.5 px-1 py-0.5', isYou ? 'items-end' : 'items-start')}>
       {segment.timestamp && (

app/renderer/src/lib/transcriptSegments.test.ts

@@ -40,6 +40,18 @@ describe('parseTranscript — diarised', () => {
     expect(segs[0].text).toContain('line two');
     expect(segs[1].timestamp).toBe('00:05');
   });
+
+  test('parses acoustically-diarized "Speaker N" labels alongside You/Others', () => {
+    const segs = parseTranscript(
+      '[00:00] [You] Hello\n\n[00:05] [Speaker 2] Hi there\n\n[00:10] [Others] Welcome',
+      true,
+    );
+    expect(segs).toEqual([
+      { speaker: 'You', text: 'Hello', timestamp: '00:00' },
+      { speaker: 'Speaker 2', text: 'Hi there', timestamp: '00:05' },
+      { speaker: 'Others', text: 'Welcome', timestamp: '00:10' },
+    ]);
+  });
 });
 
 describe('parseTranscript — non-diarised', () => {

app/renderer/src/lib/transcriptSegments.ts

@@ -3,28 +3,32 @@
 // unit-tested in isolation, and so the live dock could share it later.
 
 export interface Segment {
-  speaker: 'You' | 'Others' | null;
+  /** 'You' / 'Others' for the channel-only fallback, or 'Speaker N' for an
+   *  acoustically-diarized secondary speaker on a channel (see
+   *  transcriber.py's _resolve_speaker_placeholders). */
+  speaker: string | null;
   text: string;
   /** `MM:SS` / `H:MM:SS` offset parsed from a diarised line's leading
    *  `[MM:SS]` marker (transcriber.py writes it). Absent on older transcripts
    *  saved before timestamps, and on the non-diarised path. */
   timestamp?: string;
 }
 
-// A diarised line: an optional `[MM:SS]` / `[H:MM:SS]` timestamp, then the
-// `[You]`/`[Others]` speaker marker, then the text up to the next marker (or
-// end). Matched globally rather than split so the optional timestamp stays
-// attached to its own segment instead of trailing the previous one.
+// A diarised line: an optional `[MM:SS]` / `[H:MM:SS]` timestamp, then a
+// `[You]` / `[Others]` / `[Speaker N]` speaker marker, then the text up to
+// the next marker (or end). Matched globally rather than split so the
+// optional timestamp stays attached to its own segment instead of trailing
+// the previous one.
 const DIARISED_SEGMENT_RE =
-  /(?:\[(\d{1,3}:\d{2}(?::\d{2})?)\]\s*)?\[(You|Others)\]\s*([\s\S]*?)(?=(?:\[\d{1,3}:\d{2}(?::\d{2})?\]\s*)?\[(?:You|Others)\]|$)/g;
+  /(?:\[(\d{1,3}:\d{2}(?::\d{2})?)\]\s*)?\[([^\]]+)\]\s*([\s\S]*?)(?=(?:\[\d{1,3}:\d{2}(?::\d{2})?\]\s*)?\[[^\]]+\]|$)/g;
 
 export function parseTranscript(text: string, isDiarised: boolean): Segment[] {
   if (isDiarised) {
     const segments: Segment[] = [];
     for (const m of text.matchAll(DIARISED_SEGMENT_RE)) {
       const body = m[3].trim();
       if (!body) continue;
-      segments.push({ speaker: m[2] as 'You' | 'Others', text: body, timestamp: m[1] });
+      segments.push({ speaker: m[2], text: body, timestamp: m[1] });
     }
     return segments;
   }

app/renderer/src/routes/Processing.tsx

@@ -16,7 +16,7 @@ import { getLiveDraft, useLiveDraftStore } from '@/hooks/liveDraftStore';
 import { ipc } from '@/lib/ipc';
 import { stripReasoning } from '@/lib/markdown';
 
-type ProcessingStage = 'transcribing' | 'summarizing' | 'finalizing' | 'error';
+type ProcessingStage = 'transcribing' | 'diarizing' | 'summarizing' | 'finalizing' | 'error';
 
 // Shared flag so the sibling ProcessingDock (the bottom "Processing" chip,
 // rendered by App while on this route) can hide once the watchdog concludes
@@ -33,6 +33,7 @@ const useProcessingWatchdogStore = create<{
 
 const STAGE_LABEL: Record<ProcessingStage, string> = {
   transcribing: 'Analyzing transcript',
+  diarizing: 'Identifying speakers',
   summarizing: 'Generating notes',
   finalizing: 'Almost done…',
   error: 'Couldn’t process this recording.',
@@ -59,6 +60,13 @@ const STAGE_LABEL: Record<ProcessingStage, string> = {
 const WATCHDOG_TICK_MS = 1500;
 const WATCHDOG_IDLE_TICKS = 8;
 
+function formatElapsedSeconds(totalSeconds: number): string {
+  if (totalSeconds < 60) return `${totalSeconds}s`;
+  const m = Math.floor(totalSeconds / 60);
+  const s = totalSeconds % 60;
+  return `${m}m ${s}s`;
+}
+
 export function Processing() {
   const navigate = useNavigate();
   const recording = useRecording();
@@ -143,6 +151,21 @@ export function Processing() {
     setRetryError(null);
   }
 
+  // Diarization's segmentation pass has no per-chunk checkpoint to report
+  // progress from -- unlike embedding extraction (not part of this branch's
+  // sidecar), so there is nothing to turn into a percentage here. A
+  // client-side elapsed-time ticker is the only way to show visible motion
+  // during that stretch without a backend change.
+  const diarizeTimerRef = React.useRef<ReturnType<typeof setInterval> | null>(null);
+  const diarizeStartedAtRef = React.useRef<number | null>(null);
+  const clearDiarizeTimer = React.useCallback(() => {
+    if (diarizeTimerRef.current) {
+      clearInterval(diarizeTimerRef.current);
+      diarizeTimerRef.current = null;
+    }
+  }, []);
+  React.useEffect(() => () => clearDiarizeTimer(), [clearDiarizeTimer]);
+
   // Buffer streamed chunks and flush at most every 50ms (~20fps). At a
   // typical token rate of 30-60 tokens/sec, this batches ~3 tokens per
   // commit which keeps the UI smooth without re-parsing the entire markdown
@@ -172,8 +195,9 @@ export function Processing() {
         // "saw activity" flag (which would hide a real no-job dead-end).
         if (e.summaryFile) return;
         if (activeSession && e.sessionName !== activeSession) return;
+        clearDiarizeTimer();
         pendingChunkRef.current += e.chunk;
-        setStage((s) => (s === 'transcribing' ? 'summarizing' : s));
+        setStage((s) => (s === 'transcribing' || s === 'diarizing' ? 'summarizing' : s));
         if (!flushTimerRef.current) {
           flushTimerRef.current = setTimeout(flushPending, 50);
         }
@@ -188,12 +212,16 @@ export function Processing() {
         // fresh-recording job.
         if (e.summaryFile) return;
         if (activeSession && e.sessionName !== activeSession) return;
+        clearDiarizeTimer();
+        setChunkProgress(null);
         setStage((s) => (s === 'error' ? s : 'finalizing'));
       }),
       ipc().on.processingComplete((e) => {
         if (activeSession && e.sessionName !== activeSession) return;
         if (!e.success) {
           setRetryAudioFile(e.audioFile ?? null);
+          clearDiarizeTimer();
+          setChunkProgress(null);
           setStage('error');
           return;
         }
@@ -214,20 +242,41 @@ export function Processing() {
         // DIFFERENT meeting emits summaryFile-scoped progress — ignore those so
         // another meeting's "Summarizing part N" can't hijack this screen's stage.
         if (e.summaryFile) return;
-        const raw = e.line.replace(/^PROGRESS:summarize:/, '');
-        if (raw === 'reducing') {
-          setChunkProgress('Merging summaries…');
-        } else {
-          const [step, total] = raw.split('/').map(Number);
-          if (!Number.isNaN(step) && !Number.isNaN(total)) {
-            setChunkProgress(`Summarizing part ${step} of ${total}…`);
+        if (e.line.startsWith('PROGRESS:summarize:')) {
+          const raw = e.line.slice('PROGRESS:summarize:'.length);
+          if (raw === 'reducing') {
+            setChunkProgress('Merging summaries…');
+          } else {
+            const [step, total] = raw.split('/').map(Number);
+            if (!Number.isNaN(step) && !Number.isNaN(total)) {
+              setChunkProgress(`Summarizing part ${step} of ${total}…`);
+            }
+          }
+          setStage((s) => (s === 'transcribing' || s === 'diarizing' ? 'summarizing' : s));
+        } else if (e.line.startsWith('PROGRESS:diarize:')) {
+          const rest = e.line.slice('PROGRESS:diarize:'.length);
+          const [label, kind] = rest.split(':');
+          if (kind === 'start') {
+            setStage((s) => (s === 'transcribing' ? 'diarizing' : s));
+            clearDiarizeTimer();
+            diarizeStartedAtRef.current = Date.now();
+            setChunkProgress(`Diarizing ${label} channel…`);
+            diarizeTimerRef.current = setInterval(() => {
+              if (diarizeStartedAtRef.current === null) return;
+              const elapsedS = Math.floor((Date.now() - diarizeStartedAtRef.current) / 1000);
+              setChunkProgress(`Diarizing ${label} channel… (${formatElapsedSeconds(elapsedS)})`);
+            }, 1000);
+          } else if (kind === 'done') {
+            // Stop the elapsed ticker; no other UI action needed -- the
+            // next channel's :start, or the first summarize: chunk,
+            // supersedes this sub-label shortly after.
+            clearDiarizeTimer();
           }
         }
-        setStage((s) => (s === 'transcribing' ? 'summarizing' : s));
       }),
     ];
     return () => offs.forEach((fn) => fn());
-  }, [activeSession, updateMeeting]);
+  }, [activeSession, updateMeeting, clearDiarizeTimer]);
 
   // Watchdog. Any real processing activity — a streamed chunk, chunk progress,
   // a stage move past 'transcribing' (summaryComplete/processingComplete) —
@@ -523,10 +572,12 @@ function StageCard({
           style={{ color: 'var(--fg-2)' }}
         />
         <span
+          data-testid="processing-stage-label"
+          data-stage={stage}
           className="text-[13px] transition-colors"
           style={{ color: 'var(--fg-1)', fontFamily: 'var(--font-sans)' }}
         >
-          {chunkProgress && stage === 'summarizing' ? chunkProgress : STAGE_LABEL[stage]}
+          {chunkProgress && stage !== 'finalizing' && stage !== 'error' ? chunkProgress : STAGE_LABEL[stage]}
         </span>
       </div>
     </div>

diarize-sidecar/Package.swift

@@ -0,0 +1,19 @@
+// swift-tools-version: 5.9
+import PackageDescription
+
+let package = Package(
+    name: "diarize-sidecar",
+    platforms: [.macOS(.v14)],
+    dependencies: [
+        .package(url: "https://github.com/FluidInference/FluidAudio.git", from: "0.15.2"),
+    ],
+    targets: [
+        .executableTarget(
+            name: "diarize-sidecar",
+            dependencies: [
+                .product(name: "FluidAudio", package: "FluidAudio"),
+            ],
+            path: "Sources"
+        ),
+    ]
+)