feat(ios): live streaming transcription preview (Wispr-style)

ios/ScribaKeyboard/AudioRecorder.swift

@@ -22,6 +22,10 @@ final class AudioRecorder: ObservableObject {
     /// isn't silently lost (a common long-form/AirPods complaint).
     var onInterrupted: (() -> Void)?
 
+    /// Called from the audio tap with each raw input buffer, so a live transcriber
+    /// can stream it (only one tap is allowed per node, so we fan out here).
+    var onBuffer: ((AVAudioPCMBuffer) -> Void)?
+
     private let engine = AVAudioEngine()
     private let targetSampleRate: Double = 16_000
     private var converter: AVAudioConverter?
@@ -60,6 +64,7 @@ final class AudioRecorder: ObservableObject {
 
         input.installTap(onBus: 0, bufferSize: 2048, format: inputFormat) {
             [weak self] buffer, _ in
+            self?.onBuffer?(buffer)
             self?.process(buffer: buffer, outputFormat: outputFormat)
         }
 

ios/ScribaKeyboard/DictationController.swift

@@ -16,6 +16,9 @@ final class DictationController: ObservableObject {
 
     @Published private(set) var state: State = .idle
     let recorder = AudioRecorder()
+    /// Live, on-device interim transcription for a Wispr-style streaming preview.
+    /// The accurate final transcript still comes from the server.
+    let live = LiveTranscriber()
 
     /// Called with the final transcript so the host can insert it.
     var onTranscript: ((String) -> Void)?
@@ -32,6 +35,11 @@ final class DictationController: ObservableObject {
         recorder.onInterrupted = { [weak self] in
             Task { @MainActor in self?.handleInterruption() }
         }
+        // Fan the recorder's raw audio out to the live transcriber. Capture the
+        // transcriber instance (not self) so this runs off the audio thread
+        // without touching the @MainActor controller.
+        recorder.onBuffer = { [live] buffer in live.append(buffer) }
+        LiveTranscriber.requestAuthorization()
     }
 
     private func handleInterruption() {
@@ -67,6 +75,7 @@ final class DictationController: ObservableObject {
     private func startRecording() async {
         do {
             try await recorder.start()
+            live.start() // live preview; no-op if speech permission isn't granted
             state = .recording
         } catch AudioRecorder.RecorderError.microphoneDenied {
             setError("Enable microphone access in Settings")
@@ -76,6 +85,7 @@ final class DictationController: ObservableObject {
     }
 
     private func finishRecording() async {
+        live.stop()
         let audio = recorder.stop()
         guard !audio.isEmpty else {
             state = .idle

ios/ScribaKeyboard/Info.plist

@@ -16,6 +16,10 @@
 	<string>$(MARKETING_VERSION)</string>
 	<key>CFBundleVersion</key>
 	<string>$(CURRENT_PROJECT_VERSION)</string>
+	<key>NSMicrophoneUsageDescription</key>
+	<string>Scriba uses the microphone to transcribe your speech into text.</string>
+	<key>NSSpeechRecognitionUsageDescription</key>
+	<string>Scriba uses on-device speech recognition to show your words live as you dictate.</string>
 	<!-- Auth0 config (same build settings as the app) so the keyboard can refresh
 	     an expired access token on a 401 without forcing the user back into the app. -->
 	<key>AUTH0_DOMAIN</key>

ios/ScribaKeyboard/KeyboardView.swift

@@ -8,6 +8,7 @@ struct KeyboardView: View {
     @ObservedObject var dictation: DictationController
     @ObservedObject var recorder: AudioRecorder
     @ObservedObject var context: KeyboardContext
+    @ObservedObject var live: LiveTranscriber
 
     var needsInputModeSwitch: Bool
     var onAdvanceKeyboard: () -> Void
@@ -100,7 +101,17 @@ struct KeyboardView: View {
                 Text("Tap to dictate")
                     .foregroundColor(.white.opacity(0.6))
             case .recording:
-                Waveform(level: recorder.level)
+                // Show the live, on-device interim words as they're recognized
+                // (head-truncated so the latest words stay visible); fall back to
+                // the waveform until the first words arrive.
+                if live.interim.isEmpty {
+                    Waveform(level: recorder.level)
+                } else {
+                    Text(live.interim)
+                        .foregroundColor(.white)
+                        .lineLimit(1)
+                        .truncationMode(.head)
+                }
             case .transcribing:
                 Label("Transcribing…", systemImage: "waveform")
                     .foregroundColor(.white.opacity(0.8))

ios/ScribaKeyboard/KeyboardViewController.swift

@@ -26,6 +26,7 @@ final class KeyboardViewController: UIInputViewController {
             dictation: dictation,
             recorder: dictation.recorder,
             context: context,
+            live: dictation.live,
             needsInputModeSwitch: needsInputModeSwitchKey,
             onAdvanceKeyboard: { [weak self] in self?.advanceToNextInputMode() },
             onDelete: { [weak self] in self?.textDocumentProxy.deleteBackward() },

ios/ScribaKeyboard/LiveTranscriber.swift

@@ -0,0 +1,77 @@
+import AVFoundation
+import Foundation
+import Speech
+
+/// Live, on-device interim transcription (à la Wispr Flow's streaming feel) using
+/// Apple's Speech framework. This is a *preview only* — it shows words as they're
+/// spoken; the accurate final transcript still comes from the server. So if
+/// speech permission is denied or unavailable, dictation degrades gracefully to
+/// the server path with no live preview.
+///
+/// Not `@MainActor`: audio buffers are appended from the recorder's real-time
+/// tap. The request is guarded by a lock; the published `interim` is updated on
+/// the main queue.
+final class LiveTranscriber: ObservableObject {
+    @Published private(set) var interim = ""
+
+    private var request: SFSpeechAudioBufferRecognitionRequest?
+    private var task: SFSpeechRecognitionTask?
+    private let lock = NSLock()
+
+    /// Asks for speech-recognition permission if it hasn't been decided yet.
+    static func requestAuthorization() {
+        guard SFSpeechRecognizer.authorizationStatus() == .notDetermined else {
+            return
+        }
+        SFSpeechRecognizer.requestAuthorization { _ in }
+    }
+
+    /// Begins a live recognition session. No-op (server path still works) if
+    /// permission isn't granted or a recognizer isn't available.
+    func start() {
+        guard SFSpeechRecognizer.authorizationStatus() == .authorized,
+            let recognizer = SFSpeechRecognizer(locale: Self.locale()),
+            recognizer.isAvailable
+        else { return }
+
+        publish("")
+        let request = SFSpeechAudioBufferRecognitionRequest()
+        request.shouldReportPartialResults = true
+        // Keep the live preview on-device (private, no network) when supported.
+        if recognizer.supportsOnDeviceRecognition {
+            request.requiresOnDeviceRecognition = true
+        }
+        lock.withLock { self.request = request }
+
+        task = recognizer.recognitionTask(with: request) { [weak self] result, _ in
+            guard let text = result?.bestTranscription.formattedString else {
+                return
+            }
+            self?.publish(text)
+        }
+    }
+
+    /// Feed an audio buffer from the recorder's tap.
+    func append(_ buffer: AVAudioPCMBuffer) {
+        lock.withLock { request?.append(buffer) }
+    }
+
+    func stop() {
+        lock.withLock {
+            request?.endAudio()
+            request = nil
+        }
+        task?.cancel()
+        task = nil
+    }
+
+    private func publish(_ text: String) {
+        DispatchQueue.main.async { [weak self] in self?.interim = text }
+    }
+
+    /// Recognize in the user's chosen language, or the device locale for 'auto'.
+    private static func locale() -> Locale {
+        let code = TranscriptionLanguage.current
+        return code == "auto" ? Locale.current : Locale(identifier: code)
+    }
+}