diff --git a/ios/ScribaKeyboard/AudioRecorder.swift b/ios/ScribaKeyboard/AudioRecorder.swift index 6a17e3b..8c17110 100644 --- a/ios/ScribaKeyboard/AudioRecorder.swift +++ b/ios/ScribaKeyboard/AudioRecorder.swift @@ -22,6 +22,10 @@ final class AudioRecorder: ObservableObject { /// isn't silently lost (a common long-form/AirPods complaint). var onInterrupted: (() -> Void)? + /// Called from the audio tap with each raw input buffer, so a live transcriber + /// can stream it (only one tap is allowed per node, so we fan out here). + var onBuffer: ((AVAudioPCMBuffer) -> Void)? + private let engine = AVAudioEngine() private let targetSampleRate: Double = 16_000 private var converter: AVAudioConverter? @@ -60,6 +64,7 @@ final class AudioRecorder: ObservableObject { input.installTap(onBus: 0, bufferSize: 2048, format: inputFormat) { [weak self] buffer, _ in + self?.onBuffer?(buffer) self?.process(buffer: buffer, outputFormat: outputFormat) } diff --git a/ios/ScribaKeyboard/DictationController.swift b/ios/ScribaKeyboard/DictationController.swift index 32a653e..43119b6 100644 --- a/ios/ScribaKeyboard/DictationController.swift +++ b/ios/ScribaKeyboard/DictationController.swift @@ -16,6 +16,9 @@ final class DictationController: ObservableObject { @Published private(set) var state: State = .idle let recorder = AudioRecorder() + /// Live, on-device interim transcription for a Wispr-style streaming preview. + /// The accurate final transcript still comes from the server. + let live = LiveTranscriber() /// Called with the final transcript so the host can insert it. var onTranscript: ((String) -> Void)? @@ -32,6 +35,11 @@ final class DictationController: ObservableObject { recorder.onInterrupted = { [weak self] in Task { @MainActor in self?.handleInterruption() } } + // Fan the recorder's raw audio out to the live transcriber. Capture the + // transcriber instance (not self) so this runs off the audio thread + // without touching the @MainActor controller. + recorder.onBuffer = { [live] buffer in live.append(buffer) } + LiveTranscriber.requestAuthorization() } private func handleInterruption() { @@ -67,6 +75,7 @@ final class DictationController: ObservableObject { private func startRecording() async { do { try await recorder.start() + live.start() // live preview; no-op if speech permission isn't granted state = .recording } catch AudioRecorder.RecorderError.microphoneDenied { setError("Enable microphone access in Settings") @@ -76,6 +85,7 @@ final class DictationController: ObservableObject { } private func finishRecording() async { + live.stop() let audio = recorder.stop() guard !audio.isEmpty else { state = .idle diff --git a/ios/ScribaKeyboard/Info.plist b/ios/ScribaKeyboard/Info.plist index cc1ad0e..9843d24 100644 --- a/ios/ScribaKeyboard/Info.plist +++ b/ios/ScribaKeyboard/Info.plist @@ -16,6 +16,10 @@ $(MARKETING_VERSION) CFBundleVersion $(CURRENT_PROJECT_VERSION) + NSMicrophoneUsageDescription + Scriba uses the microphone to transcribe your speech into text. + NSSpeechRecognitionUsageDescription + Scriba uses on-device speech recognition to show your words live as you dictate. AUTH0_DOMAIN diff --git a/ios/ScribaKeyboard/KeyboardView.swift b/ios/ScribaKeyboard/KeyboardView.swift index 7f52a93..526abef 100644 --- a/ios/ScribaKeyboard/KeyboardView.swift +++ b/ios/ScribaKeyboard/KeyboardView.swift @@ -8,6 +8,7 @@ struct KeyboardView: View { @ObservedObject var dictation: DictationController @ObservedObject var recorder: AudioRecorder @ObservedObject var context: KeyboardContext + @ObservedObject var live: LiveTranscriber var needsInputModeSwitch: Bool var onAdvanceKeyboard: () -> Void @@ -100,7 +101,17 @@ struct KeyboardView: View { Text("Tap to dictate") .foregroundColor(.white.opacity(0.6)) case .recording: - Waveform(level: recorder.level) + // Show the live, on-device interim words as they're recognized + // (head-truncated so the latest words stay visible); fall back to + // the waveform until the first words arrive. + if live.interim.isEmpty { + Waveform(level: recorder.level) + } else { + Text(live.interim) + .foregroundColor(.white) + .lineLimit(1) + .truncationMode(.head) + } case .transcribing: Label("Transcribing…", systemImage: "waveform") .foregroundColor(.white.opacity(0.8)) diff --git a/ios/ScribaKeyboard/KeyboardViewController.swift b/ios/ScribaKeyboard/KeyboardViewController.swift index b6f9a6e..d1d84fb 100644 --- a/ios/ScribaKeyboard/KeyboardViewController.swift +++ b/ios/ScribaKeyboard/KeyboardViewController.swift @@ -26,6 +26,7 @@ final class KeyboardViewController: UIInputViewController { dictation: dictation, recorder: dictation.recorder, context: context, + live: dictation.live, needsInputModeSwitch: needsInputModeSwitchKey, onAdvanceKeyboard: { [weak self] in self?.advanceToNextInputMode() }, onDelete: { [weak self] in self?.textDocumentProxy.deleteBackward() }, diff --git a/ios/ScribaKeyboard/LiveTranscriber.swift b/ios/ScribaKeyboard/LiveTranscriber.swift new file mode 100644 index 0000000..da0159e --- /dev/null +++ b/ios/ScribaKeyboard/LiveTranscriber.swift @@ -0,0 +1,77 @@ +import AVFoundation +import Foundation +import Speech + +/// Live, on-device interim transcription (à la Wispr Flow's streaming feel) using +/// Apple's Speech framework. This is a *preview only* — it shows words as they're +/// spoken; the accurate final transcript still comes from the server. So if +/// speech permission is denied or unavailable, dictation degrades gracefully to +/// the server path with no live preview. +/// +/// Not `@MainActor`: audio buffers are appended from the recorder's real-time +/// tap. The request is guarded by a lock; the published `interim` is updated on +/// the main queue. +final class LiveTranscriber: ObservableObject { + @Published private(set) var interim = "" + + private var request: SFSpeechAudioBufferRecognitionRequest? + private var task: SFSpeechRecognitionTask? + private let lock = NSLock() + + /// Asks for speech-recognition permission if it hasn't been decided yet. + static func requestAuthorization() { + guard SFSpeechRecognizer.authorizationStatus() == .notDetermined else { + return + } + SFSpeechRecognizer.requestAuthorization { _ in } + } + + /// Begins a live recognition session. No-op (server path still works) if + /// permission isn't granted or a recognizer isn't available. + func start() { + guard SFSpeechRecognizer.authorizationStatus() == .authorized, + let recognizer = SFSpeechRecognizer(locale: Self.locale()), + recognizer.isAvailable + else { return } + + publish("") + let request = SFSpeechAudioBufferRecognitionRequest() + request.shouldReportPartialResults = true + // Keep the live preview on-device (private, no network) when supported. + if recognizer.supportsOnDeviceRecognition { + request.requiresOnDeviceRecognition = true + } + lock.withLock { self.request = request } + + task = recognizer.recognitionTask(with: request) { [weak self] result, _ in + guard let text = result?.bestTranscription.formattedString else { + return + } + self?.publish(text) + } + } + + /// Feed an audio buffer from the recorder's tap. + func append(_ buffer: AVAudioPCMBuffer) { + lock.withLock { request?.append(buffer) } + } + + func stop() { + lock.withLock { + request?.endAudio() + request = nil + } + task?.cancel() + task = nil + } + + private func publish(_ text: String) { + DispatchQueue.main.async { [weak self] in self?.interim = text } + } + + /// Recognize in the user's chosen language, or the device locale for 'auto'. + private static func locale() -> Locale { + let code = TranscriptionLanguage.current + return code == "auto" ? Locale.current : Locale(identifier: code) + } +}