diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/DictateController.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/DictateController.kt index 5f2271818..fd47091a8 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/DictateController.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/DictateController.kt @@ -250,17 +250,21 @@ object DictateController { private val _interimText = MutableStateFlow("") /** * Live transcript while a real-time recording runs: finalized segments plus the current partial. The - * Smartbar shows this as a live caption; the field only receives the finished (reworded) text on stop. - * Empty outside a realtime recording. + * Smartbar shows this as a live caption; IME fields receive a coalesced preview which final commit + * replaces with the finished (reworded) text on stop. Empty outside a realtime recording. */ val interimText: StateFlow = _interimText.asStateFlow() private var realtimeSession: RealtimeSession? = null + private val realtimeTextLock = Any() private val realtimeFinal = StringBuilder() // accumulated finalized segments @Volatile private var realtimeFailed = false // stream errored → fall back to batch on stop private var realtimeClosed: CompletableDeferred? = null private var realtimeContext: Context? = null // app context to edit the field's provisional text private val realtimeShown = StringBuilder() // text currently committed to the field this session + private var realtimePreviewJob: Job? = null + @Volatile private var realtimePreviewLatest = "" // newest transcript received, even if not flushed yet + private var realtimePreviewLastFlushMs = 0L @Volatile private var realtimeCancelled = false // block late stream callbacks from re-adding text // --- Long-form segmented dictation (issue #170) --------------------------------------------- @@ -409,6 +413,8 @@ object DictateController { /** Cache file name for the merged audio when a continued interrupted recording is stitched together. */ private const val MERGED_AUDIO_NAME = "dictate_merged.wav" + // Realtime (#128): cap IME preview writes to about one editor update per display frame. + private const val REALTIME_PREVIEW_FRAME_MS = 66L // Realtime (#128): after finish(), how long to wait for the provider to flush the last words before we // commit the already-streamed text. Short — the text is already on screen; we only wait for the tail. private const val REALTIME_FINALIZE_TIMEOUT_MS = 1_200L @@ -416,6 +422,19 @@ object DictateController { /** 20 Hz is responsive for a voice indicator while avoiding a display-rate UI loop. */ private const val AUDIO_LEVEL_SAMPLE_MS = 50L + private fun cancelRealtimePreviewJob() { + realtimePreviewJob?.cancel() + realtimePreviewJob = null + } + + private fun resetRealtimePreviewScheduler() { + cancelRealtimePreviewJob() + synchronized(realtimeTextLock) { + realtimePreviewLatest = "" + } + realtimePreviewLastFlushMs = 0L + } + /** Cumulative recorded audio (seconds) after which the rate / donate nudges appear (roadmap 9.7/9.8). */ private const val RATE_THRESHOLD_SECONDS = 180L // 3 min private const val DONATE_THRESHOLD_SECONDS = 300L // 5 min (user choice; legacy used 10 min) @@ -606,6 +625,7 @@ object DictateController { // Tear down any realtime stream (#128) and remove the live provisional text from the field. Set the // cancelled flag first so any stream callback still queued on the main thread can't re-add the text. realtimeCancelled = true + resetRealtimePreviewScheduler() realtimeSession?.cancel() realtimeSession = null realtimeClosed = null @@ -1208,37 +1228,87 @@ object DictateController { val preset = presetFor(account) val model = account.realtimeModel.takeIf { it.isNotBlank() } ?: preset.defaultRealtimeModel ?: return null val language = prefs.dictate.activeInputLanguage.get().takeIf { it != DictateLanguages.DETECT } - realtimeFinal.setLength(0) + synchronized(realtimeTextLock) { + realtimeFinal.setLength(0) + realtimePreviewLatest = "" + } realtimeFailed = false realtimeCancelled = false _interimText.value = "" realtimeContext = appContext realtimeShown.setLength(0) + resetRealtimePreviewScheduler() val closed = CompletableDeferred() realtimeClosed = closed - // Type the growing transcript live into the field, applying only the minimal diff each time (#128). - fun showLive(full: String) { - if (realtimeCancelled) return // a late callback must not re-add text after a cancel - _interimText.value = full - runCatching { sink(appContext).setDictationPreview(full, realtimeShown.toString()) } + val previewSink = sink(appContext) + val coalescePreview = outputTarget == OutputTarget.IME + + fun flushPreview() { + if (realtimeCancelled) return + val full = realtimePreviewLatest + val prev = realtimeShown.toString() + if (full == prev) return + runCatching { previewSink.setDictationPreview(full, prev) } realtimeShown.setLength(0) realtimeShown.append(full) + realtimePreviewLastFlushMs = SystemClock.elapsedRealtime() + } + + fun publishLive(full: String, force: Boolean = false) { + if (realtimeCancelled) return + if (full == _interimText.value && !force) return + _interimText.value = full + + if (!coalescePreview || force) { + cancelRealtimePreviewJob() + flushPreview() + return + } + + if (realtimePreviewJob?.isActive == true) return + val elapsed = SystemClock.elapsedRealtime() - realtimePreviewLastFlushMs + val waitMs = (REALTIME_PREVIEW_FRAME_MS - elapsed).coerceAtLeast(0L) + if (waitMs == 0L) { + flushPreview() + return + } + realtimePreviewJob = scope.launch { + delay(waitMs) + flushPreview() + realtimePreviewJob = null + } + } + fun recordPartial(text: String): String? = synchronized(realtimeTextLock) { + if (realtimeCancelled) { + null + } else { + val head = realtimeFinal.toString() + (if (head.isEmpty()) text else "$head $text").trim().also { realtimePreviewLatest = it } + } + } + fun recordFinalSegment(text: String): String? = synchronized(realtimeTextLock) { + if (realtimeCancelled) { + null + } else { + val t = text.trim() + if (t.isNotEmpty()) { + if (realtimeFinal.isNotEmpty()) realtimeFinal.append(' ') + realtimeFinal.append(t) + } + realtimeFinal.toString().also { realtimePreviewLatest = it } + } } val callbacks = object : RealtimeCallbacks { override fun onPartial(text: String) { + val full = recordPartial(text) ?: return scope.launch { - val head = realtimeFinal.toString() - showLive((if (head.isEmpty()) text else "$head $text").trim()) + publishLive(full) } } override fun onFinalSegment(text: String) { + val full = recordFinalSegment(text) ?: return scope.launch { - val t = text.trim() - if (t.isNotEmpty()) { - if (realtimeFinal.isNotEmpty()) realtimeFinal.append(' ') - realtimeFinal.append(t) - } - showLive(realtimeFinal.toString()) + publishLive(full, force = true) } } override fun onError(t: Throwable) { realtimeFailed = true } @@ -1253,9 +1323,20 @@ object DictateController { runCatching { session.sendAudio(pcm, len) } } } + var resampleBuffer = ByteArray(0) return { pcm, len -> - val out = Pcm16Resampler.resample(pcm, len, AudioDecode.TARGET_SAMPLE_RATE, targetRate) - runCatching { session.sendAudio(out, out.size) } + val outLen = Pcm16Resampler.outputLengthBytes(len, AudioDecode.TARGET_SAMPLE_RATE, targetRate) + if (outLen > 0) { + if (resampleBuffer.size < outLen) resampleBuffer = ByteArray(outLen) + val written = Pcm16Resampler.resampleInto( + pcm = pcm, + len = len, + srcRate = AudioDecode.TARGET_SAMPLE_RATE, + dstRate = targetRate, + out = resampleBuffer, + ) + runCatching { session.sendAudio(resampleBuffer, written) } + } } } @@ -1289,14 +1370,20 @@ object DictateController { // stalls us until the timeout and later trips a ping/pong failure. withTimeoutOrNull(REALTIME_FINALIZE_TIMEOUT_MS) { closed?.await() } runCatching { session?.cancel() } - // The transcript is what we already streamed into the field (finals + last partial); fall - // back to the finalized-segments buffer only if nothing was shown. - val transcript = realtimeShown.toString().trim().ifEmpty { realtimeFinal.toString().trim() } + realtimeCancelled = true + cancelRealtimePreviewJob() + // Prefer the newest received transcript; it may be newer than the coalesced field preview. + val transcript = synchronized(realtimeTextLock) { + realtimePreviewLatest.trim() + .ifEmpty { realtimeShown.toString().trim() } + .ifEmpty { realtimeFinal.toString().trim() } + } _interimText.value = "" if (realtimeFailed || transcript.isEmpty()) { // Drop the live provisional text; the batch path commits fresh from the WAV. runCatching { sink(appContext).clearDictationPreview(realtimeShown.toString()) } realtimeShown.setLength(0) + resetRealtimePreviewScheduler() if (wavFile != null && wavFile.exists() && wavFile.length() > 0L) { livePromptArmed = live transcribe(context, wavFile, recordedSeconds, gate = false) @@ -1320,13 +1407,23 @@ object DictateController { source = DictateHistorySource.REALTIME, ) finalizeAndCommit(appContext, transcript, recordedSeconds, live, alreadyFormatted = false, finalizeViaComposing = true, capture = rtCapture) + resetRealtimePreviewScheduler() wavFile?.delete() } catch (c: CancellationException) { + realtimeCancelled = true + cancelRealtimePreviewJob() + _interimText.value = "" + runCatching { session?.cancel() } + runCatching { sink(appContext).clearDictationPreview(realtimeShown.toString()) } + realtimeShown.setLength(0) + resetRealtimePreviewScheduler() + wavFile?.delete() throw c } catch (t: Throwable) { _interimText.value = "" runCatching { sink(appContext).clearDictationPreview(realtimeShown.toString()) } realtimeShown.setLength(0) + resetRealtimePreviewScheduler() if (wavFile != null && wavFile.exists() && wavFile.length() > 0L) { livePromptArmed = live transcribe(context, wavFile, recordedSeconds, gate = false) @@ -1819,6 +1916,7 @@ object DictateController { _livePromptActive.value = false // Realtime (#128): drop the stream; the WAV is stashed below and recoverable via batch as usual. realtimeCancelled = true + resetRealtimePreviewScheduler() realtimeSession?.cancel() realtimeSession = null realtimeClosed = null diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/AudioDecode.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/AudioDecode.kt index e27c651d2..d368411da 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/AudioDecode.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/AudioDecode.kt @@ -75,6 +75,56 @@ object AudioDecode { } } + /** + * Streams recorder-native WAV audio (PCM16, mono, 16 kHz) in reusable float windows. + * + * Returns false when [file] is not exactly this shape, so callers can fall back to [decodeToMono16k]. + * [onWindow] returns false to stop reading early. Full-size windows reuse the same FloatArray. + */ + internal fun streamPcm16Mono16kWav( + file: File, + windowSize: Int, + onWindow: (FloatArray) -> Boolean, + ): Boolean { + require(windowSize > 0) { "windowSize must be positive" } + + RandomAccessFile(file, "r").use { input -> + val wav = parseWav(input) ?: return false + if ( + wav.audioFormat != 1 || + wav.bitsPerSample != 16 || + wav.channels != 1 || + wav.sampleRate != TARGET_SAMPLE_RATE + ) { + return false + } + + var remainingSamples = wav.dataLength / 2L + if (remainingSamples <= 0L) return true + input.seek(wav.dataOffset) + + val bytes = ByteArray(windowSize * 2) + val floats = FloatArray(windowSize) + while (remainingSamples > 0L) { + val count = minOf(windowSize.toLong(), remainingSamples).toInt() + input.readFully(bytes, 0, count * 2) + var byteIndex = 0 + var sampleIndex = 0 + while (sampleIndex < count) { + val sample = (bytes[byteIndex].toInt() and 0xff) or + (bytes[byteIndex + 1].toInt() shl 8) + floats[sampleIndex] = sample / 32768.0f + byteIndex += 2 + sampleIndex++ + } + val chunk = if (count == windowSize) floats else floats.copyOf(count) + if (!onWindow(chunk)) return true + remainingSamples -= count + } + return true + } + } + /** * Parses a PCM WAV [file] directly into mono float samples at [TARGET_SAMPLE_RATE], or returns null * if it is not a (PCM) WAV — then the caller falls back to the MediaCodec path. Handles arbitrary @@ -147,7 +197,7 @@ object AudioDecode { chunkHeader.hasTag(0, "data") -> { if (!hasFormat) return null val len = size.coerceAtMost(input.length() - body) - if (len <= 0L) return null + if (len < 0L) return null return WavInfo(audioFormat, channels, sampleRate, bitsPerSample, body, len) } } diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/Pcm16Resampler.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/Pcm16Resampler.kt index 17fd32e5d..ef9a2bc92 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/Pcm16Resampler.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/Pcm16Resampler.kt @@ -16,16 +16,56 @@ object Pcm16Resampler { fun resample(pcm: ByteArray, len: Int, srcRate: Int, dstRate: Int): ByteArray { require(len in 0..pcm.size) { "len must be within pcm bounds" } require(srcRate > 0 && dstRate > 0) { "sample rates must be positive" } - if (srcRate == dstRate) return if (len == pcm.size) pcm else pcm.copyOfRange(0, len) - if (srcRate == 16_000 && dstRate == 24_000) return resample16kTo24k(pcm, len) - return resampleLinear(pcm, len, srcRate, dstRate) + val evenLen = len and -2 + if (evenLen == 0) return ByteArray(0) + if (srcRate == dstRate) return if (evenLen == pcm.size) pcm else pcm.copyOfRange(0, evenLen) + if (srcRate == 16_000 && dstRate == 24_000) return resample16kTo24k(pcm, evenLen) + return resampleLinear(pcm, evenLen, srcRate, dstRate) + } + + fun outputLengthBytes(len: Int, srcRate: Int, dstRate: Int): Int { + require(len >= 0) { "len must be non-negative" } + require(srcRate > 0 && dstRate > 0) { "sample rates must be positive" } + val evenLen = len and -2 + if (evenLen == 0 || srcRate == dstRate) return evenLen + val inSamples = evenLen / 2 + val outSamples = (inSamples.toLong() * dstRate / srcRate).toInt().coerceAtLeast(1) + return outSamples * 2 + } + + fun resampleInto( + pcm: ByteArray, + len: Int, + srcRate: Int, + dstRate: Int, + out: ByteArray, + ): Int { + require(len in 0..pcm.size) { "len must be within pcm bounds" } + val outLen = outputLengthBytes(len, srcRate, dstRate) + require(out.size >= outLen) { "out buffer too small: need $outLen bytes, got ${out.size}" } + val evenLen = len and -2 + if (outLen == 0) return 0 + if (srcRate == dstRate) { + pcm.copyInto(out, destinationOffset = 0, startIndex = 0, endIndex = evenLen) + return outLen + } + return if (srcRate == 16_000 && dstRate == 24_000) { + resample16kTo24kInto(pcm, evenLen, out) + } else { + resampleLinearInto(pcm, evenLen, srcRate, dstRate, out) + } } private fun resample16kTo24k(pcm: ByteArray, len: Int): ByteArray { + val out = ByteArray(outputLengthBytes(len, 16_000, 24_000)) + resample16kTo24kInto(pcm, len, out) + return out + } + + private fun resample16kTo24kInto(pcm: ByteArray, len: Int, out: ByteArray): Int { val inSamples = len / 2 - if (inSamples <= 0) return ByteArray(0) + if (inSamples <= 0) return 0 val outSamples = (inSamples.toLong() * 3 / 2).toInt().coerceAtLeast(1) - val out = ByteArray(outSamples * 2) for (outIndex in 0 until outSamples) { val scaled = outIndex * 2 val i0 = scaled / 3 @@ -33,26 +73,31 @@ object Pcm16Resampler { val s0 = pcmSampleAt(pcm, i0, inSamples) val s1 = pcmSampleAt(pcm, i0 + 1, inSamples) val v = ((s0 * 3) + ((s1 - s0) * rem)) / 3 - writeSample(out, outIndex, v.coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt())) + writeSample(out, outIndex, v) } - return out + return outSamples * 2 } private fun resampleLinear(pcm: ByteArray, len: Int, srcRate: Int, dstRate: Int): ByteArray { + val out = ByteArray(outputLengthBytes(len, srcRate, dstRate)) + resampleLinearInto(pcm, len, srcRate, dstRate, out) + return out + } + + private fun resampleLinearInto(pcm: ByteArray, len: Int, srcRate: Int, dstRate: Int, out: ByteArray): Int { val inSamples = len / 2 - if (inSamples <= 0) return ByteArray(0) + if (inSamples <= 0) return 0 val outSamples = (inSamples.toLong() * dstRate / srcRate).toInt().coerceAtLeast(1) - val out = ByteArray(outSamples * 2) for (outIndex in 0 until outSamples) { val srcPos = outIndex.toDouble() * srcRate / dstRate val i0 = srcPos.toInt() val frac = srcPos - i0 val s0 = pcmSampleAt(pcm, i0, inSamples) val s1 = pcmSampleAt(pcm, i0 + 1, inSamples) - val v = (s0 + (s1 - s0) * frac).toInt().coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()) + val v = (s0 + (s1 - s0) * frac).toInt() writeSample(out, outIndex, v) } - return out + return outSamples * 2 } private fun pcmSampleAt(pcm: ByteArray, idx: Int, count: Int): Int { diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/RecordingController.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/RecordingController.kt index 160bb39f0..b9c66998f 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/RecordingController.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/RecordingController.kt @@ -16,9 +16,11 @@ import android.media.AudioFormat import android.media.AudioRecord import android.media.MediaRecorder import java.io.File +import java.io.IOException import java.io.RandomAccessFile import java.nio.ByteBuffer import java.nio.ByteOrder +import java.util.concurrent.atomic.AtomicInteger /** * Records microphone audio into a 16 kHz mono PCM16 **WAV** file in the app cache. @@ -45,7 +47,8 @@ class RecordingController(private val context: Context) { @Volatile private var paused = false @Volatile private var pcmBytes = 0L /** Peak |sample| (0..32767) seen since the last [maxAmplitude] call; drives the waveform. */ - @Volatile private var peak = 0 + private val peak = AtomicInteger(0) + @Volatile private var recordingError: Throwable? = null /** The file the current/last recording was written to, or null if nothing was recorded yet. */ var outputFile: File? = null @@ -94,25 +97,47 @@ class RecordingController(private val context: Context) { record = rec raf = out outputFile = file - pcmBytes = 0 - peak = 0 + pcmBytes = 0L + peak.set(0) + recordingError = null paused = false + try { + rec.startRecording() + } catch (t: Throwable) { + record = null + raf = null + outputFile = null + runCatching { out.close() } + file.delete() + runCatching { rec.release() } + throw t + } recording = true - rec.startRecording() thread = Thread { - val buf = ByteArray(bufferSize) + val readSize = if (pcmSink != null) minOf(bufferSize, REALTIME_READ_BYTES) else bufferSize + val buf = ByteArray(readSize) while (recording) { val n = rec.read(buf, 0, buf.size) + if (!recording) break // Keep reading while paused (so the mic buffer never overflows) but drop the samples. - if (n > 0 && !paused) { - // Write under the lock so a concurrent rotate() sees a consistent raf/pcmBytes and the - // frame lands in the correct segment file (never split across a rotation). - synchronized(fileLock) { - runCatching { raf?.write(buf, 0, n) } - pcmBytes += n + if (n < 0) { + recordingError = IOException("AudioRecord.read failed: $n") + recording = false + } else if (n > 0 && !paused) { + try { + // Keep the file and byte count consistent with rotate()/stop(). Do not hide write + // failures: a WAV whose header claims bytes that were never written is unusable. + synchronized(fileLock) { + val currentOut = raf ?: throw IOException("Recording output is unavailable") + currentOut.write(buf, 0, n) + pcmBytes += n + } + updatePeak(buf, n) + if (pcmSink != null) runCatching { pcmSink(buf, n) } + } catch (t: Throwable) { + recordingError = t + recording = false } - updatePeak(buf, n) - if (pcmSink != null) runCatching { pcmSink(buf, n) } } } }.also { it.start() } @@ -134,12 +159,18 @@ class RecordingController(private val context: Context) { old.write(wavHeader(bytes)) old.close() if (bytes > 0 && base != null) { - val seg = File(context.cacheDir, "dictate_seg_${segmentSeq++}.wav") - seg.delete() - if (base.renameTo(seg)) seg else null + // Controllers are recreated between recordings while earlier segments may still be + // transcribing. Include a monotonic component so a new controller cannot delete one. + val seg = File(context.cacheDir, "dictate_seg_${System.nanoTime()}_${segmentSeq++}.wav") + check(base.renameTo(seg)) { "Could not preserve completed recording segment" } + seg } else null - } catch (_: Throwable) { - null + } catch (t: Throwable) { + runCatching { old.close() } + raf = null + recordingError = t + recording = false + return@synchronized null } // Reopen a fresh WAV (the base name is now free after the rename) for the continuing recording. val file = File(context.cacheDir, AUDIO_FILE_NAME) @@ -148,11 +179,13 @@ class RecordingController(private val context: Context) { setLength(0) write(ByteArray(WAV_HEADER_SIZE)) } - } catch (_: Throwable) { + } catch (t: Throwable) { + recordingError = t + recording = false null } outputFile = file - pcmBytes = 0 + pcmBytes = 0L segment } @@ -161,12 +194,12 @@ class RecordingController(private val context: Context) { * recorder is always released. */ fun stop(): File? { - if (!recording) return null + val rec = record + if (!recording && rec == null && thread == null && raf == null) return null recording = false // AudioRecord.read(byte[], …) is blocking. Stop the native recorder first so a read waiting for // microphone frames returns before we join the capture thread. On the normal path, release only // after the reader exits; if stop fails, release early so the microphone still cannot leak. - val rec = record record = null val stopped = rec != null && runCatching { rec.stop() }.isSuccess if (!stopped) runCatching { rec?.release() } @@ -174,10 +207,20 @@ class RecordingController(private val context: Context) { thread = null if (stopped) runCatching { rec?.release() } return synchronized(fileLock) { - val out = raf ?: return@synchronized null + val out = raf raf = null val bytes = pcmBytes val file = outputFile + val failure = recordingError + recordingError = null + pcmBytes = 0L + peak.set(0) + if (out == null || failure != null) { + runCatching { out?.close() } + file?.delete() + outputFile = null + return@synchronized null + } try { out.seek(0) out.write(wavHeader(bytes)) @@ -186,17 +229,14 @@ class RecordingController(private val context: Context) { } catch (_: Throwable) { runCatching { out.close() } file?.delete() + outputFile = null null } } } /** The peak microphone amplitude (0..32767) since the previous call, or 0 when not recording. */ - fun maxAmplitude(): Int { - val p = peak - peak = 0 - return p - } + fun maxAmplitude(): Int = peak.getAndSet(0) /** Pauses the in-progress recording (samples are dropped until [resume]). No-op if not recording. */ fun pause() { @@ -210,12 +250,26 @@ class RecordingController(private val context: Context) { /** Stops and discards the current recording without returning it. */ fun cancel() { - stop() - outputFile?.delete() + recording = false + val rec = record + runCatching { rec?.stop() } + runCatching { thread?.join() } + thread = null + runCatching { rec?.release() } + record = null + synchronized(fileLock) { + recordingError = null + runCatching { raf?.close() } + raf = null + outputFile?.delete() + outputFile = null + pcmBytes = 0L + peak.set(0) + } } private fun updatePeak(buf: ByteArray, length: Int) { - var max = peak + var max = 0 var i = 0 while (i + 1 < length) { val sample = (buf[i].toInt() and 0xff) or (buf[i + 1].toInt() shl 8) // little-endian PCM16 @@ -223,7 +277,12 @@ class RecordingController(private val context: Context) { if (abs > max) max = abs i += 2 } - peak = max.coerceAtMost(32767) + val capped = max.coerceAtMost(32767) + while (true) { + val current = peak.get() + val next = if (capped > current) capped else current + if (peak.compareAndSet(current, next)) return + } } private fun wavHeader(dataLen: Long): ByteArray { @@ -252,6 +311,10 @@ class RecordingController(private val context: Context) { private const val ENCODING = AudioFormat.ENCODING_PCM_16BIT private const val CHANNELS = 1 private const val BITS_PER_SAMPLE = 16 + private const val BYTES_PER_SAMPLE = BITS_PER_SAMPLE / 8 + private const val REALTIME_READ_CHUNK_MS = 40 + private const val REALTIME_READ_BYTES = + SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * REALTIME_READ_CHUNK_MS / 1000 private const val WAV_HEADER_SIZE = 44 } } diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/SpeechGate.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/SpeechGate.kt index df061917e..e0a5a72f2 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/SpeechGate.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/audio/SpeechGate.kt @@ -64,62 +64,117 @@ object SpeechGate { /** * Returns true if [audioFile] contains at least one speech segment (or if the check could not be run, * so a real recording is never dropped by a gate failure); false only when the VAD is confident there - * is no speech at all. Decoding + VAD run on [Dispatchers.Default]; a short clip that begins with - * speech exits as soon as the first segment closes. + * is no speech at all. Native PCM16 mono 16 kHz WAV recordings are streamed directly; other formats + * fall back to decoding. A short clip that begins with speech exits as soon as the first segment closes. */ suspend fun hasSpeech(context: Context, audioFile: File): Boolean = withContext(Dispatchers.Default) { val totalStartedNanos = System.nanoTime() val model = ensureVadModel(context.applicationContext) ?: return@withContext true + val streamed = runCatching { + hasSpeechInRecordedWav(model, audioFile, totalStartedNanos) + }.getOrElse { return@withContext true } + if (streamed != null) return@withContext streamed + val decodeStartedNanos = System.nanoTime() val samples = runCatching { AudioDecode.decodeToMono16k(audioFile) }.getOrNull() ?: return@withContext true val decodeMs = elapsedMillis(decodeStartedNanos) if (samples.isEmpty()) return@withContext false - vadMutex.withLock { - val createStartedNanos = System.nanoTime() - val vad = cachedVad ?: createVad(model)?.also { cachedVad = it } - ?: return@withLock true - val createMs = elapsedMillis(createStartedNanos) - val runStartedNanos = System.nanoTime() - try { - vad.reset() - val window = FloatArray(WINDOW) - var i = 0 - while (i < samples.size) { - val end = minOf(i + WINDOW, samples.size) - val chunk = if (end - i == WINDOW) { - samples.copyInto(window, destinationOffset = 0, startIndex = i, endIndex = end) - window - } else { - samples.copyOfRange(i, end) - } - vad.acceptWaveform(chunk) - i = end - if (!vad.empty()) { - Log.i( - LOG_TAG, - "speechGate decodeMs=$decodeMs createMs=$createMs " + - "runMs=${elapsedMillis(runStartedNanos)} totalMs=${elapsedMillis(totalStartedNanos)} speech=true", - ) - return@withLock true - } + hasSpeechInDecodedSamples(model, samples, decodeMs, totalStartedNanos) + } + + private suspend fun hasSpeechInRecordedWav( + model: File, + audioFile: File, + totalStartedNanos: Long, + ): Boolean? = vadMutex.withLock { + val createStartedNanos = System.nanoTime() + val vad = cachedVad ?: createVad(model)?.also { cachedVad = it } + ?: return@withLock true + val createMs = elapsedMillis(createStartedNanos) + val runStartedNanos = System.nanoTime() + try { + vad.reset() + var sawSamples = false + var sawSpeech = false + val handled = AudioDecode.streamPcm16Mono16kWav(audioFile, WINDOW) { chunk -> + sawSamples = true + vad.acceptWaveform(chunk) + sawSpeech = !vad.empty() + !sawSpeech + } + if (!handled) return@withLock null + val speech = when { + !sawSamples -> false + sawSpeech -> true + else -> { + vad.flush() + !vad.empty() + } + } + Log.i( + LOG_TAG, + "speechGate streamed=true createMs=$createMs " + + "runMs=${elapsedMillis(runStartedNanos)} totalMs=${elapsedMillis(totalStartedNanos)} speech=$speech", + ) + speech + } catch (_: Throwable) { + // A native session that faulted is not reused. The next recording gets a fresh one. + cachedVad = null + runCatching { vad.release() } + true // fail open + } + } + + private suspend fun hasSpeechInDecodedSamples( + model: File, + samples: FloatArray, + decodeMs: Long, + totalStartedNanos: Long, + ): Boolean = vadMutex.withLock { + val createStartedNanos = System.nanoTime() + val vad = cachedVad ?: createVad(model)?.also { cachedVad = it } + ?: return@withLock true + val createMs = elapsedMillis(createStartedNanos) + val runStartedNanos = System.nanoTime() + try { + vad.reset() + val window = FloatArray(WINDOW) + var i = 0 + while (i < samples.size) { + val end = minOf(i + WINDOW, samples.size) + val chunk = if (end - i == WINDOW) { + samples.copyInto(window, destinationOffset = 0, startIndex = i, endIndex = end) + window + } else { + samples.copyOfRange(i, end) + } + vad.acceptWaveform(chunk) + i = end + if (!vad.empty()) { + Log.i( + LOG_TAG, + "speechGate streamed=false decodeMs=$decodeMs createMs=$createMs " + + "runMs=${elapsedMillis(runStartedNanos)} totalMs=${elapsedMillis(totalStartedNanos)} speech=true", + ) + return@withLock true } - // No segment closed mid-stream (e.g. speech ran right up to the end): flush and re-check. - vad.flush() - val speech = !vad.empty() - Log.i( - LOG_TAG, - "speechGate decodeMs=$decodeMs createMs=$createMs " + - "runMs=${elapsedMillis(runStartedNanos)} totalMs=${elapsedMillis(totalStartedNanos)} speech=$speech", - ) - speech - } catch (_: Throwable) { - // A native session that faulted is not reused. The next recording gets a fresh one. - cachedVad = null - runCatching { vad.release() } - true // fail open } + // No segment closed mid-stream (e.g. speech ran right up to the end): flush and re-check. + vad.flush() + val speech = !vad.empty() + Log.i( + LOG_TAG, + "speechGate streamed=false decodeMs=$decodeMs createMs=$createMs " + + "runMs=${elapsedMillis(runStartedNanos)} totalMs=${elapsedMillis(totalStartedNanos)} speech=$speech", + ) + speech + } catch (_: Throwable) { + // A native session that faulted is not reused. The next recording gets a fresh one. + cachedVad = null + runCatching { vad.release() } + true // fail open } } diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/data/stats/DictateStats.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/data/stats/DictateStats.kt index 77adfc3a1..7bc2492b5 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/data/stats/DictateStats.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/data/stats/DictateStats.kt @@ -100,13 +100,20 @@ object DictateStats { val raw = d.statsPendingMilestone.get() if (raw.isNotEmpty()) d.statsPendingMilestone.set("") if (!d.statsMilestonesEnabled.get() || raw.isBlank()) return null - val parts = raw.split(':') - val kind = when (parts.getOrNull(0)) { - "time" -> Milestone.Kind.TIME_MINUTES - "count" -> Milestone.Kind.DICTATIONS + val valueStart: Int + val kind = when { + raw.startsWith("time:") -> { + valueStart = "time:".length + Milestone.Kind.TIME_MINUTES + } + raw.startsWith("count:") -> { + valueStart = "count:".length + Milestone.Kind.DICTATIONS + } else -> return null } - val value = parts.getOrNull(1)?.toLongOrNull() ?: return null + val valueEnd = raw.indexOf(':', startIndex = valueStart).let { if (it >= 0) it else raw.length } + val value = parseLongOrNull(raw, valueStart, valueEnd) ?: return null return Milestone(kind, value) } @@ -183,12 +190,39 @@ object DictateStats { private fun parseDaily(raw: String): Map { if (raw.isBlank()) return emptyMap() - return raw.split(';').mapNotNull { entry -> - val parts = entry.split(':') - val day = parts.getOrNull(0)?.toLongOrNull() ?: return@mapNotNull null - val words = parts.getOrNull(1)?.toLongOrNull() ?: return@mapNotNull null - day to words - }.toMap() + val out = mutableMapOf() + var entryStart = 0 + while (entryStart <= raw.length) { + val entryEnd = raw.indexOf(';', startIndex = entryStart).let { if (it >= 0) it else raw.length } + val separator = raw.indexOf(':', startIndex = entryStart).takeIf { it in entryStart + 1 until entryEnd } + if (separator != null) { + val day = parseLongOrNull(raw, entryStart, separator) + val words = parseLongOrNull(raw, separator + 1, entryEnd) + if (day != null && words != null) out[day] = words + } + if (entryEnd == raw.length) break + entryStart = entryEnd + 1 + } + return out + } + + private fun parseLongOrNull(raw: String, start: Int, end: Int): Long? { + if (start >= end) return null + var i = start + val negative = raw[i] == '-' + if (negative) { + i++ + if (i == end) return null + } + var value = 0L + while (i < end) { + val digit = raw[i] - '0' + if (digit !in 0..9) return null + if (value > (Long.MAX_VALUE - digit) / 10L) return null + value = value * 10L + digit + i++ + } + return if (negative) -value else value } private fun serializeDaily(map: Map): String = diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/LocalTranscriptionProvider.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/LocalTranscriptionProvider.kt index 26ac5ca41..c3ea0e9e7 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/LocalTranscriptionProvider.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/LocalTranscriptionProvider.kt @@ -152,7 +152,7 @@ class LocalTranscriptionProvider( } finally { vad.release() } - return parts.toString().trim().ifBlank { decodeOnce(recognizer, samples) } + return if (parts.isEmpty()) decodeOnce(recognizer, samples).trim() else parts.toString().trim() } private fun drainSegments(vad: Vad, recognizer: OfflineRecognizer, out: StringBuilder) { @@ -238,7 +238,7 @@ private object RecognizerCache { // Language is baked into the Whisper config at build time, so it is part of the cache key // (switching the input language rebuilds the recognizer; ~1s). A transducer decodes the audio // as-is and ignores the language, so it stays out of the key for those. - val cacheKey = modelDir.absolutePath + "|" + (if (isTransducer) "" else language) + val cacheKey = modelDir.absolutePath + "|" + numThreads + "|" + (if (isTransducer) "" else language) val existing = recognizer if (existing != null && cacheKey == key) return existing diff --git a/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/AudioWavTest.kt b/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/AudioWavTest.kt index a9b44bb45..95519230f 100644 --- a/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/AudioWavTest.kt +++ b/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/AudioWavTest.kt @@ -19,6 +19,7 @@ import java.nio.file.Files import kotlin.math.abs import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFalse import kotlin.test.assertTrue class AudioWavTest { @@ -66,11 +67,83 @@ class AudioWavTest { } } + @Test + fun streamPcm16Mono16kWavWindowsMatchDecode() { + val dir = Files.createTempDirectory("dictate-wav-stream").toFile() + try { + val wav = File(dir, "sample.wav").also { + it.writeBytes(wavBytes(shortArrayOf(Short.MIN_VALUE, 0, Short.MAX_VALUE, 1_234, -1_234))) + } + val chunks = mutableListOf() + + val handled = AudioDecode.streamPcm16Mono16kWav(wav, windowSize = 2) { chunk -> + chunks.add(chunk.copyOf()) + true + } + + assertTrue(handled) + assertEquals(listOf(2, 2, 1), chunks.map { it.size }) + val streamed = FloatArray(chunks.sumOf { it.size }) + var offset = 0 + for (chunk in chunks) { + chunk.copyInto(streamed, destinationOffset = offset) + offset += chunk.size + } + val decoded = AudioDecode.decodeToMono16k(wav) + assertEquals(decoded.size, streamed.size) + decoded.indices.forEach { i -> assertNear(decoded[i], streamed[i]) } + } finally { + dir.deleteRecursively() + } + } + + @Test + fun streamPcm16Mono16kWavCanStopEarly() { + val dir = Files.createTempDirectory("dictate-wav-stream-stop").toFile() + try { + val wav = File(dir, "sample.wav").also { + it.writeBytes(wavBytes(shortArrayOf(1, 2, 3, 4, 5, 6))) + } + var calls = 0 + + val handled = AudioDecode.streamPcm16Mono16kWav(wav, windowSize = 2) { + calls++ + false + } + + assertTrue(handled) + assertEquals(1, calls) + } finally { + dir.deleteRecursively() + } + } + + @Test + fun streamPcm16Mono16kWavRejectsNonNativeRateForFallback() { + val dir = Files.createTempDirectory("dictate-wav-stream-reject").toFile() + try { + val wav = File(dir, "sample.wav").also { + it.writeBytes(wavBytes(shortArrayOf(1, 2), sampleRate = 8_000)) + } + var called = false + + val handled = AudioDecode.streamPcm16Mono16kWav(wav, windowSize = 2) { + called = true + true + } + + assertFalse(handled) + assertFalse(called) + } finally { + dir.deleteRecursively() + } + } + private fun assertNear(expected: Float, actual: Float) { assertTrue(abs(expected - actual) < 0.000001f, "expected=$expected actual=$actual") } - private fun wavBytes(samples: ShortArray): ByteArray { + private fun wavBytes(samples: ShortArray, sampleRate: Int = AudioDecode.TARGET_SAMPLE_RATE): ByteArray { val dataLen = samples.size * 2 return ByteBuffer.allocate(WAV_HEADER_SIZE + dataLen).order(ByteOrder.LITTLE_ENDIAN).apply { put("RIFF".toByteArray(Charsets.US_ASCII)) @@ -80,8 +153,8 @@ class AudioWavTest { putInt(16) putShort(1) putShort(1) - putInt(AudioDecode.TARGET_SAMPLE_RATE) - putInt(AudioDecode.TARGET_SAMPLE_RATE * 2) + putInt(sampleRate) + putInt(sampleRate * 2) putShort(2) putShort(16) put("data".toByteArray(Charsets.US_ASCII)) diff --git a/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/DictateStatsTest.kt b/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/DictateStatsTest.kt index 1d42b24d1..89711b3b2 100644 --- a/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/DictateStatsTest.kt +++ b/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/DictateStatsTest.kt @@ -29,4 +29,31 @@ class DictateStatsTest { fun wordCountTreatsUnspacedCjkAsOneToken() { assertEquals(1, DictateStats.wordCount("你好世界")) } + + @Test + fun activityParsesDailyStatsIntoFixedWindow() { + val bars = DictateStats.activity("10:3;12:7", today = 12) + + assertEquals( + listOf( + DictateStats.DayBar(6, 0), + DictateStats.DayBar(7, 0), + DictateStats.DayBar(8, 0), + DictateStats.DayBar(9, 0), + DictateStats.DayBar(10, 3), + DictateStats.DayBar(11, 0), + DictateStats.DayBar(12, 7), + ), + bars, + ) + } + + @Test + fun activityIgnoresMalformedDailyStatsEntries() { + val bars = DictateStats.activity("bad;8:x;9:4:extra;10:5;11:", today = 10) + + assertEquals(5, bars.last().words) + assertEquals(0, bars.first { it.epochDay == 8L }.words) + assertEquals(0, bars.first { it.epochDay == 9L }.words) + } } diff --git a/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/Pcm16ResamplerTest.kt b/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/Pcm16ResamplerTest.kt index 1d81aa896..74f9ec4f5 100644 --- a/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/Pcm16ResamplerTest.kt +++ b/app/src/test/kotlin/dev/patrickgold/florisboard/dictate/Pcm16ResamplerTest.kt @@ -52,6 +52,75 @@ class Pcm16ResamplerTest { ) } + @Test + fun resamplingDropsIncompleteTrailingPcmByte() { + val pcm = byteArrayOf(1, 0, 2, 0, 99) + + assertContentEquals( + byteArrayOf(1, 0, 2, 0), + Pcm16Resampler.resample(pcm, len = 5, srcRate = 16_000, dstRate = 16_000), + ) + assertContentEquals( + Pcm16Resampler.resample(pcm, len = 4, srcRate = 16_000, dstRate = 24_000), + Pcm16Resampler.resample(pcm, len = 5, srcRate = 16_000, dstRate = 24_000), + ) + } + + @Test + fun resampleInto16kTo24kMatchesAllocatingPathAndLeavesTailUntouched() { + val pcm = deterministicPcm16(sampleCount = 257) + val len = 257 * 2 + val expected = Pcm16Resampler.resample(pcm, len, srcRate = 16_000, dstRate = 24_000) + val out = tailBuffer(expected.size + 9) + + val written = Pcm16Resampler.resampleInto(pcm, len, srcRate = 16_000, dstRate = 24_000, out = out) + + assertEquals(expected.size, written) + assertContentEquals(expected, out.copyOf(written)) + assertTailUntouched(out, start = written) + } + + @Test + fun resampleIntoGenericMatchesAllocatingPath() { + val pcm = deterministicPcm16(sampleCount = 193) + val len = 193 * 2 + val expected = Pcm16Resampler.resample(pcm, len, srcRate = 16_000, dstRate = 11_025) + val out = tailBuffer(expected.size + 7) + + val written = Pcm16Resampler.resampleInto(pcm, len, srcRate = 16_000, dstRate = 11_025, out = out) + + assertEquals(expected.size, written) + assertContentEquals(expected, out.copyOf(written)) + assertTailUntouched(out, start = written) + } + + @Test + fun resampleIntoSameRateCopiesOnlyEvenLength() { + val pcm = byteArrayOf(1, 0, 2, 0, 99, 42) + val out = tailBuffer(size = 8) + + val written = Pcm16Resampler.resampleInto(pcm, len = 5, srcRate = 16_000, dstRate = 16_000, out = out) + + assertEquals(4, written) + assertContentEquals(byteArrayOf(1, 0, 2, 0), out.copyOf(written)) + assertTailUntouched(out, start = written) + } + + @Test + fun outputLengthDropsIncompleteTrailingByte() { + val pcm = byteArrayOf(1, 0, 2, 0, 99) + + assertEquals(4, Pcm16Resampler.outputLengthBytes(len = 5, srcRate = 16_000, dstRate = 16_000)) + assertEquals( + Pcm16Resampler.outputLengthBytes(len = 4, srcRate = 16_000, dstRate = 24_000), + Pcm16Resampler.outputLengthBytes(len = 5, srcRate = 16_000, dstRate = 24_000), + ) + assertEquals( + Pcm16Resampler.resample(pcm, len = 5, srcRate = 16_000, dstRate = 24_000).size, + Pcm16Resampler.outputLengthBytes(len = 5, srcRate = 16_000, dstRate = 24_000), + ) + } + private fun deterministicPcm16(sampleCount: Int): ByteArray { val out = ByteArray(sampleCount * 2) var state = 0x13579bdf @@ -107,4 +176,16 @@ class Pcm16ResamplerTest { val hi = pcm[i * 2 + 1].toInt() return (hi shl 8) or lo } + + private fun tailBuffer(size: Int): ByteArray = ByteArray(size) { TAIL_BYTE } + + private fun assertTailUntouched(buffer: ByteArray, start: Int) { + for (i in start until buffer.size) { + assertEquals(TAIL_BYTE, buffer[i], "tail byte $i was modified") + } + } + + private companion object { + val TAIL_BYTE: Byte = 0x5a + } } diff --git a/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/data/prompts/DictatePromptDefaults.kt b/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/data/prompts/DictatePromptDefaults.kt index 969dff659..89077df1f 100644 --- a/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/data/prompts/DictatePromptDefaults.kt +++ b/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/data/prompts/DictatePromptDefaults.kt @@ -66,6 +66,12 @@ object DictatePromptDefaults { "6) Input: Mention italicize needs review before sending -> Output: Mention _needs review_ before sending.\n" + "7) Input: Just checking in with you today -> Output: Just checking in with you today." + private val AUTO_FORMATTING_ECHO_MARKERS = arrayOf( + "attentive, adaptive formatting assistant", + "Follow these rules:", + "Language hint:", + ) + /** * Assembles the full auto-formatting request: the [AUTO_FORMATTING_PROMPT] rules, a *language hint*, * and the [transcript] to clean up. Extracted (and unit-tested) so the wording stays stable across @@ -84,12 +90,7 @@ object DictatePromptDefaults { * contains. */ fun looksLikeAutoFormattingPrompt(text: String): Boolean { - val markers = listOf( - "attentive, adaptive formatting assistant", - "Follow these rules:", - "Language hint:", - ) - return markers.any { text.contains(it, ignoreCase = true) } + return AUTO_FORMATTING_ECHO_MARKERS.any { text.contains(it, ignoreCase = true) } } /** @@ -103,16 +104,37 @@ object DictatePromptDefaults { * [base] is null/blank, or `" "` otherwise. Returns null only when both are empty. */ fun appendCustomWords(base: String?, rawWords: String?): String? { - val words = rawWords.orEmpty() - .split(',', '\n') - .map { it.trim() } - .filter { it.isNotEmpty() } val baseClean = base?.trim()?.takeIf { it.isNotEmpty() } - if (words.isEmpty()) return baseClean - val glossary = words.joinToString(", ") + val glossary = rawWords?.let { buildGlossary(it) } ?: return baseClean return if (baseClean == null) glossary else "$baseClean $glossary" } + private fun buildGlossary(rawWords: String): String? { + var out: StringBuilder? = null + var start = 0 + while (start <= rawWords.length) { + val end = nextWordSeparator(rawWords, start) + var wordStart = start + var wordEnd = end + while (wordStart < wordEnd && rawWords[wordStart].isWhitespace()) wordStart++ + while (wordEnd > wordStart && rawWords[wordEnd - 1].isWhitespace()) wordEnd-- + if (wordStart < wordEnd) { + val builder = out ?: StringBuilder(rawWords.length).also { out = it } + if (builder.isNotEmpty()) builder.append(", ") + builder.append(rawWords, wordStart, wordEnd) + } + if (end == rawWords.length) break + start = end + 1 + } + return out?.toString() + } + + private fun nextWordSeparator(rawWords: String, start: Int): Int { + var i = start + while (i < rawWords.length && rawWords[i] != ',' && rawWords[i] != '\n') i++ + return i + } + /** * Returns a short example sentence (in [languageCode]) that demonstrates capitalization and * punctuation. Sent as the transcription `prompt` so the model mirrors that style. Falls back to diff --git a/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/OpenAiCompatibleClient.kt b/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/OpenAiCompatibleClient.kt index ee05b4e66..bda133156 100644 --- a/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/OpenAiCompatibleClient.kt +++ b/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/OpenAiCompatibleClient.kt @@ -966,7 +966,7 @@ class OpenAiCompatibleClient( val out = Base64StringOutput(base64Capacity(file.length())) file.inputStream().use { input -> Base64.getEncoder().wrap(out).use { base64 -> - input.copyTo(base64) + input.copyTo(base64, bufferSize = BASE64_COPY_BUFFER_SIZE) } } return out.toString() @@ -980,18 +980,20 @@ class OpenAiCompatibleClient( private class Base64StringOutput(initialCapacity: Int) : OutputStream() { private val builder = StringBuilder(initialCapacity) + private var chars = CharArray(BASE64_COPY_BUFFER_SIZE) override fun write(b: Int) { builder.append((b and 0xff).toChar()) } override fun write(buffer: ByteArray, offset: Int, length: Int) { - var i = offset - val end = offset + length - while (i < end) { - builder.append((buffer[i].toInt() and 0xff).toChar()) + if (chars.size < length) chars = CharArray(length) + var i = 0 + while (i < length) { + chars[i] = (buffer[offset + i].toInt() and 0xff).toChar() i++ } + builder.append(chars, 0, length) } override fun toString(): String = builder.toString() @@ -1227,6 +1229,7 @@ class OpenAiCompatibleClient( companion object { private val JSON_MEDIA_TYPE = "application/json; charset=utf-8".toMediaType() private const val RETRY_DELAY_MS = 3000L + private const val BASE64_COPY_BUFFER_SIZE = 64 * 1024 internal const val OPENROUTER_TRANSCRIPTION_MAX_RETRIES = 0 private const val OPENROUTER_TRANSCRIPTION_TEMPERATURE = 0.0 internal const val NETWORK_CONNECT_TIMEOUT_SECONDS = 8L diff --git a/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/RealtimeClient.kt b/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/RealtimeClient.kt index cc7a9c02d..0fd3d4407 100644 --- a/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/RealtimeClient.kt +++ b/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/RealtimeClient.kt @@ -80,6 +80,9 @@ object RealtimeClient { } } +private fun base64AudioJson(prefix: String, pcm16: ByteArray, len: Int, suffix: String): String = + prefix + Base64.encodeToString(pcm16, 0, len, Base64.NO_WRAP) + suffix + /** * OpenAI realtime transcription over `wss://api.openai.com/v1/realtime?intent=transcription`. Sends a * `session.update` transcription config on open, streams 24 kHz mono PCM16 as base64 @@ -104,6 +107,8 @@ private class OpenAiRealtimeSession( private companion object { const val URL = "wss://api.openai.com/v1/realtime?intent=transcription" + const val AUDIO_APPEND_PREFIX = "{\"type\":\"input_audio_buffer.append\",\"audio\":\"" + const val AUDIO_APPEND_SUFFIX = "\"}" } fun connect() { @@ -175,17 +180,14 @@ private class OpenAiRealtimeSession( }.toString() override fun sendAudio(pcm16: ByteArray, len: Int) { + if (done) return audioGate.sendAudio(pcm16, len) { audio, length -> ws?.let { sendAudioFrame(it, audio, length) } } } private fun sendAudioFrame(socket: WebSocket, pcm16: ByteArray, len: Int) { - val b64 = Base64.encodeToString(pcm16, 0, len, Base64.NO_WRAP) - val msg = buildJsonObject { - put("type", "input_audio_buffer.append") - put("audio", b64) - }.toString() + val msg = base64AudioJson(AUDIO_APPEND_PREFIX, pcm16, len, AUDIO_APPEND_SUFFIX) runCatching { socket.send(msg) } } @@ -297,6 +299,7 @@ private class SonioxRealtimeSession( }.toString() override fun sendAudio(pcm16: ByteArray, len: Int) { + if (done) return audioGate.sendAudio(pcm16, len) { audio, length -> runCatching { ws?.send(audio.toByteString(0, length)) } } @@ -387,7 +390,9 @@ private class AssemblyAiRealtimeSession( } override fun sendAudio(pcm16: ByteArray, len: Int) { - runCatching { ws?.send(pcm16.toByteString(0, len)) } + if (done) return + val socket = ws ?: return + runCatching { socket.send(pcm16.toByteString(0, len)) } } override fun finish() { @@ -436,6 +441,13 @@ private class ElevenLabsRealtimeSession( private val audioGate = RealtimeAudioGate() @Volatile private var done = false + private companion object { + const val AUDIO_CHUNK_PREFIX = + "{\"message_type\":\"input_audio_chunk\",\"audio_base_64\":\"" + const val AUDIO_CHUNK_SUFFIX = "\",\"commit\":false,\"sample_rate\":16000}" + const val FINAL_CHUNK = "{\"message_type\":\"input_audio_chunk\",\"audio_base_64\":\"\",\"commit\":true}" + } + fun connect() { val lang = if (!language.isNullOrBlank() && language != "detect") "&language_code=$language" else "" val url = "wss://api.elevenlabs.io/v1/speech-to-text/realtime" + @@ -469,28 +481,19 @@ private class ElevenLabsRealtimeSession( } override fun sendAudio(pcm16: ByteArray, len: Int) { + if (done) return audioGate.sendAudio(pcm16, len) { audio, length -> ws?.let { sendAudioFrame(it, audio, length) } } } private fun sendAudioFrame(socket: WebSocket, pcm16: ByteArray, len: Int) { - val msg = buildJsonObject { - put("message_type", "input_audio_chunk") - put("audio_base_64", Base64.encodeToString(pcm16, 0, len, Base64.NO_WRAP)) - put("commit", false) - put("sample_rate", 16_000) - }.toString() + val msg = base64AudioJson(AUDIO_CHUNK_PREFIX, pcm16, len, AUDIO_CHUNK_SUFFIX) runCatching { socket.send(msg) } } private fun sendCommit(socket: WebSocket) { - val msg = buildJsonObject { - put("message_type", "input_audio_chunk") - put("audio_base_64", "") - put("commit", true) - }.toString() - runCatching { socket.send(msg) } + runCatching { socket.send(FINAL_CHUNK) } } override fun finish() { @@ -545,6 +548,11 @@ private class GeminiRealtimeSession( @Volatile private var finishing = false @Volatile private var done = false + private companion object { + const val AUDIO_PREFIX = "{\"realtimeInput\":{\"audio\":{\"data\":\"" + const val AUDIO_SUFFIX = "\",\"mimeType\":\"audio/pcm;rate=16000\"}}}" + } + fun connect() { val url = "wss://generativelanguage.googleapis.com/ws/" + "google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=$apiKey" @@ -599,20 +607,14 @@ private class GeminiRealtimeSession( }.toString() override fun sendAudio(pcm16: ByteArray, len: Int) { + if (done) return audioGate.sendAudio(pcm16, len) { audio, length -> ws?.let { sendAudioFrame(it, audio, length) } } } private fun sendAudioFrame(socket: WebSocket, pcm16: ByteArray, len: Int) { - val msg = buildJsonObject { - putJsonObject("realtimeInput") { - putJsonObject("audio") { - put("data", Base64.encodeToString(pcm16, 0, len, Base64.NO_WRAP)) - put("mimeType", "audio/pcm;rate=16000") - } - } - }.toString() + val msg = base64AudioJson(AUDIO_PREFIX, pcm16, len, AUDIO_SUFFIX) runCatching { socket.send(msg) } } @@ -716,7 +718,9 @@ private class MistralRealtimeSession( }.toString() override fun sendAudio(pcm16: ByteArray, len: Int) { - runCatching { ws?.send(pcm16.toByteString(0, len)) } + if (done) return + val socket = ws ?: return + runCatching { socket.send(pcm16.toByteString(0, len)) } } override fun finish() { @@ -799,7 +803,9 @@ private class DeepgramRealtimeSession( } override fun sendAudio(pcm16: ByteArray, len: Int) { - runCatching { ws?.send(pcm16.toByteString(0, len)) } + if (done) return + val socket = ws ?: return + runCatching { socket.send(pcm16.toByteString(0, len)) } } override fun finish() { diff --git a/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/RealtimeTranscription.kt b/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/RealtimeTranscription.kt index 599561a5e..740d5a692 100644 --- a/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/RealtimeTranscription.kt +++ b/lib/dictate-core/src/main/kotlin/dev/patrickgold/florisboard/dictate/provider/RealtimeTranscription.kt @@ -61,7 +61,10 @@ data class RealtimeRequest( * waiting for a final result. All methods are safe to call from the audio/capture thread. */ interface RealtimeSession { - /** Push [len] bytes of mono 16-bit LE PCM (at [RealtimeRequest.sampleRate]) to the provider. */ + /** + * Push [len] bytes of mono 16-bit LE PCM (at [RealtimeRequest.sampleRate]) to the provider. + * Implementations must consume or copy the buffer before returning; callers may reuse it. + */ fun sendAudio(pcm16: ByteArray, len: Int) /** Signal that speaking has ended: flush buffered audio and request the final transcript, then close. */ diff --git a/wear/src/main/kotlin/net/devemperor/dictate/wear/audio/WearAudioRecorder.kt b/wear/src/main/kotlin/net/devemperor/dictate/wear/audio/WearAudioRecorder.kt index 97a99c722..3c954800e 100644 --- a/wear/src/main/kotlin/net/devemperor/dictate/wear/audio/WearAudioRecorder.kt +++ b/wear/src/main/kotlin/net/devemperor/dictate/wear/audio/WearAudioRecorder.kt @@ -16,9 +16,11 @@ import android.media.AudioFormat import android.media.AudioRecord import android.media.MediaRecorder import java.io.File +import java.io.IOException import java.io.RandomAccessFile import java.nio.ByteBuffer import java.nio.ByteOrder +import java.util.concurrent.atomic.AtomicInteger /** * Minimal microphone recorder for the watch (#106). Captures 16 kHz mono 16-bit PCM via [AudioRecord] @@ -37,17 +39,14 @@ class WearAudioRecorder(private val context: Context) { @Volatile private var paused = false @Volatile private var pcmBytes = 0L /** Peak |sample| (0..32767) seen since the last [maxAmplitude] call; drives the live waveform. */ - @Volatile private var peak = 0 + private val peak = AtomicInteger(0) + @Volatile private var recordingError: Throwable? = null private var outputFile: File? = null val isRecording: Boolean get() = recording /** Peak microphone amplitude (0..32767) since the previous call, then resets. 0 when not recording. */ - fun maxAmplitude(): Int { - val p = peak - peak = 0 - return p - } + fun maxAmplitude(): Int = peak.getAndSet(0) @SuppressLint("MissingPermission") // caller guarantees RECORD_AUDIO; init failure is handled below. fun start() { @@ -76,31 +75,51 @@ class WearAudioRecorder(private val context: Context) { runCatching { recorder.release() } throw t } - peak = 0 + peak.set(0) + recordingError = null pcmBytes = 0L outputFile = file raf = out record = recorder - recording = true paused = false - recorder.startRecording() + try { + recorder.startRecording() + } catch (t: Throwable) { + record = null + raf = null + outputFile = null + runCatching { out.close() } + file.delete() + runCatching { recorder.release() } + throw t + } + recording = true thread = Thread { val buf = ByteArray(bufferSize) while (recording) { val read = recorder.read(buf, 0, buf.size) + if (!recording) break // Keep draining the mic while paused (so the buffer never overflows) but drop the audio, // so paused time contributes no samples — matching the phone's pause behavior. - if (read > 0 && !paused) { - runCatching { out.write(buf, 0, read) } - pcmBytes += read - updatePeak(buf, read) + if (read < 0) { + recordingError = IOException("AudioRecord.read failed: $read") + recording = false + } else if (read > 0 && !paused) { + try { + out.write(buf, 0, read) + pcmBytes += read + updatePeak(buf, read) + } catch (t: Throwable) { + recordingError = t + recording = false + } } } }.also { it.start() } } private fun updatePeak(buf: ByteArray, length: Int) { - var max = peak + var max = 0 var i = 0 while (i + 1 < length) { val sample = (buf[i].toInt() and 0xff) or (buf[i + 1].toInt() shl 8) // little-endian PCM16 @@ -108,7 +127,12 @@ class WearAudioRecorder(private val context: Context) { if (abs > max) max = abs i += 2 } - peak = max.coerceAtMost(32767) + val capped = max.coerceAtMost(32767) + while (true) { + val current = peak.get() + val next = if (capped > current) capped else current + if (peak.compareAndSet(current, next)) return + } } /** Pause capture: the mic keeps running but recorded samples are dropped until [resume]. */ @@ -120,14 +144,26 @@ class WearAudioRecorder(private val context: Context) { /** Stops capture, releases the recorder and returns the recorded audio as a `.wav` file. */ fun stop(): File { recording = false - thread?.join() + val recorder = record + runCatching { recorder?.stop() } + runCatching { thread?.join() } thread = null - record?.run { stop(); release() } + runCatching { recorder?.release() } record = null val out = checkNotNull(raf) { "Recorder was not started" } raf = null val file = checkNotNull(outputFile) { "Recorder output file missing" } + val failure = recordingError + recordingError = null + if (failure != null) { + runCatching { out.close() } + file.delete() + outputFile = null + pcmBytes = 0L + peak.set(0) + throw failure + } return try { out.seek(0) out.write(wavHeader(pcmBytes)) @@ -140,20 +176,25 @@ class WearAudioRecorder(private val context: Context) { } finally { outputFile = null pcmBytes = 0L + peak.set(0) } } fun cancel() { recording = false - thread?.join() + val recorder = record + runCatching { recorder?.stop() } + runCatching { thread?.join() } thread = null - record?.run { stop(); release() } + runCatching { recorder?.release() } record = null + recordingError = null runCatching { raf?.close() } raf = null outputFile?.delete() outputFile = null pcmBytes = 0L + peak.set(0) } private fun wavHeader(dataLen: Long): ByteArray {