From bdd253343c71997454ccb9d3b6f5f5fc04d87190 Mon Sep 17 00:00:00 2001 From: AliReza Taleghani Date: Fri, 12 Jun 2026 00:30:50 +0200 Subject: [PATCH] Honor system pitch and fix English sample in TTS settings Pitch (dsp/PitchShifter): - Add a duration-preserving SOLA pitch shifter (time-stretch + resample) so the Android system pitch setting is honored. The default path (factor ~1.0) returns audio untouched, so normal playback is unchanged and free. - Wire into AvaTtsService: when pitch != default, synthesize the sentence, shift, then write; otherwise keep the streaming path. - Fix an index overflow in the resampler on long buffers (compute position directly in double precision instead of accumulating, with a bound guard). Found on device (ArrayIndexOutOfBounds at ~63k samples, factor 0.87). TTS settings sample (TtsDataCheckActivity): - GET_SAMPLE_TEXT now returns result code LANG_AVAILABLE instead of RESULT_OK. The settings screen only accepts the engine's sample when the code is LANG_AVAILABLE; with RESULT_OK it silently fell back to its built-in English string, which the Persian voice spoke as gibberish. Diagnostics: - Log the request rate/pitch and a text preview in onSynthesizeText. Verified on device (OnePlus Nord): "Play sample" now speaks Persian; pitch 0.79 applied with 65k frames delivered and no crash. Unit tests cover pitch up/down, duration preservation, and the large-buffer regression. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../avacore/TtsDataCheckActivity.kt | 8 +- .../opscalehub/avacore/dsp/PitchShifter.kt | 110 ++++++++++++++++++ .../avacore/service/AvaTtsService.kt | 24 +++- .../avacore/dsp/PitchShifterTest.kt | 72 ++++++++++++ 4 files changed, 207 insertions(+), 7 deletions(-) create mode 100644 app/src/main/java/com/github/opscalehub/avacore/dsp/PitchShifter.kt create mode 100644 app/src/test/java/com/github/opscalehub/avacore/dsp/PitchShifterTest.kt diff --git a/app/src/main/java/com/github/opscalehub/avacore/TtsDataCheckActivity.kt b/app/src/main/java/com/github/opscalehub/avacore/TtsDataCheckActivity.kt index ef47237..63255b1 100644 --- a/app/src/main/java/com/github/opscalehub/avacore/TtsDataCheckActivity.kt +++ b/app/src/main/java/com/github/opscalehub/avacore/TtsDataCheckActivity.kt @@ -36,9 +36,13 @@ class TtsDataCheckActivity : Activity() { val lang = intent.getStringExtra("language") Log.d(TAG, "GET_SAMPLE_TEXT for language: $lang") - val sampleText = "این یک آزمایش از موتور بازگو کننده آوا است." + val sampleText = "این یک آزمایش از موتور بازگوکننده آوا است." resultIntent.putExtra(TextToSpeech.Engine.EXTRA_SAMPLE_TEXT, sampleText) - setResult(RESULT_OK, resultIntent) + // The TTS settings screen only accepts the sample when the result + // code is LANG_AVAILABLE; returning RESULT_OK makes it silently fall + // back to its built-in English string (spoken by the Persian voice + // as gibberish). This is the fix for that. + setResult(TextToSpeech.LANG_AVAILABLE, resultIntent) } TextToSpeech.Engine.ACTION_INSTALL_TTS_DATA -> { // Since it's offline and included, we just say okay diff --git a/app/src/main/java/com/github/opscalehub/avacore/dsp/PitchShifter.kt b/app/src/main/java/com/github/opscalehub/avacore/dsp/PitchShifter.kt new file mode 100644 index 0000000..df05261 --- /dev/null +++ b/app/src/main/java/com/github/opscalehub/avacore/dsp/PitchShifter.kt @@ -0,0 +1,110 @@ +package com.github.opscalehub.avacore.dsp + +import kotlin.math.abs +import kotlin.math.roundToInt + +/** + * Duration-preserving pitch shifter for 16 kHz–24 kHz speech. + * + * VITS/Sherpa exposes no pitch parameter, so to honor the Android system pitch + * setting we post-process the synthesized waveform. A naive resample would shift + * formants and sound robotic; instead we use SOLA (Synchronized Overlap-Add) + * time-stretching followed by resampling, which keeps duration constant while + * moving the perceived pitch and is far cleaner on speech. + * + * pitch up (factor > 1): time-stretch longer, then resample faster + * pitch down (factor < 1): time-compress, then resample slower + * + * The default path (factor ≈ 1.0) returns the input untouched, so normal + * playback is bit-for-bit unchanged and incurs zero cost. + * + * Pure Kotlin, no dependencies — unit-testable on the JVM. + */ +object PitchShifter { + + private const val FRAME = 1024 + private const val SYNTH_HOP = FRAME / 2 // 512, => overlap = 512 + private const val SEARCH = 256 // ± similarity search range + private const val CORR_STRIDE = 8 // subsample correlation for speed + private const val MIN_FACTOR = 0.5f + private const val MAX_FACTOR = 2.0f + private const val DEADBAND = 0.01f + + /** Shift pitch by [pitchFactor] (1.0 = unchanged) while preserving duration. */ + fun process(input: FloatArray, pitchFactor: Float): FloatArray { + if (input.size < FRAME * 2 || abs(pitchFactor - 1f) < DEADBAND) return input + val factor = pitchFactor.coerceIn(MIN_FACTOR, MAX_FACTOR) + val stretched = solaTimeStretch(input, factor) + return resampleByStep(stretched, factor) + } + + /** SOLA time-stretch: output length ≈ x.size * alpha, pitch unchanged. */ + private fun solaTimeStretch(x: FloatArray, alpha: Float): FloatArray { + val analysisHop = (SYNTH_HOP / alpha).roundToInt().coerceAtLeast(1) + val overlap = FRAME - SYNTH_HOP + val out = FloatArray((x.size * alpha).toInt() + FRAME) + + System.arraycopy(x, 0, out, 0, minOf(FRAME, x.size)) + var outPos = SYNTH_HOP + var inBase = analysisHop + + while (inBase + FRAME + SEARCH < x.size && outPos + FRAME < out.size) { + // Find the read offset whose overlap region best matches what we have + // already written, so the overlap-add stays phase-aligned (no warble). + var bestK = 0 + var bestCorr = -Float.MAX_VALUE + val kStart = maxOf(-SEARCH, -inBase) + for (k in kStart..SEARCH) { + var corr = 0f + var j = 0 + while (j < overlap) { + corr += out[outPos + j] * x[inBase + k + j] + j += CORR_STRIDE + } + if (corr > bestCorr) { + bestCorr = corr + bestK = k + } + } + val readPos = inBase + bestK + + // Linear cross-fade over the overlap, then copy the frame's tail. + var j = 0 + while (j < overlap) { + val w = j.toFloat() / overlap + out[outPos + j] = out[outPos + j] * (1f - w) + x[readPos + j] * w + j++ + } + var t = overlap + while (t < FRAME && outPos + t < out.size && readPos + t < x.size) { + out[outPos + t] = x[readPos + t] + t++ + } + + outPos += SYNTH_HOP + inBase += analysisHop + } + return out.copyOf(minOf(outPos + overlap, out.size)) + } + + /** Linear-interpolating resample reading [step] input samples per output sample. */ + private fun resampleByStep(x: FloatArray, step: Float): FloatArray { + if (x.isEmpty()) return x + val outLen = (x.size / step).toInt().coerceAtLeast(1) + val out = FloatArray(outLen) + val last = x.size - 1 + for (i in 0 until outLen) { + // Compute position directly (not accumulated) in double precision so + // rounding cannot drift the index past the end over long buffers. + val pos = i * step.toDouble() + val idx = pos.toInt() + if (idx >= last) { + out[i] = x[last] + } else { + val frac = (pos - idx).toFloat() + out[i] = x[idx] + (x[idx + 1] - x[idx]) * frac + } + } + return out + } +} diff --git a/app/src/main/java/com/github/opscalehub/avacore/service/AvaTtsService.kt b/app/src/main/java/com/github/opscalehub/avacore/service/AvaTtsService.kt index 0670f38..6294793 100644 --- a/app/src/main/java/com/github/opscalehub/avacore/service/AvaTtsService.kt +++ b/app/src/main/java/com/github/opscalehub/avacore/service/AvaTtsService.kt @@ -7,8 +7,10 @@ import android.speech.tts.TextToSpeech import android.speech.tts.TextToSpeechService import android.speech.tts.Voice import android.util.Log +import com.github.opscalehub.avacore.dsp.PitchShifter import com.github.opscalehub.avacore.nlp.PronunciationLexicon import com.github.opscalehub.avacore.nlp.TextProcessor +import kotlin.math.abs import com.k2fsa.sherpa.onnx.OfflineTts import com.k2fsa.sherpa.onnx.OfflineTtsConfig import com.k2fsa.sherpa.onnx.OfflineTtsModelConfig @@ -247,8 +249,13 @@ class AvaTtsService : TextToSpeechService() { val speed = computeSpeed(request) val sampleRate = engine.sampleRate() - // request.pitch is read but intentionally not applied: VITS has no pitch - // control and naive resampling shifts formants and degrades quality. + // Honor the system pitch setting (100 = normal). Pitch shifting is applied + // as a post-process (SOLA) only when non-default; the default path streams + // untouched audio at full speed. + val pitchFactor = (request.pitch / 100f).coerceIn(0.5f, 2.0f) + val applyPitch = abs(pitchFactor - 1f) >= 0.01f + Log.d(TAG, "synthesize: rate=${request.speechRate} pitch=${request.pitch} " + + "len=${rawText.length} text='${rawText.take(60).replace('\n', ' ')}'") try { val units = processor.process(rawText) @@ -265,9 +272,16 @@ class AvaTtsService : TextToSpeechService() { for (unit in units) { if (isInterrupted.get() || isDestroyed.get()) break if (unit.text.isNotBlank()) { - engine.generateWithCallback(unit.text, 0, speed) { chunk -> - writer.write(chunk) - if (isInterrupted.get() || isDestroyed.get()) 0 else 1 + if (applyPitch) { + // Pitch path: synthesize the (short) sentence, shift, then write. + val audio = engine.generate(unit.text, 0, speed) + if (isInterrupted.get() || isDestroyed.get()) break + writer.write(PitchShifter.process(audio.samples, pitchFactor)) + } else { + engine.generateWithCallback(unit.text, 0, speed) { chunk -> + writer.write(chunk) + if (isInterrupted.get() || isDestroyed.get()) 0 else 1 + } } } if (isInterrupted.get() || isDestroyed.get()) break diff --git a/app/src/test/java/com/github/opscalehub/avacore/dsp/PitchShifterTest.kt b/app/src/test/java/com/github/opscalehub/avacore/dsp/PitchShifterTest.kt new file mode 100644 index 0000000..d5da04a --- /dev/null +++ b/app/src/test/java/com/github/opscalehub/avacore/dsp/PitchShifterTest.kt @@ -0,0 +1,72 @@ +package com.github.opscalehub.avacore.dsp + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertSame +import org.junit.Assert.assertTrue +import org.junit.Test +import kotlin.math.PI +import kotlin.math.sin + +class PitchShifterTest { + + private val sampleRate = 22050 + + private fun sine(freq: Double, samples: Int): FloatArray = + FloatArray(samples) { sin(2.0 * PI * freq * it / sampleRate).toFloat() } + + /** Estimate fundamental via positive-going zero crossings. */ + private fun estimateFreq(x: FloatArray): Double { + var crossings = 0 + for (i in 1 until x.size) if (x[i - 1] <= 0f && x[i] > 0f) crossings++ + return crossings.toDouble() * sampleRate / x.size + } + + @Test fun defaultFactor_returnsInputUntouched() { + val x = sine(200.0, 8000) + assertSame("factor 1.0 must be a no-op", x, PitchShifter.process(x, 1.0f)) + } + + @Test fun shortInput_returnsInputUntouched() { + val x = sine(200.0, 100) + assertSame(x, PitchShifter.process(x, 1.5f)) + } + + @Test fun pitchUp_preservesDuration() { + val x = sine(200.0, 22050) // 1s + val y = PitchShifter.process(x, 1.5f) + // duration within 5% + assertTrue("len ${y.size}", kotlin.math.abs(y.size - x.size) < x.size * 0.05) + } + + @Test fun pitchUp_raisesFundamental() { + val base = sine(200.0, 22050) + val up = PitchShifter.process(base, 1.5f) + val f = estimateFreq(up) + // expect ~300 Hz; allow generous tolerance for the estimator + assertTrue("got $f Hz", f in 255.0..345.0) + } + + @Test fun pitchDown_lowersFundamental() { + val base = sine(200.0, 22050) + val down = PitchShifter.process(base, 0.75f) + val f = estimateFreq(down) + // expect ~150 Hz + assertTrue("got $f Hz", f in 120.0..180.0) + } + + @Test fun largeBuffer_nonTrivialFactor_doesNotOverflow() { + // Regression: a ~63k-sample buffer with factor 0.87 previously drove the + // resampler index one past the end via float accumulation drift. + val x = sine(190.0, 63488) + val y = PitchShifter.process(x, 0.87f) + assertTrue(y.isNotEmpty()) + for (v in y) assertTrue(v.isFinite()) + } + + @Test fun output_isFinite() { + val x = sine(180.0, 16000) + for (v in PitchShifter.process(x, 1.3f)) { + assertTrue("non-finite sample", v.isFinite()) + } + } +}