From 2489454987aa5b95eab82b60c5016c0c55fc57ea Mon Sep 17 00:00:00 2001 From: silentbil Date: Sat, 4 Jul 2026 12:19:58 +0300 Subject: [PATCH 01/11] find best match --- .../player/AiSubtitleRenderersFactory.kt | 97 +++- .../screens/player/AudioCaptureProcessor.kt | 141 ++++++ .../player/GeminiLiveTranslationService.kt | 316 ++++++++++++ .../tv/ui/screens/player/PlayerScreen.kt | 162 +++++- .../tv/ui/screens/player/PlayerViewModel.kt | 470 +++++++++++++++++- .../ui/screens/player/SubtitleSyncMatcher.kt | 179 +++++++ .../tv/ui/screens/settings/SettingsScreen.kt | 18 +- .../ui/screens/settings/SettingsViewModel.kt | 12 + app/src/main/res/values/strings.xml | 2 + 9 files changed, 1376 insertions(+), 21 deletions(-) create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/player/AudioCaptureProcessor.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/player/GeminiLiveTranslationService.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt index ab12d5a75..d268604a4 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt @@ -6,6 +6,8 @@ import androidx.media3.common.text.Cue import androidx.media3.common.text.CueGroup import androidx.media3.exoplayer.DefaultRenderersFactory import androidx.media3.exoplayer.Renderer +import androidx.media3.exoplayer.audio.AudioSink +import androidx.media3.exoplayer.audio.DefaultAudioSink import androidx.media3.exoplayer.text.TextOutput import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job @@ -26,6 +28,39 @@ class AiSubtitleRenderersFactory( val syncOffsetUs = java.util.concurrent.atomic.AtomicLong(0L) + var audioCaptureProcessor: AudioCaptureProcessor? = null + private set + + private val offsetRenderers = mutableListOf() + + /** + * Cue-visible intervals (startMs to endMs) of the currently selected text track, read from + * ExoPlayer's buffered cue list via reflection — i.e. upcoming cues that haven't rendered yet. + * Lets "Find best match" build its built-in reference from prefetched cues instead of waiting + * for playback to reach them. Returns empty if nothing is buffered yet. + */ + fun extractBufferedReferenceIntervals(maxCount: Int): List> { + for (renderer in offsetRenderers) { + val intervals = renderer.extractBufferedIntervals(maxCount) + if (intervals.isNotEmpty()) return intervals + } + return emptyList() + } + + override fun buildAudioSink( + context: Context, + enableFloatOutput: Boolean, + enableAudioTrackPlaybackParams: Boolean + ): AudioSink { + val capture = AudioCaptureProcessor() + audioCaptureProcessor = capture + return DefaultAudioSink.Builder(context) + .setEnableFloatOutput(enableFloatOutput) + .setEnableAudioTrackPlaybackParams(enableAudioTrackPlaybackParams) + .setAudioProcessors(arrayOf(capture)) + .build() + } + override fun buildTextRenderers( context: Context, output: TextOutput, @@ -41,7 +76,7 @@ class AiSubtitleRenderersFactory( ) val startIndex = out.size super.buildTextRenderers(context, translatingOutput, outputLooper, extensionRendererMode, out) - val offsetRenderers = mutableListOf() + offsetRenderers.clear() for (index in startIndex until out.size) { val offsetRenderer = SubtitleOffsetRenderer( baseRenderer = out[index], @@ -374,6 +409,66 @@ private class SubtitleOffsetRenderer( return texts.toList() } + /** Buffered cue intervals (startMs, endMs) for text-carrying cues, from the modern resolver. */ + fun extractBufferedIntervals(maxCount: Int): List> { + val intervals = ArrayList>() + // Cue buffer timestamps are on ExoPlayer's internal stream timeline, which adds a large + // base offset; subtract the renderer's stream offset to get 0-based media time. + val offsetMs = readStreamOffsetUs() / 1000L + try { + val resolverField = findField(baseRenderer.javaClass, "cuesResolver") ?: return emptyList() + val resolver = resolverField.get(baseRenderer) ?: return emptyList() + for (candidate in listOf("cuesWithTimingList", "cueGroupsByStartTime", "cueGroups", "cueGroupList", "groups")) { + val f = findField(resolver.javaClass, candidate) ?: continue + val v = f.get(resolver) ?: continue + val items: Collection<*> = when (v) { + is Map<*, *> -> v.values + is Collection<*> -> v + else -> continue + } + for (item in items) { + val (s, e) = item?.let(::intervalFromCueWrapper) ?: continue + intervals.add((s - offsetMs) to (e - offsetMs)) + } + if (intervals.isNotEmpty()) break + } + } catch (_: Exception) { + } + return intervals.filter { it.first >= 0 }.distinct().sortedBy { it.first }.take(maxCount) + } + + /** The renderer's stream offset (µs) — the base added to buffer sample timestamps. */ + private fun readStreamOffsetUs(): Long { + for (name in listOf("streamOffsetUs", "outputStreamOffsetUs")) { + val v = runCatching { findField(baseRenderer.javaClass, name)?.getLong(baseRenderer) }.getOrNull() + if (v != null && v > 0L) return v + } + return 0L + } + + private fun intervalFromCueWrapper(obj: Any): Pair? { + val cues: List? = when (obj) { + is CueGroup -> obj.cues + is List<*> -> obj.filterIsInstance() + else -> (findField(obj.javaClass, "cues")?.get(obj) as? List<*>)?.filterIsInstance() + } + if (cues.isNullOrEmpty() || cues.none { !it.text?.toString()?.trim().isNullOrBlank() }) return null + + val startUs: Long + val endUs: Long + if (obj is CueGroup) { + startUs = obj.presentationTimeUs + endUs = startUs + 2_000_000L // CueGroup carries no duration — assume 2s on screen + } else { + val start = runCatching { findField(obj.javaClass, "startTimeUs")?.getLong(obj) }.getOrNull() ?: return null + val dur = runCatching { findField(obj.javaClass, "durationUs")?.getLong(obj) }.getOrNull() ?: 2_000_000L + startUs = start + endUs = start + dur + } + if (endUs <= startUs) return null + return (startUs / 1000L) to (endUs / 1000L) + } + private fun extractFromCollectionOrMap(v: Any, texts: MutableSet, removeHI: Boolean): Int { var count = 0 when (v) { diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AudioCaptureProcessor.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AudioCaptureProcessor.kt new file mode 100644 index 000000000..e37bd5384 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AudioCaptureProcessor.kt @@ -0,0 +1,141 @@ +package com.arflix.tv.ui.screens.player + +import android.util.Log +import androidx.annotation.OptIn +import androidx.media3.common.C +import androidx.media3.common.audio.BaseAudioProcessor +import androidx.media3.common.audio.AudioProcessor.AudioFormat +import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException +import androidx.media3.common.util.UnstableApi +import java.nio.ByteBuffer +import java.nio.ByteOrder + +private const val TARGET_RATE = 16000 +private const val CHUNK_SAMPLES = TARGET_RATE / 10 // 100 ms per chunk + +@OptIn(UnstableApi::class) +class AudioCaptureProcessor : BaseAudioProcessor() { + + /** Set to receive (pcm16kHz, captureTimeMs) pairs; null = capture inactive. */ + @Volatile var onChunk: ((ByteArray, Long) -> Unit)? = null + + // Resampler state + private var srcStep = 1.0 // input samples per output sample + private var srcFrac = 0.0 // fractional position within current input block + private val accum = ArrayList(CHUNK_SAMPLES * 2) + private var prevMono: Short = 0 // last sample of previous block for interpolation + + @Throws(UnhandledAudioFormatException::class) + override fun onConfigure(inputAudioFormat: AudioFormat): AudioFormat { + Log.d("AudioCapture", "onConfigure encoding=${inputAudioFormat.encoding} " + + "rate=${inputAudioFormat.sampleRate} ch=${inputAudioFormat.channelCount} " + + "onChunk=${onChunk != null}") + if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT && + inputAudioFormat.encoding != C.ENCODING_PCM_FLOAT) { + // Still passthrough — return same format, just won't capture. + } + srcStep = if (inputAudioFormat.sampleRate > 0) + inputAudioFormat.sampleRate.toDouble() / TARGET_RATE else 1.0 + srcFrac = 0.0 + accum.clear() + prevMono = 0 + return inputAudioFormat + } + + override fun onFlush() { + srcFrac = 0.0 + accum.clear() + prevMono = 0 + } + + override fun onReset() { + srcFrac = 0.0 + accum.clear() + prevMono = 0 + } + + private var queueInputCallCount = 0 + override fun queueInput(inputBuffer: ByteBuffer) { + queueInputCallCount++ + if (queueInputCallCount <= 3 || queueInputCallCount % 500 == 0) { + Log.d("AudioCapture", "queueInput #$queueInputCallCount remaining=${inputBuffer.remaining()} " + + "onChunk=${onChunk != null} encoding=${inputAudioFormat.encoding}") + } + val size = inputBuffer.remaining() + val bytes = ByteArray(size) + inputBuffer.get(bytes) // advances position to limit + + // Write passthrough output + replaceOutputBuffer(size).put(bytes).flip() + + val callback = onChunk ?: return + val fmt = inputAudioFormat + if (fmt == AudioFormat.NOT_SET) return + + when (fmt.encoding) { + C.ENCODING_PCM_16BIT -> processPcm16(bytes, fmt.channelCount, callback) + C.ENCODING_PCM_FLOAT -> processPcmFloat(bytes, fmt.channelCount, callback) + // Other encodings: skip capture, passthrough only + } + } + + private fun processPcm16(bytes: ByteArray, channels: Int, callback: (ByteArray, Long) -> Unit) { + val shorts = ShortArray(bytes.size / 2) { i -> + ByteBuffer.wrap(bytes, i * 2, 2).order(ByteOrder.LITTLE_ENDIAN).short + } + resampleAndEmit(mixToMono(shorts, channels), callback) + } + + private fun processPcmFloat(bytes: ByteArray, channels: Int, callback: (ByteArray, Long) -> Unit) { + val floatBuf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer() + val shorts = ShortArray(floatBuf.remaining()) { floatBuf.get().coerceIn(-1f, 1f).let { (it * Short.MAX_VALUE).toInt().toShort() } } + resampleAndEmit(mixToMono(shorts, channels), callback) + } + + private fun mixToMono(samples: ShortArray, channels: Int): ShortArray { + if (channels == 1) return samples + if (channels == 6) { + // 5.1 channel order: FL(0), FR(1), C(2), LFE(3), BL(4), BR(5) + // ITU-R BS.775: mono = C*1.0 + (FL+FR)*0.707 + (BL+BR)*0.707 + LFE*0.0 + // Integer arithmetic: multiply by 1000, sum of weights ≈ 3828, divide by 3828 + return ShortArray(samples.size / 6) { i -> + val fl = samples[i * 6 + 0].toLong() + val fr = samples[i * 6 + 1].toLong() + val c = samples[i * 6 + 2].toLong() + val bl = samples[i * 6 + 4].toLong() + val br = samples[i * 6 + 5].toLong() + ((c * 1000 + (fl + fr + bl + br) * 707) / 3828).toInt().toShort() + } + } + return ShortArray(samples.size / channels) { i -> + var sum = 0L + for (c in 0 until channels) sum += samples[i * channels + c] + (sum / channels).toShort() + } + } + + private fun resampleAndEmit(mono: ShortArray, callback: (ByteArray, Long) -> Unit) { + if (mono.isEmpty()) return + var pos = srcFrac + while (pos < mono.size) { + val idx = pos.toInt().coerceIn(0, mono.size - 1) + val next = if (idx + 1 < mono.size) mono[idx + 1] else if (mono.size > 0) prevMono else mono[0] + val frac = pos - idx + val sample = ((mono[idx] * (1.0 - frac)) + (next * frac)).toInt().toShort() + accum.add(sample) + if (accum.size >= CHUNK_SAMPLES) { + callback(shortsToBytes(accum), System.currentTimeMillis()) + accum.clear() + } + pos += srcStep + } + srcFrac = pos - mono.size + if (mono.isNotEmpty()) prevMono = mono.last() + } + + private fun shortsToBytes(shorts: List): ByteArray { + val buf = ByteBuffer.allocate(shorts.size * 2).order(ByteOrder.LITTLE_ENDIAN) + for (s in shorts) buf.putShort(s) + return buf.array() + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/GeminiLiveTranslationService.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/GeminiLiveTranslationService.kt new file mode 100644 index 000000000..cf45b94d8 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/GeminiLiveTranslationService.kt @@ -0,0 +1,316 @@ +package com.arflix.tv.ui.screens.player + +import android.util.Base64 +import android.util.Log +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import okhttp3.WebSocket +import okhttp3.WebSocketListener +import okio.ByteString +import org.json.JSONArray +import org.json.JSONObject +import java.util.concurrent.TimeUnit + +private const val TAG = "GeminiLive" +// Specialized streaming speech-to-text-translation model: lowest latency (~0.8s) because it +// translates continuously without waiting for speech turns. Trade-off: it ignores system +// instructions and exposes no quality/style/gender controls, so translation quality is fixed. +// The general (prompt-steerable) Live models were evaluated but rejected: the TEXT-capable +// ones aren't available on this key, and the reachable native-audio model adds ~3s latency. +private const val MODEL = "models/gemini-3.5-live-translate-preview" +private const val WS_BASE = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" +private const val MIME_PCM_16K = "audio/pcm;rate=16000" +private const val CLEAR_DELAY_MS = 3_000L +// Minimum time a completed sentence stays on screen before the next queued one replaces it. +private const val MIN_LINE_MS = 900L +// If we fall this far behind on rapid dialogue, drop the oldest lines to catch back up. +private const val MAX_PENDING_LINES = 3 +private val SENTENCE_TERMINATORS = charArrayOf('.', '!', '?', '…') +// Split a run of completed text into individual sentences (break after each terminator). +private val SENTENCE_SPLIT = Regex("(?<=[.!?…])\\s+") + +enum class GeminiLiveState { DISCONNECTED, CONNECTING, READY, ERROR } + +class GeminiLiveTranslationService( + private val apiKeyProvider: () -> String, + private val scope: CoroutineScope +) { + private val _text = MutableStateFlow(null) + val translatedText: StateFlow = _text.asStateFlow() + + private val _state = MutableStateFlow(GeminiLiveState.DISCONNECTED) + val state: StateFlow = _state.asStateFlow() + + private val _errorMessage = MutableStateFlow(null) + val errorMessage: StateFlow = _errorMessage.asStateFlow() + + private var ws: WebSocket? = null + private val audioQueue = Channel(Channel.UNLIMITED) + private var senderJob: Job? = null + private var clearJob: Job? = null + + private var sentenceStartMs = 0L // wall time when first chunk was queued after last sentence end + private var firstFragmentLogged = false + private var lastFragmentTimeMs = 0L + + // Segmentation: show one sentence at a time so lines from different speakers/utterances + // never merge onto a single subtitle line. Completed sentences queue up and are drained + // one at a time; the trailing in-progress text shows live once the queue is empty. + private val pendingLines = ArrayDeque() + private var livePartial = StringBuilder() + private var lineJob: Job? = null + + private val client = OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(0, TimeUnit.SECONDS) + .build() + + fun connect() { + Log.d(TAG, "connect() state=${_state.value}") + if (_state.value == GeminiLiveState.CONNECTING || _state.value == GeminiLiveState.READY) return + val apiKey = apiKeyProvider() + if (apiKey.isBlank()) { + _state.value = GeminiLiveState.ERROR + _errorMessage.value = "API key missing" + return + } + _state.value = GeminiLiveState.CONNECTING + _errorMessage.value = null + + val request = Request.Builder().url("$WS_BASE?key=$apiKey").build() + ws = client.newWebSocket(request, object : WebSocketListener() { + override fun onOpen(socket: WebSocket, response: Response) { + Log.d(TAG, "WS open, sending setup") + socket.send(buildSetupMessage()) + _state.value = GeminiLiveState.READY + startSender() + } + override fun onMessage(socket: WebSocket, text: String) { + handleMessage(text) + } + override fun onMessage(socket: WebSocket, bytes: ByteString) { + handleMessage(bytes.utf8()) + } + override fun onFailure(socket: WebSocket, t: Throwable, response: Response?) { + val body = try { response?.body?.string() } catch (_: Exception) { null } + Log.e(TAG, "WS failure code=${response?.code} body=$body err=${t.message}") + _state.value = GeminiLiveState.ERROR + _errorMessage.value = t.message ?: "Connection failed" + stopSender() + } + override fun onClosed(socket: WebSocket, code: Int, reason: String) { + Log.e(TAG, "WS closed code=$code reason=$reason") + _state.value = GeminiLiveState.DISCONNECTED + stopSender() + } + }) + } + + private fun buildSetupMessage() = JSONObject().apply { + put("setup", JSONObject().apply { + put("model", MODEL) + put("generation_config", JSONObject().apply { + // TEXT output keeps latency low (no dubbed-speech generation). + put("response_modalities", JSONArray().apply { put("TEXT") }) + // The translate model's only real control: target language. It ignores + // system instructions and has no quality/style/gender knobs, so we don't send any. + put("translation_config", JSONObject().apply { + put("target_language_code", "he") + }) + }) + }) + }.toString() + + /** Concatenate all text parts from serverContent.modelTurn (TEXT response modality). */ + private fun extractModelTurnText(content: JSONObject): String? { + val modelTurn = content.optJSONObject("modelTurn") + ?: content.optJSONObject("model_turn") ?: return null + val parts = modelTurn.optJSONArray("parts") ?: return null + val sb = StringBuilder() + for (i in 0 until parts.length()) { + val text = parts.optJSONObject(i)?.optString("text", "").orEmpty() + if (text.isNotEmpty()) sb.append(text) + } + return sb.toString().ifBlank { null } + } + + private fun handleMessage(raw: String) { + try { + val json = JSONObject(raw) + if (json.has("setupComplete") || json.has("setup_complete")) { + Log.d(TAG, "setupComplete") + return + } + val content = json.optJSONObject("serverContent") + ?: json.optJSONObject("server_content") ?: return + + // With TEXT modality the translated text arrives in modelTurn.parts[].text. + // Fall back to outputTranscription for the AUDIO-modality shape so either works. + val fragment = extractModelTurnText(content) + ?: content.optJSONObject("outputTranscription")?.optString("text", "") + ?: content.optJSONObject("output_transcription")?.optString("text", "") + + if (!fragment.isNullOrBlank()) { + val now = System.currentTimeMillis() + // A gap of >3s means the previous utterance ended without a clean boundary — reset + // so stale text doesn't bleed into the next speaker's line. + if (lastFragmentTimeMs > 0 && now - lastFragmentTimeMs > 3_000L) { + resetSegmentation() + } + lastFragmentTimeMs = now + if (!firstFragmentLogged) { + val elapsedMs = if (sentenceStartMs > 0L) now - sentenceStartMs else -1L + Log.i(TAG, "⏱ first fragment after ${elapsedMs}ms | text=\"${fragment.trim()}\"") + firstFragmentLogged = true + } + + // TEXT parts are incremental token chunks that carry their own spacing, + // so append verbatim (no trim / no injected space) to avoid splitting words. + livePartial.append(fragment) + extractCompletedSentences() + Log.d(TAG, "live: partial=\"${livePartial.toString().trim()}\" queued=${pendingLines.size}") + startDrainer() + } + + // Turn-based (general) model: a completed turn is a clean utterance boundary — flush + // the trailing text as its own line and reset per-turn latency timing. + if (content.optBoolean("turnComplete", false) || content.optBoolean("turn_complete", false)) { + onTurnComplete() + } + } catch (e: Exception) { + Log.w(TAG, "parse error: ${e.message}") + } + } + + private fun onTurnComplete() { + val rest = livePartial.toString().trim() + if (rest.isNotEmpty()) { + pendingLines.addLast(rest) + while (pendingLines.size > MAX_PENDING_LINES) pendingLines.removeFirst() + livePartial = StringBuilder() + startDrainer() + } + sentenceStartMs = 0L + firstFragmentLogged = false + } + + /** + * Move every complete sentence out of [livePartial] into the [pendingLines] queue, leaving + * only the trailing in-progress fragment. This is what stops two speakers' sentences from + * ever being concatenated onto one line — each terminated sentence becomes its own line. + */ + private fun extractCompletedSentences() { + val s = livePartial.toString() + val lastTerm = s.indexOfLast { it in SENTENCE_TERMINATORS } + if (lastTerm < 0) return + val completed = s.substring(0, lastTerm + 1) + val rest = s.substring(lastTerm + 1) + for (sentence in SENTENCE_SPLIT.split(completed)) { + val t = sentence.trim() + if (t.isNotEmpty()) pendingLines.addLast(t) + } + livePartial = StringBuilder(rest) + // Bound the backlog so rapid dialogue doesn't drift ever further behind. + while (pendingLines.size > MAX_PENDING_LINES) pendingLines.removeFirst() + } + + /** + * Drains queued sentences one at a time, holding each for [MIN_LINE_MS] so it's readable, + * then shows the live in-progress text once caught up. Only one drainer runs at a time; + * new sentences appended while it's draining are picked up by the loop. + */ + private fun startDrainer() { + if (lineJob?.isActive == true) return + lineJob = scope.launch { + while (pendingLines.isNotEmpty()) { + val line = pendingLines.removeFirst() + _text.value = line + sentenceStartMs = 0L + firstFragmentLogged = false + delay(MIN_LINE_MS) + } + // Caught up — show whatever partial sentence is currently being spoken. + val partial = livePartial.toString().trim() + if (partial.isNotEmpty()) _text.value = partial + scheduleClear() + } + } + + private fun scheduleClear() { + clearJob?.cancel() + clearJob = scope.launch { + delay(CLEAR_DELAY_MS) + if (pendingLines.isEmpty() && livePartial.isBlank()) _text.value = null + } + } + + private fun resetSegmentation() { + lineJob?.cancel() + clearJob?.cancel() + pendingLines.clear() + livePartial = StringBuilder() + sentenceStartMs = 0L + firstFragmentLogged = false + _text.value = null + } + + fun sendAudioChunk(pcm16Bytes: ByteArray, captureTimeMs: Long) { + if (_state.value != GeminiLiveState.READY) return + if (sentenceStartMs == 0L) { + sentenceStartMs = System.currentTimeMillis() + firstFragmentLogged = false + } + audioQueue.trySend(pcm16Bytes) + } + + private fun startSender() { + Log.d(TAG, "startSender launched") + var n = 0 + senderJob = scope.launch { + for (chunk in audioQueue) { + val socket = ws ?: break + n++ + if (n <= 3 || n % 50 == 0) Log.d(TAG, "chunk #$n") + val encoded = Base64.encodeToString(chunk, Base64.NO_WRAP) + socket.send(JSONObject().apply { + put("realtimeInput", JSONObject().apply { + put("audio", JSONObject().apply { + put("data", encoded) + put("mimeType", MIME_PCM_16K) + }) + }) + }.toString()) + } + Log.d(TAG, "sender ended after $n chunks") + } + } + + private fun stopSender() { + senderJob?.cancel() + senderJob = null + } + + fun disconnect() { + stopSender() + lineJob?.cancel() + clearJob?.cancel() + ws?.close(1000, "stopped") + ws = null + _state.value = GeminiLiveState.DISCONNECTED + _text.value = null + pendingLines.clear() + livePartial = StringBuilder() + sentenceStartMs = 0L + firstFragmentLogged = false + lastFragmentTimeMs = 0L + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt index 2fe6f9a98..06eea2469 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt @@ -71,6 +71,8 @@ import androidx.compose.material.icons.filled.Settings import androidx.compose.material.icons.filled.VolumeDown import androidx.compose.material.icons.filled.VolumeMute import androidx.compose.material.icons.filled.VolumeUp +import androidx.compose.material.icons.filled.Mic +import androidx.compose.ui.platform.LocalConfiguration import androidx.compose.ui.graphics.Brush import androidx.compose.ui.graphics.vector.ImageVector import androidx.compose.ui.unit.Dp @@ -451,6 +453,8 @@ fun PlayerScreen( groups.none { (name, _) -> name.equals(uiState.aiTargetLanguageName, ignoreCase = true) }) { groups.add(0, Pair(uiState.aiTargetLanguageName, emptyList())) } + // "Find Best Match (AI)" is rendered inside the target-language (e.g. Hebrew) group as its + // first item, alongside the AI translation option — not as a global picker entry. groups.toList() } // Audio tracks from ExoPlayer @@ -791,6 +795,30 @@ fun PlayerScreen( scope = coroutineScope ) } + + // Wire AudioCaptureProcessor → GeminiLiveTranslationService when live audio is active. + LaunchedEffect(uiState.isLiveAudioTranslating) { + aiRenderersFactory.audioCaptureProcessor?.onChunk = if (uiState.isLiveAudioTranslating) { + { bytes, captureMs -> viewModel.geminiLiveService.sendAudioChunk(bytes, captureMs) } + } else { + null + } + } + // Let "Find best match" read the selected reference track's buffered (upcoming) cues, + // so built-in matching can complete before dialogue is spoken. + LaunchedEffect(aiRenderersFactory) { + viewModel.bufferedReferenceIntervalsProvider = { max -> + aiRenderersFactory.extractBufferedReferenceIntervals(max) + } + } + DisposableEffect(aiRenderersFactory) { + onDispose { + aiRenderersFactory.audioCaptureProcessor?.onChunk = null + viewModel.bufferedReferenceIntervalsProvider = null + viewModel.geminiLiveService.disconnect() + } + } + val exoPlayer = remember(isConstrainedPlaybackDevice, preferExtensionDecoder, playbackBufferProfile) { val loadControl = DefaultLoadControl.Builder() .setBufferDurationsMs( @@ -875,6 +903,12 @@ fun PlayerScreen( } } + // Feed the selected text track's rendered cues to "Find best match" so it can + // read a built-in subtitle's timing (embedded tracks have no URL to parse). + override fun onCues(cueGroup: androidx.media3.common.text.CueGroup) { + viewModel.onPlayerCues(cueGroup.cues.isNotEmpty(), cueGroup.presentationTimeUs / 1000L) + } + override fun onIsPlayingChanged(playing: Boolean) { if (BuildConfig.DEBUG) { } @@ -2374,7 +2408,7 @@ fun PlayerScreen( val aiGroup = latestUiState.isAiAvailable && latestUiState.aiTargetLanguageName.isNotBlank() && group?.first?.equals(latestUiState.aiTargetLanguageName, ignoreCase = true) == true - val trackCount = (group?.second?.size ?: 0) + if (aiGroup) 1 else 0 + val trackCount = (group?.second?.size ?: 0) + if (aiGroup) 2 else 0 if (subtitleTrackIndex < trackCount - 1) subtitleTrackIndex++ } } @@ -2442,9 +2476,12 @@ fun PlayerScreen( val aiGroup = latestUiState.isAiAvailable && latestUiState.aiTargetLanguageName.isNotBlank() && group?.first?.equals(latestUiState.aiTargetLanguageName, ignoreCase = true) == true - val realIdx = if (aiGroup) subtitleTrackIndex - 1 else subtitleTrackIndex + // In the AI/target-language group, index 0 = Find Best Match, 1 = AI translate, 2+ = subs. + val realIdx = if (aiGroup) subtitleTrackIndex - 2 else subtitleTrackIndex if (aiGroup && subtitleTrackIndex == 0) { - if (!latestUiState.isAiTranslating) viewModel.activateAiSubtitle() + viewModel.runFindBestMatch() + } else if (aiGroup && subtitleTrackIndex == 1) { + if (!latestUiState.isAiTranslating) viewModel.activateAiTranslation() } else { group?.second?.getOrNull(realIdx)?.second ?.let { viewModel.selectSubtitle(it) } @@ -2753,6 +2790,37 @@ fun PlayerScreen( } } + // (AI-hearing on-screen overlay removed — the hearing still runs under the hood to power + // "Find Best Match", but its transcription is no longer displayed.) + + // "Find Best Match" persistent searching indicator + if (hasPlaybackStarted && uiState.isFindingBestMatch) { + androidx.compose.foundation.layout.Row( + modifier = Modifier + .align(Alignment.TopCenter) + .padding(top = 16.dp) + .background( + color = androidx.compose.ui.graphics.Color.Black.copy(alpha = 0.72f), + shape = RoundedCornerShape(20.dp) + ) + .padding(horizontal = 16.dp, vertical = 8.dp) + .zIndex(6f), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(10.dp) + ) { + androidx.compose.material3.CircularProgressIndicator( + modifier = Modifier.size(14.dp), + strokeWidth = 2.dp, + color = androidx.compose.ui.graphics.Color(0xFF7EC8F0) + ) + Text( + text = uiState.matchStatusText.ifBlank { "Searching for a match…" }, + style = androidx.compose.material3.MaterialTheme.typography.labelLarge, + color = androidx.compose.ui.graphics.Color.White.copy(alpha = 0.9f) + ) + } + } + // AI translation API error toast uiState.aiErrorToast?.let { msg -> Toast( @@ -2764,6 +2832,17 @@ fun PlayerScreen( ) } + // "Find best match" status toast + uiState.matchToast?.let { msg -> + Toast( + message = msg, + type = ToastType.INFO, + isVisible = true, + durationMs = 4000, + onDismiss = { viewModel.dismissMatchToast() } + ) + } + // Netflix-style Controls Overlay AnimatedVisibility( visible = hasPlaybackStarted && showControls && !showSubtitleMenu && !showSourceMenu && !isInPipMode, @@ -3279,7 +3358,11 @@ fun PlayerScreen( try { subtitleButtonFocusRequester.requestFocus() } catch (_: Exception) {} } }, - onToggleAi = { viewModel.activateAiSubtitle() }, + isLiveAudioTranslating = uiState.isLiveAudioTranslating, + isFindingBestMatch = uiState.isFindingBestMatch, + onToggleAi = { viewModel.activateAiTranslation() }, + onToggleLiveAudio = { viewModel.toggleLiveAudioTranslation() }, + onFindBestMatch = { viewModel.runFindBestMatch() }, onClose = { showSubtitleMenu = false showControls = true @@ -4101,6 +4184,8 @@ private fun SubtitleMenu( isAiTranslating: Boolean = false, isAiAvailable: Boolean = false, aiTargetLanguageName: String = "", + isLiveAudioTranslating: Boolean = false, + isFindingBestMatch: Boolean = false, audioTracks: List, selectedAudioIndex: Int, activeTab: Int, @@ -4114,6 +4199,8 @@ private fun SubtitleMenu( onSelectSubtitle: (Int) -> Unit, onSelectAudio: (AudioTrackInfo) -> Unit, onToggleAi: () -> Unit = {}, + onToggleLiveAudio: () -> Unit = {}, + onFindBestMatch: () -> Unit = {}, onClose: () -> Unit ) { val isMobile = LocalDeviceType.current.isTouchDevice() @@ -4203,7 +4290,7 @@ private fun SubtitleMenu( count = 0, isFocused = subtitlePanelFocus == 0 && subtitleLangIndex == 0, isActivePanel = subtitleLangIndex == 0, - isSelected = selectedSubtitle == null + isSelected = selectedSubtitle == null && !isLiveAudioTranslating ) } itemsIndexed(subtitleGroups) { idx, (langName, items) -> @@ -4212,11 +4299,10 @@ private fun SubtitleMenu( count = items.size, isFocused = subtitlePanelFocus == 0 && subtitleLangIndex == idx + 1, isActivePanel = subtitleLangIndex == idx + 1, - isSelected = if (isAiTranslating && aiTargetLanguageName.isNotBlank() && - langName.equals(aiTargetLanguageName, ignoreCase = true)) { - true - } else { - selectedSubtitle != null && + isSelected = when { + (isAiTranslating || isFindingBestMatch) && aiTargetLanguageName.isNotBlank() && + langName.equals(aiTargetLanguageName, ignoreCase = true) -> true + else -> selectedSubtitle != null && items.any { (_, sub) -> sub.id == selectedSubtitle.id } } ) @@ -4248,6 +4334,7 @@ private fun SubtitleMenu( ) } } else { + val isLiveAudioGroup = selectedGroup.first == "Live Audio" val isAiGroup = isAiAvailable && aiTargetLanguageName.isNotBlank() && selectedGroup.first.equals(aiTargetLanguageName, ignoreCase = true) LazyColumn( @@ -4259,13 +4346,25 @@ private fun SubtitleMenu( verticalArrangement = Arrangement.spacedBy(2.dp) ) { if (isAiGroup) { + // First: "Find Best Match (AI)" — runs the match logic. + item { + TrackMenuItem( + label = if (isFindingBestMatch) "Scanning…" else "Find Best Match", + subtitle = "AI", + subtitleDetail = "Auto-pick the best-synced subtitle", + isSelected = isFindingBestMatch, + isFocused = subtitlePanelFocus == 1 && subtitleTrackIndex == 0, + onClick = { /* D-pad only */ } + ) + } + // Second: translate the built-in subtitle with AI. item { TrackMenuItem( label = aiTargetLanguageName, subtitle = "AI", subtitleDetail = null, isSelected = isAiTranslating, - isFocused = subtitlePanelFocus == 1 && subtitleTrackIndex == 0, + isFocused = subtitlePanelFocus == 1 && subtitleTrackIndex == 1, onClick = { /* D-pad only */ } ) } @@ -4289,12 +4388,12 @@ private fun SubtitleMenu( .ifBlank { subtitle.id } .ifBlank { null } } - val itemIdx = if (isAiGroup) idx + 1 else idx + val itemIdx = if (isAiGroup) idx + 2 else idx TrackMenuItem( label = mainLabel, subtitle = badge, subtitleDetail = detail, - isSelected = !isAiTranslating && selectedSubtitle?.id == subtitle.id, + isSelected = !isAiTranslating && !isLiveAudioTranslating && selectedSubtitle?.id == subtitle.id, isFocused = subtitlePanelFocus == 1 && subtitleTrackIndex == itemIdx, onClick = { /* D-pad only */ } ) @@ -4485,7 +4584,7 @@ private fun SubtitleMenu( MobileTrackItem( name = "Off", description = null, - isSelected = selectedSubtitle == null, + isSelected = selectedSubtitle == null && !isLiveAudioTranslating, onClick = { onSelectSubtitle(0) } ) Box( @@ -4499,6 +4598,7 @@ private fun SubtitleMenu( // Grouped by language subtitleGroups.forEach { (langName, indexedSubs) -> + val isLiveAudioGroup = langName == "Live Audio" val isAiGroup = isAiAvailable && aiTargetLanguageName.isNotBlank() && langName.equals(aiTargetLanguageName, ignoreCase = true) item(key = "mobile_header_$langName") { @@ -4514,7 +4614,39 @@ private fun SubtitleMenu( .padding(start = 16.dp, top = 8.dp, bottom = 2.dp) ) } + if (isLiveAudioGroup) { + item(key = "mobile_live_audio_item") { + MobileTrackItem( + name = "Translate Audio", + description = "AI", + isSelected = isLiveAudioTranslating, + onClick = { onToggleLiveAudio(); onClose() } + ) + Box( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 8.dp) + .height(1.dp) + .background(Color.White.copy(alpha = 0.06f)) + ) + } + } if (isAiGroup) { + item(key = "mobile_find_best_match_item") { + MobileTrackItem( + name = if (isFindingBestMatch) "Scanning…" else "Find Best Match", + description = "AI", + isSelected = isFindingBestMatch, + onClick = { onFindBestMatch(); onClose() } + ) + Box( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 8.dp) + .height(1.dp) + .background(Color.White.copy(alpha = 0.06f)) + ) + } item(key = "mobile_ai_item") { MobileTrackItem( name = aiTargetLanguageName, @@ -4548,7 +4680,7 @@ private fun SubtitleMenu( MobileTrackItem( name = displayName, description = description, - isSelected = !isAiTranslating && selectedSubtitle?.id == sub.id, + isSelected = !isAiTranslating && !isLiveAudioTranslating && selectedSubtitle?.id == sub.id, onClick = { onSelectSubtitle(originalIndex + 1) } ) Box( diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt index 82b2ebbb8..4330c2a23 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt @@ -39,6 +39,7 @@ import com.google.gson.reflect.TypeToken import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -47,8 +48,11 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull import javax.inject.Inject private fun isSupplementalStream(stream: StreamSource): Boolean = @@ -124,6 +128,13 @@ data class PlayerUiState( val aiTargetLanguageName: String = "", // Non-null while an AI translation API error toast should be visible val aiErrorToast: String? = null, + // True while Gemini Live audio translation is active for this session + val isLiveAudioTranslating: Boolean = false, + // True while "Find best match" is scanning subtitles; message shown as a transient toast. + val isFindingBestMatch: Boolean = false, + val matchToast: String? = null, + // Persistent status shown on screen for the whole duration of a "Find best match" scan. + val matchStatusText: String = "", // Episode title for TV shows (e.g. "The Devil's Verdict"), populated from TMDB season details val episodeTitle: String? = null, // Plot synopsis from TMDB, used in the pause overlay metadata block @@ -185,11 +196,17 @@ class PlayerViewModel @Inject constructor( private var lastIsPlaying: Boolean = false private var hasMarkedWatched: Boolean = false private var hasManualSubtitleSelection: Boolean = false + // True only when the user explicitly picked a subtitle track from the menu (not AI/auto-match). + // A late-arriving embedded preferred-language track overrides auto selections but never this. + private var userPickedSubtitle: Boolean = false private var playbackSessionStartTime: Long = 0L // AI subtitle settings (read once per video load) private var aiSubtitleEnabled = false private var aiSubtitleAutoSelect = false + // When true (default), activating AI first tries to auto-pick a synced Hebrew addon subtitle + // (Find Best Match); only if nothing matches does it translate the built-in subtitle. + private var aiFindBestMatchFirst = true private var aiApiKey = "" private var aiModel = SubtitleAiModel.GROQ_LLAMA_70B private var aiRemoveHearingImpaired = true @@ -197,6 +214,7 @@ class PlayerViewModel @Inject constructor( // Global AI subtitle DataStore keys (device-wide, not profile-scoped) private val aiEnabledKey = booleanPreferencesKey("subtitle_ai_enabled") private val aiAutoSelectKey = booleanPreferencesKey("subtitle_ai_auto_select") + private val aiFindBestMatchKey = booleanPreferencesKey("subtitle_ai_find_best_match") private val aiApiKeyKey = globalStringPreferencesKey("subtitle_ai_api_key") private val aiModelKey = globalStringPreferencesKey("subtitle_ai_model") private val aiRemoveHearingImpairedKey = booleanPreferencesKey("subtitle_remove_hearing_impaired") @@ -233,6 +251,38 @@ class PlayerViewModel @Inject constructor( } } + val geminiLiveService: GeminiLiveTranslationService = GeminiLiveTranslationService( + apiKeyProvider = { aiApiKey }, + scope = viewModelScope + ) + val liveAudioText = geminiLiveService.translatedText + val liveAudioState = geminiLiveService.state + + // Latest playback media position, used to time-align AI-hearing samples with subtitle cues. + @Volatile private var lastKnownPositionMs = 0L + + private var findMatchJob: Job? = null + + // Built-in reference collection: while active, rendered cues of the selected (built-in English) + // track are turned into cue-visible intervals used as the sync reference. + @Volatile private var collectingReference = false + @Volatile private var refIntervalStart = -1L + private val referenceIntervals = mutableListOf>() + + /** Set by the player screen: reads the selected text track's already-buffered cue intervals. */ + @Volatile var bufferedReferenceIntervalsProvider: ((Int) -> List>)? = null + + /** Rendered-cue callback from the player; records reference intervals during a built-in scan. */ + fun onPlayerCues(hasText: Boolean, timeMs: Long) { + if (!collectingReference || timeMs < 0) return + if (hasText) { + if (refIntervalStart < 0) refIntervalStart = timeMs + } else if (refIntervalStart in 0 until timeMs) { + synchronized(referenceIntervals) { referenceIntervals.add(refIntervalStart to timeMs) } + refIntervalStart = -1L + } + } + // Skip intro private var skipIntervals: List = emptyList() private var lastActiveSkipType: String? = null @@ -348,6 +398,7 @@ class PlayerViewModel @Inject constructor( playbackErrorReportJob?.cancel() primaryStreamResolutionFinal = false hasManualSubtitleSelection = false + userPickedSubtitle = false aiSourceSubtitle = null val cachedItem = mediaRepository.getCachedItem(mediaType, mediaId) val cachedLogoUrl = mediaRepository.peekCachedLogoUrl(mediaType, mediaId) @@ -403,6 +454,7 @@ class PlayerViewModel @Inject constructor( // Load AI subtitle settings aiSubtitleEnabled = prefs[aiEnabledKey] ?: false aiSubtitleAutoSelect = prefs[aiAutoSelectKey] ?: false + aiFindBestMatchFirst = prefs[aiFindBestMatchKey] ?: true aiApiKey = prefs[aiApiKeyKey] ?: "" aiModel = runCatching { SubtitleAiModel.valueOf(prefs[aiModelKey] ?: SubtitleAiModel.GROQ_LLAMA_70B.name) @@ -1037,6 +1089,7 @@ class PlayerViewModel @Inject constructor( } fun onPlaybackPosition(positionMs: Long) { + lastKnownPositionMs = positionMs updateActiveSkipInterval(positionMs) } @@ -1155,13 +1208,40 @@ class PlayerViewModel @Inject constructor( } private fun applyPreferredSubtitle(preference: String, subtitles: List, fallbackLanguage: String?) { + val normalizedPref = normalizeLanguage(preference) + + // Highest priority: an embedded (muxed) track in the preferred language is inherently in + // sync, so it beats any auto logic — it overrides an auto-selected/auto-matched subtitle and + // cancels a running "Find best match" scan. It never overrides a real user pick. This also + // handles embedded tracks that resolve *after* the auto-match already started. + if (!userPickedSubtitle && normalizedPref.isNotBlank() && !isSubtitleDisabledPreference(preference)) { + val embeddedPref = subtitles.firstOrNull { + it.isEmbedded && !it.isBitmap && !it.isForced && + !it.label.contains("forced", ignoreCase = true) && + (normalizeLanguage(it.lang) == normalizedPref || normalizeLanguage(it.label) == normalizedPref) + } + if (embeddedPref != null && _uiState.value.selectedSubtitle?.id != embeddedPref.id) { + cancelFindBestMatch() + hasManualSubtitleSelection = true + translationManager.isEnabled = false + aiSourceSubtitle = null + _uiState.value = _uiState.value.copy( + selectedSubtitle = embeddedPref, + isAiTranslating = false, + isAiAvailable = false, + aiTargetLanguageName = "", + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + return + } + } + if (hasManualSubtitleSelection) return if (isSubtitleDisabledPreference(preference)) { _uiState.value = _uiState.value.copy(selectedSubtitle = null) return } - val normalizedPref = normalizeLanguage(preference) val normalizedFallback = fallbackLanguage ?.let { normalizeLanguage(it) } ?.takeIf { it.isNotBlank() && it != normalizedPref } @@ -1249,10 +1329,20 @@ class PlayerViewModel @Inject constructor( } else if (aiModeActive) { // No embedded preferred-language subtitle → activate AI (addon subs are ignored) val source = findAiSourceSubtitle(subtitles) + val targetLangName = languageCodeToName(normalizedPref) if (source != null) { - val targetLangName = languageCodeToName(normalizedPref) aiSourceSubtitle = source translationManager.targetLanguage = targetLangName + } + if (aiFindBestMatchFirst) { + // Auto-run "Find best match" first: pick a synced Hebrew addon sub if one exists, + // otherwise fall back to AI-translating the built-in source (handled inside). + _uiState.value = _uiState.value.copy( + isAiAvailable = source != null, + aiTargetLanguageName = if (source != null) targetLangName else _uiState.value.aiTargetLanguageName + ) + activateAiSubtitle() + } else if (source != null) { translationManager.isEnabled = true translationManager.reset() aiErrorToastShown = false @@ -2283,8 +2373,10 @@ class PlayerViewModel @Inject constructor( } } - fun selectSubtitle(subtitle: Subtitle) { + fun selectSubtitle(subtitle: Subtitle, isUserAction: Boolean = true) { hasManualSubtitleSelection = true + userPickedSubtitle = isUserAction + if (isUserAction) cancelFindBestMatch() subtitleSelectionJob?.cancel() translationManager.isEnabled = false // Keep isAiAvailable/aiTargetLanguageName so the AI entry stays in the menu for re-selection @@ -2296,9 +2388,59 @@ class PlayerViewModel @Inject constructor( recordSubtitleUsage(subtitle) } + /** Cancel a running/queued "Find best match" scan and clear its transient state. */ + private fun cancelFindBestMatch() { + findMatchJob?.cancel() + collectingReference = false + if (_uiState.value.isFindingBestMatch || _uiState.value.isLiveAudioTranslating) { + geminiLiveService.disconnect() + _uiState.value = _uiState.value.copy( + isFindingBestMatch = false, + isLiveAudioTranslating = false, + matchStatusText = "" + ) + } + } + + /** + * The AI subtitle entry point (menu option + auto-select). When "find best match first" is on, + * it first tries to auto-pick a synced Hebrew addon subtitle; only if nothing matches does it + * fall back to AI-translating the built-in subtitle. + */ fun activateAiSubtitle() { - val source = aiSourceSubtitle ?: return + if (aiFindBestMatchFirst) { + runFindBestMatch() + } else { + activateAiTranslation() + } + } + + /** + * The "Find Best Match (AI)" menu button: always run the match logic; if nothing synced is + * found, fall back to the AI translation option when one is available. + */ + fun runFindBestMatch() { + // Show AI translation immediately (if a source exists) so the user sees subtitles right + // away — and because the AI source (English) track is then selected under the hood, the + // match can read its cue timing without ever putting English on screen. Then run the match + // in the background and upgrade to a synced addon sub if one is found; otherwise stay on AI. + val source = aiSourceSubtitle ?: findAiSourceSubtitle(_uiState.value.subtitles) + if (source != null) { + activateAiTranslation() + findBestSubtitleMatch(onNoMatch = { /* already on AI translation — the fallback */ }) + } else { + findBestSubtitleMatch(onNoMatch = { showMatchToast("No well-synced Hebrew subtitle found") }) + } + } + + /** Existing behavior: translate the built-in/source subtitle to the target language. */ + fun activateAiTranslation() { + val source = aiSourceSubtitle ?: findAiSourceSubtitle(_uiState.value.subtitles) ?: return + aiSourceSubtitle = source hasManualSubtitleSelection = true + // AI activation is not a specific user track pick — a late embedded preferred-language + // track may still displace it. + userPickedSubtitle = false val targetLangName = _uiState.value.aiTargetLanguageName if (targetLangName.isNotBlank()) translationManager.targetLanguage = targetLangName translationManager.isEnabled = true @@ -2312,6 +2454,312 @@ class PlayerViewModel @Inject constructor( ) } + private var subtitleBeforeLiveAudio: Subtitle? = null + + fun toggleLiveAudioTranslation() { + if (_uiState.value.isLiveAudioTranslating) { + geminiLiveService.disconnect() + // Restore whatever subtitle was active before live AI took over + _uiState.value = _uiState.value.copy( + isLiveAudioTranslating = false, + selectedSubtitle = subtitleBeforeLiveAudio, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + subtitleBeforeLiveAudio = null + } else { + // Clear current subtitle so ExoPlayer stops rendering it while live AI is active. + // Must mark this as a manual selection — otherwise a subtitle-fetch job that + // resolves after this point (scheduleSubtitleSelection) would silently re-select + // a subtitle on top of the live AI overlay, since it only checks this flag. + hasManualSubtitleSelection = true + subtitleSelectionJob?.cancel() + translationManager.isEnabled = false + subtitleBeforeLiveAudio = _uiState.value.selectedSubtitle + geminiLiveService.connect() + _uiState.value = _uiState.value.copy( + isLiveAudioTranslating = true, + selectedSubtitle = null, + isAiTranslating = false, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + } + } + + /** + * "Find best match": scan the available Hebrew subtitle tracks and auto-select the one that is + * best synced with the audio. Loads every candidate's cues, listens to the AI hearing for a + * short window, then scores each candidate by how often its on-screen cue matches what was + * spoken at that moment (see [SubtitleSyncMatcher]). + */ + fun findBestSubtitleMatch( + onNoMatch: () -> Unit = { showMatchToast("No well-synced Hebrew subtitle found") } + ) { + if (_uiState.value.isFindingBestMatch) return + val previousSubtitle = _uiState.value.selectedSubtitle + findMatchJob?.cancel() + findMatchJob = viewModelScope.launch { + beginMatch("Finding best subtitle…") + + // Embedded tracks resolve asynchronously (a beat after playback starts), so wait briefly + // for a usable muxed reference to appear — otherwise the match falls back to hearing just + // because the built-in track hadn't loaded yet. + val waitStart = System.currentTimeMillis() + while (_uiState.value.subtitles.none { it.isEmbedded && !it.isBitmap && !it.isForced } && + System.currentTimeMillis() - waitStart < MATCH_EMBEDDED_WAIT_MS + ) { + delay(200) + } + val subs = _uiState.value.subtitles + + // An embedded Hebrew track is muxed into the media → guaranteed 100% match, pick it now. + val embedded = subs.firstOrNull { + it.isEmbedded && !it.isBitmap && !it.isForced && + !it.label.contains("forced", ignoreCase = true) && + normalizeLanguage(it.lang) == "he" + } + if (embedded != null) { + endMatch() + selectSubtitle(embedded, isUserAction = false) + showMatchToast("Matched: embedded Hebrew subtitle (in sync)") + return@launch + } + + val candidates = subs.filter { + !it.isEmbedded && !it.isBitmap && it.url.isNotBlank() && normalizeLanguage(it.lang) == "he" + } + if (candidates.isEmpty()) { + endMatch() + onNoMatch() + return@launch + } + + // Prefer any embedded (muxed) track as the sync reference (English first, else any). + // SDH/CC is fine for *timing*; only forced/bitmap are unsuitable. + val embeddedRefs = subs.filter { + it.isEmbedded && !it.isBitmap && !it.isForced && + !it.label.contains("forced", ignoreCase = true) + } + val builtInReference = embeddedRefs.firstOrNull { normalizeLanguage(it.lang) == "en" } + ?: embeddedRefs.firstOrNull() + val sourceLabel = if (builtInReference != null) "Built-in" else "Hearing" + android.util.Log.i( + "SubMatch", + "reference source=$sourceLabel embeddedRefs=${embeddedRefs.size} " + + "ref=\"${builtInReference?.label ?: "-"}\" (lang=${builtInReference?.lang}) " + + "allEmbedded=${subs.count { it.isEmbedded }}" + ) + updateMatchStatus("Finding best subtitle ($sourceLabel)…") + + val loaded = candidates.map { sub -> + async { sub to SubtitleSyncMatcher.loadCues(sub.url) } + }.awaitAll().filter { it.second.isNotEmpty() } + + if (loaded.isEmpty()) { + endMatch() + restoreSubtitle(previousSubtitle) + onNoMatch() + return@launch + } + + val scored = when { + builtInReference != null -> scoreAgainstBuiltIn(loaded, builtInReference, sourceLabel) + // AI translation is already on screen as the fallback — the hearing path is too + // unreliable to risk replacing it, so just stay on AI. + _uiState.value.isAiTranslating -> null + else -> scoreAgainstHearing(loaded, sourceLabel) + } + endMatch() + + val best = scored?.maxByOrNull { it.second } + if (best != null && best.second >= MATCH_SUCCESS_THRESHOLD) { + selectSubtitle(best.first, isUserAction = false) + showMatchToast("Matched: ${best.first.label} · ${(best.second * 100).toInt()}% ($sourceLabel)") + } else { + // Nothing synced found (or too little dialogue) → let the caller fall back (AI translate). + restoreSubtitle(previousSubtitle) + onNoMatch() + } + } + } + + /** Reference = a built-in track's cue timing. Returns null if too little dialogue. */ + private suspend fun scoreAgainstBuiltIn( + loaded: List>>, + englishSub: Subtitle, + sourceLabel: String + ): List>? { + synchronized(referenceIntervals) { referenceIntervals.clear() } + refIntervalStart = -1L + // If AI translation is already showing (it has the source/English track selected under the + // hood), read that track's cues without switching the displayed subtitle — keeps Hebrew on + // screen. Otherwise select the reference track so ExoPlayer renders/buffers its cues. + val aiAlreadyShowingSource = _uiState.value.isAiTranslating && aiSourceSubtitle?.id == englishSub.id + if (!aiAlreadyShowingSource) { + hasManualSubtitleSelection = true + _uiState.value = _uiState.value.copy( + selectedSubtitle = englishSub, + isAiTranslating = false, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + } + + // Fast path: read the reference track's already-buffered (upcoming) cues so the match can + // complete before dialogue is even spoken. It's free, so use many intervals for accuracy. + val buffered = awaitBufferedReferenceIntervals(sourceLabel) + // Sanity-check: buffered timestamps must land within the candidate subtitles' time range, + // otherwise the stream-offset correction failed — discard and use the real-time path. + val candMin = loaded.minOf { it.second.firstOrNull()?.startMs ?: Long.MAX_VALUE } + val candMax = loaded.maxOf { it.second.lastOrNull()?.endMs ?: 0L } + val bufferedInRange = buffered.count { it.first in candMin..candMax } + val fromBuffer = buffered.size >= MATCH_MIN_REF_INTERVALS && + bufferedInRange >= (buffered.size + 1) / 2 + val refs = if (fromBuffer) { + buffered + } else { + // Fallback: collect cues as they render in real time (waits for playback to reach them). + collectingReference = true + val startedAt = System.currentTimeMillis() + while (currentRefCount() < MATCH_MAX_REF_INTERVALS && + System.currentTimeMillis() - startedAt < MATCH_HARD_CAP_MS + ) { + val n = currentRefCount() + updateMatchStatus( + if (n == 0) "Searching for a match ($sourceLabel) — waiting for speech…" + else "Searching for a match ($sourceLabel) — comparing subtitles… ($n)" + ) + delay(300) + } + collectingReference = false + synchronized(referenceIntervals) { referenceIntervals.toList() } + } + if (refs.size < MATCH_MIN_REF_INTERVALS) return null + + loaded.firstOrNull()?.let { (_, cues) -> + android.util.Log.i( + "SubMatch", + "align src=${if (fromBuffer) "buffer" else "realtime"} " + + "refs=${refs.take(3)} candCues=${cues.take(3).map { it.startMs to it.endMs }} " + + "candRange=${cues.firstOrNull()?.startMs}..${cues.lastOrNull()?.endMs}" + ) + } + + return loaded.map { (sub, cues) -> + val s = SubtitleSyncMatcher.scoreByTiming(cues, refs) + android.util.Log.i( + "SubMatch", + "[builtin] candidate label=\"${sub.label}\" provider=${sub.provider} cues=${cues.size} " + + "refIntervals=${refs.size} src=${if (fromBuffer) "buffer" else "realtime"} score=${"%.2f".format(s)}" + ) + sub to s + } + } + + /** Poll the buffered-cue provider briefly (giving the track a moment to buffer). */ + private suspend fun awaitBufferedReferenceIntervals(sourceLabel: String): List> { + val provider = bufferedReferenceIntervalsProvider ?: return emptyList() + updateMatchStatus("Searching for a match ($sourceLabel) — reading buffered subtitles…") + var best = emptyList>() + val startedAt = System.currentTimeMillis() + while (System.currentTimeMillis() - startedAt < MATCH_BUFFERED_WAIT_MS) { + val got = runCatching { provider(MATCH_BUFFERED_REF_INTERVALS) }.getOrDefault(emptyList()) + if (got.size > best.size) best = got + if (best.size >= MATCH_BUFFERED_REF_INTERVALS) break + delay(150) + } + return best + } + + private fun currentRefCount(): Int = synchronized(referenceIntervals) { referenceIntervals.size } + + /** Reference = AI hearing transcription (fallback when there's no built-in English track). */ + private suspend fun scoreAgainstHearing( + loaded: List>>, + sourceLabel: String + ): List>? { + startMatchListening() + val samples = mutableListOf() + val collector = viewModelScope.launch { + geminiLiveService.translatedText + .filterNotNull() + .distinctUntilChanged() + .collect { line -> + val t = line.trim() + if (t.length >= 2) samples.add(SubtitleSyncMatcher.SpokenSample(t, lastKnownPositionMs)) + } + } + val startedAt = System.currentTimeMillis() + while (samples.size < MATCH_MAX_SAMPLES && + System.currentTimeMillis() - startedAt < MATCH_HARD_CAP_MS + ) { + updateMatchStatus( + if (samples.isEmpty()) "Searching for a match ($sourceLabel) — waiting for speech…" + else "Searching for a match ($sourceLabel) — scanning subtitles… (${samples.size})" + ) + delay(300) + } + collector.cancel() + if (samples.size < MATCH_MIN_SAMPLES) return null + + return loaded.map { (sub, cues) -> + val s = SubtitleSyncMatcher.score( + cues, samples, MATCH_LATENCY_MS, MATCH_TOLERANCE_MS, MATCH_MIN_SIMILARITY + ) + android.util.Log.i( + "SubMatch", + "[hearing] candidate label=\"${sub.label}\" provider=${sub.provider} cues=${cues.size} " + + "samples=${samples.size} score=${"%.2f".format(s)}" + ) + sub to s + } + } + + private fun beginMatch(status: String) { + _uiState.value = _uiState.value.copy(isFindingBestMatch = true, matchStatusText = status) + } + + private fun updateMatchStatus(status: String) { + if (_uiState.value.matchStatusText != status) { + _uiState.value = _uiState.value.copy(matchStatusText = status) + } + } + + private fun startMatchListening() { + hasManualSubtitleSelection = true + subtitleSelectionJob?.cancel() + translationManager.isEnabled = false + geminiLiveService.connect() + _uiState.value = _uiState.value.copy( + isLiveAudioTranslating = true, + selectedSubtitle = null, + isAiTranslating = false, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + } + + private fun endMatch() { + geminiLiveService.disconnect() + _uiState.value = _uiState.value.copy( + isFindingBestMatch = false, + isLiveAudioTranslating = false, + matchStatusText = "" + ) + } + + private fun restoreSubtitle(subtitle: Subtitle?) { + _uiState.value = _uiState.value.copy( + selectedSubtitle = subtitle, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + } + + private fun showMatchToast(message: String) { + _uiState.value = _uiState.value.copy(matchToast = message) + } + + fun dismissMatchToast() { + _uiState.value = _uiState.value.copy(matchToast = null) + } + fun disableSubtitles() { hasManualSubtitleSelection = true subtitleSelectionJob?.cancel() @@ -3077,6 +3525,20 @@ class PlayerViewModel @Inject constructor( companion object { private const val TAG = "PlayerViewModel" + // ── "Find best match" tuning ────────────────────────────────────────── + private const val MATCH_HARD_CAP_MS = 90_000L // safety cap; otherwise waits for speech + private const val MATCH_MAX_SAMPLES = 12 // stop early once we have this many lines + private const val MATCH_MIN_REF_INTERVALS = 1 // built-in mode: min reference cues to decide + private const val MATCH_MAX_REF_INTERVALS = 6 // built-in realtime fallback: stop after this many + private const val MATCH_BUFFERED_REF_INTERVALS = 12 // buffered path is free → use many for accuracy + private const val MATCH_BUFFERED_WAIT_MS = 2_500L // how long to wait for the track to buffer cues + private const val MATCH_EMBEDDED_WAIT_MS = 8_000L // wait for async embedded tracks (background; AI shows meanwhile) + private const val MATCH_MIN_SAMPLES = 4 // need at least this many to decide + private const val MATCH_LATENCY_MS = 800L // AI hearing lag: audio time ≈ arrival − this + private const val MATCH_TOLERANCE_MS = 1_500L // cue search window around the audio time + private const val MATCH_MIN_SIMILARITY = 0.35 // word-overlap needed to count a cue as a hit + private const val MATCH_SUCCESS_THRESHOLD = 0.30 // hit ratio needed to accept a subtitle + /** Known debrid service CDN domains. Reachability checks are skipped for these. */ private val DEBRID_CDN_DOMAINS = setOf( // Real-Debrid diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt new file mode 100644 index 000000000..111bf00c3 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt @@ -0,0 +1,179 @@ +package com.arflix.tv.ui.screens.player + +import android.util.Log +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import okhttp3.OkHttpClient +import okhttp3.Request +import java.util.zip.GZIPInputStream +import java.util.concurrent.TimeUnit + +/** + * "Find best match": scores candidate subtitle tracks by how well their on-screen cue at a given + * moment matches what the AI hearing transcribed being spoken at that same moment. A well-synced + * subtitle shows the right words at the right time; an out-of-sync one does not. Comparison is + * language-specific (currently Hebrew), so it ranks Hebrew subtitle tracks by sync quality. + */ +object SubtitleSyncMatcher { + + private const val TAG = "SubMatch" + + data class TimedCue(val startMs: Long, val endMs: Long, val text: String) + + /** One collected AI-hearing sample: the spoken line and the media position when it arrived. */ + data class SpokenSample(val text: String, val positionMs: Long) + + private val client = OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(15, TimeUnit.SECONDS) + .build() + + suspend fun loadCues(url: String): List = withContext(Dispatchers.IO) { + runCatching { + val request = Request.Builder().url(url).build() + client.newCall(request).execute().use { response -> + if (!response.isSuccessful) return@use emptyList() + val body = response.body ?: return@use emptyList() + val raw = body.bytes() + val text = if (looksGzipped(url, raw)) { + GZIPInputStream(raw.inputStream()).bufferedReader(Charsets.UTF_8).readText() + } else { + String(raw, Charsets.UTF_8) + } + parseCues(text) + } + }.onFailure { error -> + Log.w(TAG, "loadCues failed url=$url err=${error.message}") + }.getOrDefault(emptyList()) + } + + /** + * Score a candidate's cues against the collected spoken samples. For each sample we look at + * the audio's media time (arrival position minus the AI latency), find cues within a tolerance + * window, and count a hit when the best word-overlap clears [minSimilarity]. Returns the hit + * ratio in 0..1. + */ + fun score( + cues: List, + samples: List, + latencyMs: Long, + toleranceMs: Long, + minSimilarity: Double + ): Double { + if (cues.isEmpty() || samples.isEmpty()) return 0.0 + var hits = 0 + for (sample in samples) { + val audioTime = sample.positionMs - latencyMs + val best = cues.asSequence() + .filter { it.endMs >= audioTime - toleranceMs && it.startMs <= audioTime + toleranceMs } + .maxOfOrNull { similarity(sample.text, it.text) } ?: 0.0 + if (best >= minSimilarity) hits++ + } + return hits.toDouble() / samples.size + } + + /** + * Score a candidate against a synced reference (a built-in subtitle's cue-visible intervals) + * purely by timing. For each reference interval we take the best temporal overlap fraction + * with any candidate cue and average them. A synced candidate has cues at the same times → + * high overlap; an offset candidate → low. Language-agnostic (no translation needed). + */ + fun scoreByTiming(cues: List, referenceIntervals: List>): Double { + if (cues.isEmpty() || referenceIntervals.isEmpty()) return 0.0 + var sum = 0.0 + for ((rs, re) in referenceIntervals) { + val refDur = (re - rs).coerceAtLeast(1L) + val best = cues.maxOfOrNull { c -> + val overlap = minOf(re, c.endMs) - maxOf(rs, c.startMs) + overlap.toDouble() / refDur + } ?: 0.0 + sum += best.coerceIn(0.0, 1.0) + } + return sum / referenceIntervals.size + } + + // ── Similarity ──────────────────────────────────────────────────────────── + + /** Overlap coefficient of normalized word sets — tolerant of different wording/length. */ + fun similarity(a: String, b: String): Double { + val setA = tokens(a) + val setB = tokens(b) + if (setA.isEmpty() || setB.isEmpty()) return 0.0 + val intersection = setA.count { it in setB } + return intersection.toDouble() / minOf(setA.size, setB.size) + } + + private fun tokens(text: String): Set = + normalize(text).split(' ').filter { it.length >= 2 }.toSet() + + private fun normalize(text: String): String { + val sb = StringBuilder(text.length) + for (ch in text) { + when { + // Strip Hebrew niqqud/cantillation and bidi control marks. + ch in '֑'..'ׇ' -> {} + ch == '‎' || ch == '‏' || ch == '‪' || ch == '‫' || + ch == '‬' || ch == '⁦' || ch == '⁧' || ch == '⁩' -> {} + ch.isLetterOrDigit() -> sb.append(ch.lowercaseChar()) + else -> sb.append(' ') + } + } + return sb.toString().replace(Regex("\\s+"), " ").trim() + } + + fun cueTextAt(cues: List, timeMs: Long): String? = + cues.firstOrNull { timeMs in it.startMs..it.endMs }?.text + + // ── Parsing (SRT + WEBVTT) ────────────────────────────────────────────────── + + private fun looksGzipped(url: String, bytes: ByteArray): Boolean = + url.substringBefore('?').endsWith(".gz", ignoreCase = true) || + (bytes.size > 2 && bytes[0] == 0x1f.toByte() && bytes[1] == 0x8b.toByte()) + + private val TIME_LINE = Regex( + """(\d{1,2}:\d{2}:\d{2}[.,]\d{1,3}|\d{1,2}:\d{2}[.,]\d{1,3})\s*-->\s*(\d{1,2}:\d{2}:\d{2}[.,]\d{1,3}|\d{1,2}:\d{2}[.,]\d{1,3})""" + ) + private val TAG_STRIP = Regex("<[^>]*>") + + fun parseCues(content: String): List { + val normalized = content.replace("\r\n", "\n").replace('\r', '\n') + val blocks = normalized.split(Regex("\n\\s*\n")) + val cues = ArrayList() + for (block in blocks) { + val lines = block.split('\n').map { it.trim() }.filter { it.isNotEmpty() } + if (lines.isEmpty()) continue + val timeLineIndex = lines.indexOfFirst { TIME_LINE.containsMatchIn(it) } + if (timeLineIndex < 0) continue + val match = TIME_LINE.find(lines[timeLineIndex]) ?: continue + val start = parseTimestamp(match.groupValues[1]) ?: continue + val end = parseTimestamp(match.groupValues[2]) ?: continue + val text = lines.drop(timeLineIndex + 1) + .joinToString(" ") + .replace(TAG_STRIP, "") + .trim() + if (text.isNotEmpty() && end > start) cues.add(TimedCue(start, end, text)) + } + return cues + } + + private fun parseTimestamp(value: String): Long? { + val cleaned = value.replace(',', '.') + val parts = cleaned.split(':') + return runCatching { + when (parts.size) { + 3 -> { + val h = parts[0].toLong() + val m = parts[1].toLong() + val s = parts[2].toDouble() + (h * 3600 + m * 60) * 1000 + (s * 1000).toLong() + } + 2 -> { + val m = parts[0].toLong() + val s = parts[1].toDouble() + (m * 60) * 1000 + (s * 1000).toLong() + } + else -> null + } + }.getOrNull() + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt index 6b31d6170..69ce49909 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt @@ -217,7 +217,7 @@ private fun tvGeneralRowsForSection(section: String): List { return when (section) { "language" -> listOf(0, 3) "subtitles" -> listOf(1, 2, 4, 5, 6, 7, 8, 9) - "ai_subtitles" -> listOf(28, 29, 30, 31, 32, 33) + "ai_subtitles" -> listOf(28, 29, 30, 38, 31, 32, 33) "playback" -> listOf(10, 11, 12, 13, 14, 37, 34, 16, 15, 27) "appearance" -> listOf(17, 18, 20, 21, 24, 23, 22, 36) "profiles" -> listOf(19) @@ -902,6 +902,7 @@ fun SettingsScreen( 28 -> viewModel.setSubtitleAiEnabled(!uiState.subtitleAiEnabled) 29 -> showAiModelDialog = true 30 -> viewModel.setSubtitleAiAutoSelect(!uiState.subtitleAiAutoSelect) + 38 -> viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) 31 -> viewModel.setSubtitleRemoveHearingImpaired(!uiState.subtitleRemoveHearingImpaired) 32 -> showAiApiKeyDialog = true 33 -> viewModel.startAiKeyServer() @@ -1329,12 +1330,14 @@ fun SettingsScreen( onQualityFiltersClick = { showQualityFiltersModal = true }, subtitleAiEnabled = uiState.subtitleAiEnabled, subtitleAiAutoSelect = uiState.subtitleAiAutoSelect, + subtitleAiFindBestMatch = uiState.subtitleAiFindBestMatch, subtitleAiApiKey = uiState.subtitleAiApiKey, subtitleAiModel = uiState.subtitleAiModel, subtitleRemoveHearingImpaired = uiState.subtitleRemoveHearingImpaired, onSubtitleAiEnabledToggle = { viewModel.setSubtitleAiEnabled(it) }, onSubtitleAiModelClick = { showAiModelDialog = true }, onSubtitleAiAutoSelectToggle = { viewModel.setSubtitleAiAutoSelect(it) }, + onSubtitleAiFindBestMatchToggle = { viewModel.setSubtitleAiFindBestMatch(it) }, onSubtitleRemoveHearingImpairedToggle = { viewModel.setSubtitleRemoveHearingImpaired(it) }, onSubtitleAiApiKeyClick = { showAiApiKeyDialog = true }, onSubtitleAiQrClick = { viewModel.startAiKeyServer() }, @@ -3727,6 +3730,14 @@ private fun MobileSettingsSubPage( isFocused = false, onClick = { viewModel.setSubtitleAiAutoSelect(!uiState.subtitleAiAutoSelect) } ) + MobileSettingsRow( + icon = Icons.Default.AutoAwesome, + title = stringResource(R.string.ai_find_best_match_title), + subtitle = stringResource(R.string.ai_find_best_match_desc), + value = if (uiState.subtitleAiFindBestMatch) "On" else "Off", + isFocused = false, + onClick = { viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) } + ) MobileSettingsRow( icon = Icons.Default.Subtitles, title = stringResource(R.string.ai_remove_hi_title), @@ -4736,12 +4747,14 @@ private fun TvGeneralSettingsRows( onQualityFiltersClick: () -> Unit = {}, subtitleAiEnabled: Boolean = false, subtitleAiAutoSelect: Boolean = false, + subtitleAiFindBestMatch: Boolean = true, subtitleAiApiKey: String = "", subtitleAiModel: com.arflix.tv.ui.screens.player.SubtitleAiModel = com.arflix.tv.ui.screens.player.SubtitleAiModel.GROQ_LLAMA_70B, subtitleRemoveHearingImpaired: Boolean = true, onSubtitleAiEnabledToggle: (Boolean) -> Unit = {}, onSubtitleAiModelClick: () -> Unit = {}, onSubtitleAiAutoSelectToggle: (Boolean) -> Unit = {}, + onSubtitleAiFindBestMatchToggle: (Boolean) -> Unit = {}, onSubtitleRemoveHearingImpairedToggle: (Boolean) -> Unit = {}, onSubtitleAiApiKeyClick: () -> Unit = {}, onSubtitleAiQrClick: () -> Unit = {}, @@ -4848,6 +4861,7 @@ private fun TvGeneralSettingsRows( modifier = Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f) ) 30 -> SettingsToggleRow(stringResource(R.string.ai_auto_select_title), stringResource(R.string.ai_auto_select_desc), subtitleAiAutoSelect, focusedIndex == localIndex, onSubtitleAiAutoSelectToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) + 38 -> SettingsToggleRow(stringResource(R.string.ai_find_best_match_title), stringResource(R.string.ai_find_best_match_desc), subtitleAiFindBestMatch, focusedIndex == localIndex, onSubtitleAiFindBestMatchToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) 31 -> SettingsToggleRow(stringResource(R.string.ai_remove_hi_title), stringResource(R.string.ai_remove_hi_desc), subtitleRemoveHearingImpaired, focusedIndex == localIndex, onSubtitleRemoveHearingImpairedToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) 32 -> SettingsRow(Icons.Default.VpnKey, stringResource(R.string.ai_api_key_title), stringResource(R.string.ai_api_key_desc), maskAiApiKey(subtitleAiApiKey, stringResource(R.string.ai_key_not_set)), focusedIndex == localIndex, onSubtitleAiApiKeyClick, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) 33 -> SettingsRow(Icons.Default.QrCode, stringResource(R.string.ai_scan_qr_title), stringResource(R.string.ai_scan_qr_desc), "", focusedIndex == localIndex, onSubtitleAiQrClick, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) @@ -4921,12 +4935,14 @@ private fun GeneralSettings( onQualityFiltersClick: () -> Unit = {}, subtitleAiEnabled: Boolean = false, subtitleAiAutoSelect: Boolean = false, + subtitleAiFindBestMatch: Boolean = true, subtitleAiApiKey: String = "", subtitleAiModel: com.arflix.tv.ui.screens.player.SubtitleAiModel = com.arflix.tv.ui.screens.player.SubtitleAiModel.GROQ_LLAMA_70B, subtitleRemoveHearingImpaired: Boolean = true, onSubtitleAiEnabledToggle: (Boolean) -> Unit = {}, onSubtitleAiModelClick: () -> Unit = {}, onSubtitleAiAutoSelectToggle: (Boolean) -> Unit = {}, + onSubtitleAiFindBestMatchToggle: (Boolean) -> Unit = {}, onSubtitleRemoveHearingImpairedToggle: (Boolean) -> Unit = {}, onSubtitleAiApiKeyClick: () -> Unit = {}, onSubtitleAiQrClick: () -> Unit = {}, diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt index 09b2b5c04..d19c7187f 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt @@ -197,6 +197,7 @@ data class SettingsUiState( // AI Subtitles val subtitleAiEnabled: Boolean = false, val subtitleAiAutoSelect: Boolean = false, + val subtitleAiFindBestMatch: Boolean = true, val subtitleAiApiKey: String = "", val subtitleAiModel: SubtitleAiModel = SubtitleAiModel.GROQ_LLAMA_70B, val subtitleRemoveHearingImpaired: Boolean = true, @@ -286,6 +287,7 @@ class SettingsViewModel @Inject constructor( // Global (non-profile-scoped) AI subtitle settings — device-wide, not per-profile private val subtitleAiEnabledKey = booleanPreferencesKey("subtitle_ai_enabled") private val subtitleAiAutoSelectKey = booleanPreferencesKey("subtitle_ai_auto_select") + private val subtitleAiFindBestMatchKey = booleanPreferencesKey("subtitle_ai_find_best_match") private val subtitleAiApiKeyKey = stringPreferencesKey("subtitle_ai_api_key") private val subtitleAiModelKey = stringPreferencesKey("subtitle_ai_model") private val subtitleRemoveHearingImpairedKey = booleanPreferencesKey("subtitle_remove_hearing_impaired") @@ -488,6 +490,7 @@ class SettingsViewModel @Inject constructor( val subtitleAiEnabled = prefs[subtitleAiEnabledKey] ?: false val subtitleAiAutoSelect = prefs[subtitleAiAutoSelectKey] ?: false + val subtitleAiFindBestMatch = prefs[subtitleAiFindBestMatchKey] ?: true val subtitleAiApiKey = prefs[subtitleAiApiKeyKey] ?: "" val subtitleAiModel = runCatching { SubtitleAiModel.valueOf(prefs[subtitleAiModelKey] ?: SubtitleAiModel.GROQ_LLAMA_70B.name) @@ -557,6 +560,7 @@ class SettingsViewModel @Inject constructor( qualityFilterPresetLabel = detectQualityFilterPreset(qualityFilters).label, subtitleAiEnabled = subtitleAiEnabled, subtitleAiAutoSelect = subtitleAiAutoSelect, + subtitleAiFindBestMatch = subtitleAiFindBestMatch, subtitleAiApiKey = subtitleAiApiKey, subtitleAiModel = subtitleAiModel, subtitleRemoveHearingImpaired = subtitleRemoveHearingImpaired, @@ -1295,6 +1299,14 @@ class SettingsViewModel @Inject constructor( } } + fun setSubtitleAiFindBestMatch(enabled: Boolean) { + viewModelScope.launch { + context.settingsDataStore.edit { it[subtitleAiFindBestMatchKey] = enabled } + _uiState.value = _uiState.value.copy(subtitleAiFindBestMatch = enabled) + syncLocalStateToCloud(silent = true) + } + } + fun setSubtitleRemoveHearingImpaired(enabled: Boolean) { viewModelScope.launch { context.settingsDataStore.edit { it[subtitleRemoveHearingImpairedKey] = enabled } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index eed488ca4..4b032b4e9 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -235,6 +235,8 @@ Select translation provider and model Auto-Select AI Translation Automatically activate AI when any built in language is available + Use AI to Find Best Addon Match First + auto-pick a synced subtitle from your addons using AI Remove Hearing Impaired Text Strip [bracketed] hearing-impaired markers before translation API Key From 50becd143ca6bc0c20c4bbe9be775cf2dfb94aae Mon Sep 17 00:00:00 2001 From: silentbil Date: Sat, 4 Jul 2026 22:25:20 +0300 Subject: [PATCH 02/11] fixes --- .../tv/data/repository/CloudSyncRepository.kt | 3 + .../player/GeminiLiveTranslationService.kt | 5 +- .../tv/ui/screens/player/PlayerViewModel.kt | 479 ++++++++++++++---- .../ui/screens/player/SubtitleSyncMatcher.kt | 33 +- 4 files changed, 413 insertions(+), 107 deletions(-) diff --git a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt index e9af9d229..c73777c59 100644 --- a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt +++ b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt @@ -360,6 +360,7 @@ class CloudSyncRepository @Inject constructor( // Global AI subtitle keys (non-profile-scoped, device-wide) private val subtitleAiEnabledKey = androidx.datastore.preferences.core.booleanPreferencesKey("subtitle_ai_enabled") private val subtitleAiAutoSelectKey = androidx.datastore.preferences.core.booleanPreferencesKey("subtitle_ai_auto_select") + private val subtitleAiFindBestMatchKey = androidx.datastore.preferences.core.booleanPreferencesKey("subtitle_ai_find_best_match") private val subtitleAiApiKeyKey = androidx.datastore.preferences.core.stringPreferencesKey("subtitle_ai_api_key") private val subtitleAiModelKey = androidx.datastore.preferences.core.stringPreferencesKey("subtitle_ai_model") private val subtitleRemoveHearingImpairedKey = androidx.datastore.preferences.core.booleanPreferencesKey("subtitle_remove_hearing_impaired") @@ -497,6 +498,7 @@ class CloudSyncRepository @Inject constructor( // Global AI subtitle settings (non-profile-scoped) root.put("subtitleAiEnabled", prefs[subtitleAiEnabledKey] ?: false) root.put("subtitleAiAutoSelect", prefs[subtitleAiAutoSelectKey] ?: false) + root.put("subtitleAiFindBestMatch", prefs[subtitleAiFindBestMatchKey] ?: true) root.put("subtitleAiApiKey", prefs[subtitleAiApiKeyKey] ?: "") root.put("subtitleAiModel", prefs[subtitleAiModelKey] ?: "GROQ_LLAMA_70B") root.put("subtitleRemoveHearingImpaired", prefs[subtitleRemoveHearingImpairedKey] ?: true) @@ -1235,6 +1237,7 @@ class CloudSyncRepository @Inject constructor( context.settingsDataStore.edit { prefs -> prefs[subtitleAiEnabledKey] = root.optBoolean("subtitleAiEnabled", false) prefs[subtitleAiAutoSelectKey] = root.optBoolean("subtitleAiAutoSelect", false) + prefs[subtitleAiFindBestMatchKey] = root.optBoolean("subtitleAiFindBestMatch", true) val apiKey = root.optString("subtitleAiApiKey", "") if (apiKey.isNotBlank()) prefs[subtitleAiApiKeyKey] = apiKey val model = root.optString("subtitleAiModel", "GROQ_LLAMA_70B") diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/GeminiLiveTranslationService.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/GeminiLiveTranslationService.kt index cf45b94d8..acd29ac87 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/GeminiLiveTranslationService.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/GeminiLiveTranslationService.kt @@ -44,6 +44,9 @@ class GeminiLiveTranslationService( private val apiKeyProvider: () -> String, private val scope: CoroutineScope ) { + /** BCP-47 code the live translation outputs (the user's preferred subtitle language). */ + @Volatile var targetLanguageCode: String = "he" + private val _text = MutableStateFlow(null) val translatedText: StateFlow = _text.asStateFlow() @@ -124,7 +127,7 @@ class GeminiLiveTranslationService( // The translate model's only real control: target language. It ignores // system instructions and has no quality/style/gender knobs, so we don't send any. put("translation_config", JSONObject().apply { - put("target_language_code", "he") + put("target_language_code", targetLanguageCode) }) }) }) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt index 4330c2a23..002dcfb09 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt @@ -204,9 +204,13 @@ class PlayerViewModel @Inject constructor( // AI subtitle settings (read once per video load) private var aiSubtitleEnabled = false private var aiSubtitleAutoSelect = false - // When true (default), activating AI first tries to auto-pick a synced Hebrew addon subtitle - // (Find Best Match); only if nothing matches does it translate the built-in subtitle. + // When true (default), activating AI first tries to auto-pick a synced addon subtitle in the + // user's preferred language (Find Best Match); only if nothing matches does it translate the + // built-in subtitle. private var aiFindBestMatchFirst = true + // Normalized code of the user's preferred subtitle language (e.g. "he") — the target of both + // "find best match" and AI translation. Blank when unset/disabled. + @Volatile private var targetSubtitleLangCode = "" private var aiApiKey = "" private var aiModel = SubtitleAiModel.GROQ_LLAMA_70B private var aiRemoveHearingImpaired = true @@ -218,6 +222,9 @@ class PlayerViewModel @Inject constructor( private val aiApiKeyKey = globalStringPreferencesKey("subtitle_ai_api_key") private val aiModelKey = globalStringPreferencesKey("subtitle_ai_model") private val aiRemoveHearingImpairedKey = booleanPreferencesKey("subtitle_remove_hearing_impaired") + // Remembered "find best match" results, keyed by stream identity (sync is a property of the + // exact video file, not the title — a different rip needs a fresh scan). + private val subtitleMatchCacheKey = globalStringPreferencesKey("subtitle_match_cache_v1") private val _isTranslatingLive = MutableStateFlow(false) val isTranslatingLive: kotlinx.coroutines.flow.StateFlow = _isTranslatingLive.asStateFlow() @@ -278,7 +285,14 @@ class PlayerViewModel @Inject constructor( if (hasText) { if (refIntervalStart < 0) refIntervalStart = timeMs } else if (refIntervalStart in 0 until timeMs) { - synchronized(referenceIntervals) { referenceIntervals.add(refIntervalStart to timeMs) } + // An interval spanning a forward seek would record a bogus minutes-long "cue"; + // real cues never stay on screen this long — drop it. + if (timeMs - refIntervalStart <= MATCH_MAX_REF_INTERVAL_MS) { + synchronized(referenceIntervals) { referenceIntervals.add(refIntervalStart to timeMs) } + } + refIntervalStart = -1L + } else if (refIntervalStart >= 0) { + // Position jumped backwards past the interval start (seek) — discard the open interval. refIntervalStart = -1L } } @@ -450,6 +464,7 @@ class PlayerViewModel @Inject constructor( .let { if (isSubtitleDisabledPreference(it)) "" else it } val secondarySub = prefs[secondarySubtitleKey()]?.trim().orEmpty() .let { if (isSubtitleDisabledPreference(it)) "" else it } + targetSubtitleLangCode = normalizeLanguage(preferredSub) // Load AI subtitle settings aiSubtitleEnabled = prefs[aiEnabledKey] ?: false @@ -1209,6 +1224,9 @@ class PlayerViewModel @Inject constructor( private fun applyPreferredSubtitle(preference: String, subtitles: List, fallbackLanguage: String?) { val normalizedPref = normalizeLanguage(preference) + if (normalizedPref.isNotBlank() && !isSubtitleDisabledPreference(preference)) { + targetSubtitleLangCode = normalizedPref + } // Highest priority: an embedded (muxed) track in the preferred language is inherently in // sync, so it beats any auto logic — it overrides an auto-selected/auto-matched subtitle and @@ -1403,11 +1421,13 @@ class PlayerViewModel @Inject constructor( ?: embedded.firstOrNull { it.isUsableSource() } } - // Prefer English embedded > any embedded with lang > any embedded > any non-forced subtitle + // Prefer English embedded > any embedded with lang > any embedded > external English. + // External subs in other languages are never a sane source: their timing is unverified + // and a target-language addon sub as "source" means translating a language to itself. return subtitles.filter { normalizeLanguage(it.lang) == "en" }.bestEmbedded() ?: subtitles.filter { it.lang.isNotBlank() }.bestEmbedded() ?: subtitles.bestEmbedded() - ?: subtitles.firstOrNull { it.isUsableSource() } + ?: subtitles.firstOrNull { it.isUsableSource() && normalizeLanguage(it.lang) == "en" } } private fun languageCodeToName(code: String): String { @@ -2139,12 +2159,29 @@ class PlayerViewModel @Inject constructor( } } + // A different source is a different file: the previous stream's track selection, AI + // translation state, any running match scan, and — crucially — its embedded subtitle + // entries are all meaningless for it. Drop them so the auto-selection flow (find best + // match / AI) runs fresh: a stale embedded entry would otherwise be re-selected against + // the stale list and map onto an arbitrary track of the new file, while the fresh + // embedded tracks arriving later would be ignored because the selection "already + // matches" (no nonce bump → the override is never applied to the new media item). + cancelFindBestMatch() + hasManualSubtitleSelection = false + userPickedSubtitle = false + aiSourceSubtitle = null + translationManager.isEnabled = false + // Direct URL - use immediately (ExoPlayer handles redirects) _uiState.value = _uiState.value.copy( selectedStream = resolvedStream, selectedStreamUrl = url, savedPosition = requestedResumePosition ?: _uiState.value.savedPosition, streamSelectionNonce = _uiState.value.streamSelectionNonce + 1, + subtitles = _uiState.value.subtitles.filterNot { it.isEmbedded }, + selectedSubtitle = null, + isAiTranslating = false, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1, isLoading = false, isLoadingStreams = false, streamProgress = null, @@ -2376,7 +2413,10 @@ class PlayerViewModel @Inject constructor( fun selectSubtitle(subtitle: Subtitle, isUserAction: Boolean = true) { hasManualSubtitleSelection = true userPickedSubtitle = isUserAction - if (isUserAction) cancelFindBestMatch() + if (isUserAction) { + cancelFindBestMatch() + updateMatchCacheForManualPick(subtitle) + } subtitleSelectionJob?.cancel() translationManager.isEnabled = false // Keep isAiAvailable/aiTargetLanguageName so the AI entry stays in the menu for re-selection @@ -2404,12 +2444,12 @@ class PlayerViewModel @Inject constructor( /** * The AI subtitle entry point (menu option + auto-select). When "find best match first" is on, - * it first tries to auto-pick a synced Hebrew addon subtitle; only if nothing matches does it - * fall back to AI-translating the built-in subtitle. + * it first tries to auto-pick a synced addon subtitle in the user's preferred language; only + * if nothing matches does it fall back to AI-translating the built-in subtitle. */ fun activateAiSubtitle() { if (aiFindBestMatchFirst) { - runFindBestMatch() + runFindBestMatch(isUserAction = false) } else { activateAiTranslation() } @@ -2419,23 +2459,45 @@ class PlayerViewModel @Inject constructor( * The "Find Best Match (AI)" menu button: always run the match logic; if nothing synced is * found, fall back to the AI translation option when one is available. */ - fun runFindBestMatch() { + fun runFindBestMatch(isUserAction: Boolean = true) { + // Defense-in-depth: the menu entries are already hidden when the AI feature is off, but + // nothing below should ever run without it. + if (!aiSubtitleEnabled) return + // A user-triggered rescan means the current (possibly remembered) pick is unwanted — + // forget the cached entry and scan fresh; the auto-run keeps using the cache. + if (isUserAction) writeCachedMatch(null) // Show AI translation immediately (if a source exists) so the user sees subtitles right // away — and because the AI source (English) track is then selected under the hood, the // match can read its cue timing without ever putting English on screen. Then run the match // in the background and upgrade to a synced addon sub if one is found; otherwise stay on AI. - val source = aiSourceSubtitle ?: findAiSourceSubtitle(_uiState.value.subtitles) + val source = findAiSourceSubtitle(_uiState.value.subtitles) ?: aiSourceSubtitle if (source != null) { activateAiTranslation() - findBestSubtitleMatch(onNoMatch = { /* already on AI translation — the fallback */ }) + findBestSubtitleMatch( + onNoMatch = { + // Staying on the AI fallback — re-activate to upgrade the translation source: + // an embedded track may have resolved while the scan ran, and the early + // activation could still be translating an external fallback source. Say so — + // otherwise the searching indicator just vanishes with no visible outcome. + if (_uiState.value.isAiTranslating) { + activateAiTranslation() + showMatchToast("No well-synced subtitle found — keeping AI translation") + } + }, + useCache = !isUserAction + ) } else { - findBestSubtitleMatch(onNoMatch = { showMatchToast("No well-synced Hebrew subtitle found") }) + // default onNoMatch: "no well-synced subtitle" toast + findBestSubtitleMatch(useCache = !isUserAction) } } /** Existing behavior: translate the built-in/source subtitle to the target language. */ fun activateAiTranslation() { - val source = aiSourceSubtitle ?: findAiSourceSubtitle(_uiState.value.subtitles) ?: return + // Re-resolve the source on every activation: an early activation (before embedded tracks + // resolve) may have cached an external fallback source, and translation inherits the + // source's timing — keep the cached one only when nothing better is available now. + val source = findAiSourceSubtitle(_uiState.value.subtitles) ?: aiSourceSubtitle ?: return aiSourceSubtitle = source hasManualSubtitleSelection = true // AI activation is not a specific user track pick — a late embedded preferred-language @@ -2475,6 +2537,7 @@ class PlayerViewModel @Inject constructor( subtitleSelectionJob?.cancel() translationManager.isEnabled = false subtitleBeforeLiveAudio = _uiState.value.selectedSubtitle + targetSubtitleLangCode.takeIf { it.isNotBlank() }?.let { geminiLiveService.targetLanguageCode = it } geminiLiveService.connect() _uiState.value = _uiState.value.copy( isLiveAudioTranslating = true, @@ -2486,53 +2549,117 @@ class PlayerViewModel @Inject constructor( } /** - * "Find best match": scan the available Hebrew subtitle tracks and auto-select the one that is - * best synced with the audio. Loads every candidate's cues, listens to the AI hearing for a - * short window, then scores each candidate by how often its on-screen cue matches what was - * spoken at that moment (see [SubtitleSyncMatcher]). + * "Find best match": scan the addon subtitle tracks in the user's preferred language and + * auto-select the one that is best synced with the audio. Loads every candidate's cues, + * listens to the AI hearing for a short window, then scores each candidate by how often its + * on-screen cue matches what was spoken at that moment (see [SubtitleSyncMatcher]). */ - fun findBestSubtitleMatch( - onNoMatch: () -> Unit = { showMatchToast("No well-synced Hebrew subtitle found") } - ) { + fun findBestSubtitleMatch(onNoMatch: (() -> Unit)? = null, useCache: Boolean = true) { if (_uiState.value.isFindingBestMatch) return val previousSubtitle = _uiState.value.selectedSubtitle findMatchJob?.cancel() findMatchJob = viewModelScope.launch { + val targetLang = targetSubtitleLangCode.ifBlank { normalizeLanguage(getDefaultSubtitle()) } + val targetLangName = languageCodeToName(targetLang) + val noMatch = onNoMatch + ?: { showMatchToast("No well-synced $targetLangName subtitle found") } + if (targetLang.isBlank() || isSubtitleDisabledPreference(targetLang)) { + noMatch() + return@launch + } beginMatch("Finding best subtitle…") - // Embedded tracks resolve asynchronously (a beat after playback starts), so wait briefly - // for a usable muxed reference to appear — otherwise the match falls back to hearing just - // because the built-in track hadn't loaded yet. + // Subtitle sources resolve asynchronously after playback starts: embedded tracks a beat + // later, addon subtitles when their fetch completes. Wait for both a usable muxed + // reference and target-language addon candidates — otherwise the scan can fire before + // the addon list exists and silently find nothing. (AI translation is already showing + // meanwhile, so waiting here costs the user nothing.) val waitStart = System.currentTimeMillis() - while (_uiState.value.subtitles.none { it.isEmbedded && !it.isBitmap && !it.isForced } && - System.currentTimeMillis() - waitStart < MATCH_EMBEDDED_WAIT_MS - ) { + var playingSinceMs = -1L + while (true) { + val nowMs = System.currentTimeMillis() + val elapsed = nowMs - waitStart + // Grace periods count *continuous playback* time, not wall time: embedded tracks + // cannot resolve before the stream actually plays, so a scan that gives up while + // the player is still buffering (e.g. right after a source switch) is meaningless. + if (lastIsPlaying) { + if (playingSinceMs < 0) playingSinceMs = nowMs + } else { + playingSinceMs = -1L + } + val playingFor = if (playingSinceMs < 0) -1L else nowMs - playingSinceMs + val current = _uiState.value.subtitles + val hasEmbedded = current.any { it.isEmbedded && !it.isBitmap && !it.isForced } + val hasCandidates = current.any { + !it.isEmbedded && !it.isBitmap && it.url.isNotBlank() && + normalizeLanguage(it.lang) == targetLang + } + if (hasEmbedded && hasCandidates) break + if (playingFor >= MATCH_SOURCES_WAIT_MS) break + // Past the embedded grace period and the addon fetch is done — whatever is missing + // now isn't coming; proceed with what we have. + if (playingFor >= MATCH_EMBEDDED_WAIT_MS && !_uiState.value.isLoadingSubtitles) break + // Stream never started (stall/error) — stop waiting eventually. + if (elapsed >= MATCH_PLAYBACK_WAIT_CAP_MS) break delay(200) } val subs = _uiState.value.subtitles - // An embedded Hebrew track is muxed into the media → guaranteed 100% match, pick it now. + // An embedded target-language track is muxed into the media → guaranteed in sync. val embedded = subs.firstOrNull { it.isEmbedded && !it.isBitmap && !it.isForced && !it.label.contains("forced", ignoreCase = true) && - normalizeLanguage(it.lang) == "he" + normalizeLanguage(it.lang) == targetLang } if (embedded != null) { endMatch() selectSubtitle(embedded, isUserAction = false) - showMatchToast("Matched: embedded Hebrew subtitle (in sync)") + showMatchToast("Matched: embedded $targetLangName subtitle (in sync)") return@launch } + // Cap the number of candidates we download/score — addons can return dozens. Rank by + // the release-name heuristic first so the subs most likely cut for this rip survive. + val streamSrc = _uiState.value.selectedStream?.source.orEmpty() val candidates = subs.filter { - !it.isEmbedded && !it.isBitmap && it.url.isNotBlank() && normalizeLanguage(it.lang) == "he" + !it.isEmbedded && !it.isBitmap && it.url.isNotBlank() && + normalizeLanguage(it.lang) == targetLang } + .sortedByDescending { weightedSubtitleScore(streamSrc, it.id) } + .take(MATCH_MAX_CANDIDATES) if (candidates.isEmpty()) { endMatch() - onNoMatch() + noMatch() return@launch } + // Last resort when the scan fails outright and nothing else is showing (no AI + // fallback active, no previous selection to restore): behave like the classic non-AI + // flow and pick the best release-name-scored candidate. Sync is unverified, so it is + // deliberately not remembered in the match cache. + fun selectLastResort() { + if (_uiState.value.isAiTranslating || _uiState.value.selectedSubtitle != null) return + candidates.firstOrNull()?.let { + selectSubtitle(it, isUserAction = false) + showMatchToast("Selected ${it.label} (sync unverified)") + } + } + + // A previously matched subtitle for this exact stream (same file → same sync) that is + // still offered by the addons wins immediately — no need to rescan. Skipped for + // user-triggered rescans, which mean the remembered pick is unwanted. + if (useCache) { + val remembered = readCachedMatch()?.let { cached -> + candidates.firstOrNull { it.id == cached.id && it.provider == cached.provider } + } + if (remembered != null) { + endMatch() + selectSubtitle(remembered, isUserAction = false) + showMatchToast("Matched: ${remembered.label} (remembered)") + return@launch + } + } + // Prefer any embedded (muxed) track as the sync reference (English first, else any). // SDH/CC is fine for *timing*; only forced/bitmap are unsuitable. val embeddedRefs = subs.filter { @@ -2557,87 +2684,163 @@ class PlayerViewModel @Inject constructor( if (loaded.isEmpty()) { endMatch() restoreSubtitle(previousSubtitle) - onNoMatch() + noMatch() + selectLastResort() return@launch } val scored = when { - builtInReference != null -> scoreAgainstBuiltIn(loaded, builtInReference, sourceLabel) + builtInReference != null -> + scoreAgainstBuiltIn(loaded, builtInReference, sourceLabel, previousSubtitle) // AI translation is already on screen as the fallback — the hearing path is too // unreliable to risk replacing it, so just stay on AI. _uiState.value.isAiTranslating -> null + // The hearing reference streams audio to the Gemini Live API — a Groq key can't + // open that connection, so don't try (it would just hang until the hard cap). + aiModel != SubtitleAiModel.GEMINI_FLASH_25 -> null else -> scoreAgainstHearing(loaded, sourceLabel) } endMatch() + val successThreshold = if (builtInReference != null) { + MATCH_SUCCESS_THRESHOLD_TIMING + } else { + MATCH_SUCCESS_THRESHOLD_HEARING + } val best = scored?.maxByOrNull { it.second } - if (best != null && best.second >= MATCH_SUCCESS_THRESHOLD) { + if (best != null && best.second >= successThreshold) { selectSubtitle(best.first, isUserAction = false) + writeCachedMatch(best.first) showMatchToast("Matched: ${best.first.label} · ${(best.second * 100).toInt()}% ($sourceLabel)") } else { // Nothing synced found (or too little dialogue) → let the caller fall back (AI translate). restoreSubtitle(previousSubtitle) - onNoMatch() + noMatch() + selectLastResort() } } } - /** Reference = a built-in track's cue timing. Returns null if too little dialogue. */ + /** Reference = a built-in track's cue timing (any language). Returns null if too little dialogue. */ private suspend fun scoreAgainstBuiltIn( loaded: List>>, - englishSub: Subtitle, - sourceLabel: String + referenceSub: Subtitle, + sourceLabel: String, + previousSubtitle: Subtitle? ): List>? { synchronized(referenceIntervals) { referenceIntervals.clear() } refIntervalStart = -1L // If AI translation is already showing (it has the source/English track selected under the // hood), read that track's cues without switching the displayed subtitle — keeps Hebrew on // screen. Otherwise select the reference track so ExoPlayer renders/buffers its cues. - val aiAlreadyShowingSource = _uiState.value.isAiTranslating && aiSourceSubtitle?.id == englishSub.id + val aiAlreadyShowingSource = _uiState.value.isAiTranslating && aiSourceSubtitle?.id == referenceSub.id if (!aiAlreadyShowingSource) { hasManualSubtitleSelection = true _uiState.value = _uiState.value.copy( - selectedSubtitle = englishSub, + selectedSubtitle = referenceSub, isAiTranslating = false, subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 ) } - // Fast path: read the reference track's already-buffered (upcoming) cues so the match can - // complete before dialogue is even spoken. It's free, so use many intervals for accuracy. - val buffered = awaitBufferedReferenceIntervals(sourceLabel) - // Sanity-check: buffered timestamps must land within the candidate subtitles' time range, - // otherwise the stream-offset correction failed — discard and use the real-time path. + // The reference-track selection propagates asynchronously (compose → track selector → + // decoder flush), so reading the cue buffer too early returns the *previous* track's + // cues — a previously-displayed candidate would then score a perfect match against + // itself. Give the switch a moment to settle before trusting the buffer. + delay(MATCH_TRACK_SWITCH_SETTLE_MS) + + // Unified reference collection: every tick, read the track's already-buffered (upcoming) + // cues — they land in ExoPlayer's buffer many seconds before they're spoken, so the match + // can complete before the dialogue even happens — and merge them with cues observed + // rendering in real time. Stops as soon as enough evidence accumulates. val candMin = loaded.minOf { it.second.firstOrNull()?.startMs ?: Long.MAX_VALUE } val candMax = loaded.maxOf { it.second.lastOrNull()?.endMs ?: 0L } - val bufferedInRange = buffered.count { it.first in candMin..candMax } - val fromBuffer = buffered.size >= MATCH_MIN_REF_INTERVALS && - bufferedInRange >= (buffered.size + 1) / 2 - val refs = if (fromBuffer) { - buffered - } else { - // Fallback: collect cues as they render in real time (waits for playback to reach them). - collectingReference = true - val startedAt = System.currentTimeMillis() - while (currentRefCount() < MATCH_MAX_REF_INTERVALS && - System.currentTimeMillis() - startedAt < MATCH_HARD_CAP_MS - ) { - val n = currentRefCount() - updateMatchStatus( - if (n == 0) "Searching for a match ($sourceLabel) — waiting for speech…" - else "Searching for a match ($sourceLabel) — comparing subtitles… ($n)" - ) - delay(300) + // Cues of the candidate that was on screen when the scan started — used to detect stale + // buffer reads that still mirror that track instead of the reference. + val previousCues = previousSubtitle?.let { prev -> + loaded.firstOrNull { it.first.id == prev.id }?.second + } + collectingReference = true + var refs: List> = emptyList() + var bufferedCount = 0 + var lastRefCount = 0 + var lastProgressElapsed = 0L + val startedAt = System.currentTimeMillis() + while (true) { + // Sanity-filter buffered timestamps: they must land within the candidate subtitles' + // time range, otherwise the stream-offset correction failed — ignore those. + val bufferedRaw = runCatching { + bufferedReferenceIntervalsProvider?.invoke(MATCH_BUFFERED_REF_INTERVALS) + }.getOrNull().orEmpty().filter { it.first in candMin..candMax } + // Self-match guard: if most buffered intervals coincide (to the millisecond range) + // with the previously-displayed candidate's own cues, the buffer still holds that + // track, not the reference — discard the read and rely on realtime this tick. + val selfMatches = if (previousCues == null) 0 else bufferedRaw.count { b -> + previousCues.any { c -> + kotlin.math.abs(c.startMs - b.first) <= 60 && kotlin.math.abs(c.endMs - b.second) <= 60 + } + } + val buffered = if (bufferedRaw.isNotEmpty() && selfMatches * 2 >= bufferedRaw.size) { + emptyList() + } else { + bufferedRaw + } + val realtime = synchronized(referenceIntervals) { referenceIntervals.toList() } + // Realtime intervals that overlap a buffered one describe the same cue — drop them. + val merged = buffered.toMutableList() + for (r in realtime) { + if (buffered.none { b -> r.first < b.second && b.first < r.second }) merged.add(r) + } + refs = merged.sortedBy { it.first } + bufferedCount = buffered.size + + val elapsed = System.currentTimeMillis() - startedAt + if (refs.size > lastRefCount) { + lastRefCount = refs.size + lastProgressElapsed = elapsed + } + // Interval count alone is not evidence: cues clustered in one moment (a recap, a + // titles sequence) can agree with an overall-drifting sub. Only accept once the refs + // also cover a minimum stretch of the video. + val spanOk = refs.size >= 2 && + refs.last().second - refs.first().first >= MATCH_MIN_REF_SPAN_MS + if (spanOk && refs.size >= MATCH_TARGET_REF_INTERVALS) break + // Early accept: scoring is free, so score incrementally — once some candidate is + // already clearly in sync there's no need to keep collecting up to the target. + if (spanOk && refs.size >= MATCH_EARLY_ACCEPT_REFS && + loaded.maxOf { (_, cues) -> SubtitleSyncMatcher.scoreByTiming(cues, refs) } >= MATCH_EARLY_ACCEPT_SCORE + ) break + // Give-up timers: a long one while waiting for speech to exist at all (openings can + // run minutes without dialogue), and a progress-anchored one afterwards — a silent + // scene mid-scan pauses the clock instead of draining it. Absolute ceiling on top. + val deadline = if (lastRefCount == 0) { + MATCH_NO_SPEECH_CAP_MS + } else { + minOf(lastProgressElapsed + MATCH_HARD_CAP_MS, MATCH_TOTAL_CAP_MS) } - collectingReference = false - synchronized(referenceIntervals) { referenceIntervals.toList() } + if (elapsed >= deadline) break + updateMatchStatus( + if (refs.isEmpty()) "Searching for a match ($sourceLabel) — waiting for speech…" + else "Searching for a match ($sourceLabel) — comparing subtitles… (${refs.size})" + ) + delay(300) + } + collectingReference = false + // Inconclusive after the hard cap (too few refs, or all clustered in one moment) — a pick + // would be a guess; return null so the caller stays on the AI-translation fallback. + if (refs.size < MATCH_MIN_REF_INTERVALS || + refs.last().second - refs.first().first < MATCH_MIN_REF_SPAN_MS + ) return null + val srcLabel = when { + bufferedCount == refs.size -> "buffer" + bufferedCount == 0 -> "realtime" + else -> "buffer+realtime" } - if (refs.size < MATCH_MIN_REF_INTERVALS) return null loaded.firstOrNull()?.let { (_, cues) -> android.util.Log.i( "SubMatch", - "align src=${if (fromBuffer) "buffer" else "realtime"} " + + "align src=$srcLabel buffered=$bufferedCount total=${refs.size} " + "refs=${refs.take(3)} candCues=${cues.take(3).map { it.startMs to it.endMs }} " + "candRange=${cues.firstOrNull()?.startMs}..${cues.lastOrNull()?.endMs}" ) @@ -2648,29 +2851,12 @@ class PlayerViewModel @Inject constructor( android.util.Log.i( "SubMatch", "[builtin] candidate label=\"${sub.label}\" provider=${sub.provider} cues=${cues.size} " + - "refIntervals=${refs.size} src=${if (fromBuffer) "buffer" else "realtime"} score=${"%.2f".format(s)}" + "refIntervals=${refs.size} src=$srcLabel score=${"%.2f".format(s)}" ) sub to s } } - /** Poll the buffered-cue provider briefly (giving the track a moment to buffer). */ - private suspend fun awaitBufferedReferenceIntervals(sourceLabel: String): List> { - val provider = bufferedReferenceIntervalsProvider ?: return emptyList() - updateMatchStatus("Searching for a match ($sourceLabel) — reading buffered subtitles…") - var best = emptyList>() - val startedAt = System.currentTimeMillis() - while (System.currentTimeMillis() - startedAt < MATCH_BUFFERED_WAIT_MS) { - val got = runCatching { provider(MATCH_BUFFERED_REF_INTERVALS) }.getOrDefault(emptyList()) - if (got.size > best.size) best = got - if (best.size >= MATCH_BUFFERED_REF_INTERVALS) break - delay(150) - } - return best - } - - private fun currentRefCount(): Int = synchronized(referenceIntervals) { referenceIntervals.size } - /** Reference = AI hearing transcription (fallback when there's no built-in English track). */ private suspend fun scoreAgainstHearing( loaded: List>>, @@ -2688,9 +2874,28 @@ class PlayerViewModel @Inject constructor( } } val startedAt = System.currentTimeMillis() - while (samples.size < MATCH_MAX_SAMPLES && - System.currentTimeMillis() - startedAt < MATCH_HARD_CAP_MS - ) { + var lastSampleCount = 0 + var lastProgressElapsed = 0L + while (samples.size < MATCH_MAX_SAMPLES) { + val elapsed = System.currentTimeMillis() - startedAt + if (samples.size > lastSampleCount) { + lastSampleCount = samples.size + lastProgressElapsed = elapsed + } + // Same timer scheme as the built-in path: wait long for speech to exist, then a + // progress-anchored budget so mid-scan silence doesn't drain it. Absolute ceiling. + val deadline = if (lastSampleCount == 0) { + MATCH_NO_SPEECH_CAP_MS + } else { + minOf(lastProgressElapsed + MATCH_HARD_CAP_MS, MATCH_TOTAL_CAP_MS) + } + if (elapsed >= deadline) break + // Connection failed (bad key, network) → no samples will ever arrive; bail out now + // instead of spinning until the hard cap. + if (geminiLiveService.state.value == GeminiLiveState.ERROR) { + android.util.Log.w("SubMatch", "hearing aborted: Gemini Live error=${geminiLiveService.errorMessage.value}") + break + } updateMatchStatus( if (samples.isEmpty()) "Searching for a match ($sourceLabel) — waiting for speech…" else "Searching for a match ($sourceLabel) — scanning subtitles… (${samples.size})" @@ -2727,6 +2932,7 @@ class PlayerViewModel @Inject constructor( hasManualSubtitleSelection = true subtitleSelectionJob?.cancel() translationManager.isEnabled = false + targetSubtitleLangCode.takeIf { it.isNotBlank() }?.let { geminiLiveService.targetLanguageCode = it } geminiLiveService.connect() _uiState.value = _uiState.value.copy( isLiveAudioTranslating = true, @@ -2752,6 +2958,72 @@ class PlayerViewModel @Inject constructor( ) } + // ── Per-stream match cache ────────────────────────────────────────────────── + + private data class CachedSubMatch(val key: String, val provider: String, val id: String) + + /** + * Identity of the currently playing file, stable across sessions. Torrent infoHash+fileIdx is + * best; videoHash/filename+size next; the raw URL last (debrid links change per resolve, so a + * URL key may simply never hit again — harmless). + */ + private fun currentStreamCacheKey(): String? { + val stream = _uiState.value.selectedStream + if (stream != null) { + val infoHash = stream.infoHash + if (!infoHash.isNullOrBlank()) return "ih:$infoHash:${stream.fileIdx ?: -1}" + val hints = stream.behaviorHints + val videoHash = hints?.videoHash + if (!videoHash.isNullOrBlank()) return "vh:$videoHash" + val filename = hints?.filename + if (!filename.isNullOrBlank()) { + return "fn:$filename:${hints.videoSize ?: stream.sizeBytes ?: -1}" + } + } + val url = stream?.url?.takeIf { it.isNotBlank() } ?: _uiState.value.selectedStreamUrl + return url?.takeIf { it.isNotBlank() }?.let { "url:${it.hashCode()}" } + } + + private suspend fun loadMatchCache(): MutableList { + val json = context.settingsDataStore.data.first()[subtitleMatchCacheKey] + if (json.isNullOrBlank()) return mutableListOf() + val type = TypeToken.getParameterized(MutableList::class.java, CachedSubMatch::class.java).type + return runCatching { gson.fromJson>(json, type) } + .getOrNull() ?: mutableListOf() + } + + private suspend fun readCachedMatch(): CachedSubMatch? { + val key = currentStreamCacheKey() ?: return null + return loadMatchCache().lastOrNull { it.key == key } + } + + /** Remember [subtitle] as the match for the current stream (null = forget the entry). */ + private fun writeCachedMatch(subtitle: Subtitle?) { + val key = currentStreamCacheKey() ?: return + viewModelScope.launch { + runCatching { + val cache = loadMatchCache() + val removed = cache.removeAll { it.key == key } + if (subtitle == null && !removed) return@runCatching + if (subtitle != null) cache.add(CachedSubMatch(key, subtitle.provider, subtitle.id)) + while (cache.size > MATCH_CACHE_MAX_ENTRIES) cache.removeAt(0) + context.settingsDataStore.edit { it[subtitleMatchCacheKey] = gson.toJson(cache) } + }.onFailure { Log.w("SubMatch", "match cache write failed: ${it.message}") } + } + } + + /** + * A manual track pick overrides what the matcher remembered for this stream: remember the + * user's choice if it's an addon sub in the AI target language, otherwise drop the entry so + * the stale match can't fight the user on the next playback. + */ + private fun updateMatchCacheForManualPick(subtitle: Subtitle) { + val target = targetSubtitleLangCode + val cacheable = target.isNotBlank() && !subtitle.isEmbedded && !subtitle.isBitmap && + subtitle.url.isNotBlank() && normalizeLanguage(subtitle.lang) == target + writeCachedMatch(if (cacheable) subtitle else null) + } + private fun showMatchToast(message: String) { _uiState.value = _uiState.value.copy(matchToast = message) } @@ -3526,18 +3798,33 @@ class PlayerViewModel @Inject constructor( private const val TAG = "PlayerViewModel" // ── "Find best match" tuning ────────────────────────────────────────── - private const val MATCH_HARD_CAP_MS = 90_000L // safety cap; otherwise waits for speech + private const val MATCH_HARD_CAP_MS = 90_000L // give up this long after the FIRST cue/sample appeared + private const val MATCH_NO_SPEECH_CAP_MS = 240_000L // how long to wait for speech to exist at all + private const val MATCH_TOTAL_CAP_MS = 600_000L // absolute ceiling on a single scan + private const val MATCH_MAX_REF_INTERVAL_MS = 20_000L // longer "cues" are seek artifacts private const val MATCH_MAX_SAMPLES = 12 // stop early once we have this many lines - private const val MATCH_MIN_REF_INTERVALS = 1 // built-in mode: min reference cues to decide - private const val MATCH_MAX_REF_INTERVALS = 6 // built-in realtime fallback: stop after this many - private const val MATCH_BUFFERED_REF_INTERVALS = 12 // buffered path is free → use many for accuracy - private const val MATCH_BUFFERED_WAIT_MS = 2_500L // how long to wait for the track to buffer cues - private const val MATCH_EMBEDDED_WAIT_MS = 8_000L // wait for async embedded tracks (background; AI shows meanwhile) + private const val MATCH_MIN_REF_INTERVALS = 2 // built-in mode: min reference cues to decide (1 is too noisy) + private const val MATCH_TARGET_REF_INTERVALS = 8 // built-in mode: stop collecting once we have this many + private const val MATCH_EARLY_ACCEPT_REFS = 4 // min refs before the incremental early-accept check + private const val MATCH_EARLY_ACCEPT_SCORE = 0.8 // a candidate this well-synced ends collection early + private const val MATCH_MIN_REF_SPAN_MS = 30_000L // refs must cover this much video before any accept + private const val MATCH_TRACK_SWITCH_SETTLE_MS = 1_500L // let the reference-track switch reach the renderer + private const val MATCH_BUFFERED_REF_INTERVALS = 12 // max upcoming cues to read from the buffer per poll + private const val MATCH_EMBEDDED_WAIT_MS = 8_000L // grace period for async embedded tracks + private const val MATCH_SOURCES_WAIT_MS = 15_000L // max continuous-playback wait for embedded + addon subs + private const val MATCH_PLAYBACK_WAIT_CAP_MS = 90_000L // give up if playback never starts at all private const val MATCH_MIN_SAMPLES = 4 // need at least this many to decide private const val MATCH_LATENCY_MS = 800L // AI hearing lag: audio time ≈ arrival − this private const val MATCH_TOLERANCE_MS = 1_500L // cue search window around the audio time private const val MATCH_MIN_SIMILARITY = 0.35 // word-overlap needed to count a cue as a hit - private const val MATCH_SUCCESS_THRESHOLD = 0.30 // hit ratio needed to accept a subtitle + // Separate accept thresholds per reference type: timing overlap of a truly synced sub + // scores 0.8+, while a consistently-offset one still reaches 0.5-0.7 — accepting those + // puts a visibly off sub on screen when staying on AI translation would be better. + // Hearing hit-ratios run much lower, so they keep the permissive bar. + private const val MATCH_SUCCESS_THRESHOLD_TIMING = 0.75 + private const val MATCH_SUCCESS_THRESHOLD_HEARING = 0.30 + private const val MATCH_MAX_CANDIDATES = 10 // cap downloaded candidates (best release-name matches first) + private const val MATCH_CACHE_MAX_ENTRIES = 50 // per-stream remembered matches (oldest evicted) /** Known debrid service CDN domains. Reachability checks are skipped for these. */ private val DEBRID_CDN_DOMAINS = setOf( diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt index 111bf00c3..3d627e1f5 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt @@ -11,8 +11,10 @@ import java.util.concurrent.TimeUnit /** * "Find best match": scores candidate subtitle tracks by how well their on-screen cue at a given * moment matches what the AI hearing transcribed being spoken at that same moment. A well-synced - * subtitle shows the right words at the right time; an out-of-sync one does not. Comparison is - * language-specific (currently Hebrew), so it ranks Hebrew subtitle tracks by sync quality. + * subtitle shows the right words at the right time; an out-of-sync one does not. The comparison + * is language-agnostic word overlap (normalization additionally strips Hebrew niqqud and bidi + * marks, which is a no-op for other languages); callers pass candidates in the user's preferred + * subtitle language. */ object SubtitleSyncMatcher { @@ -74,20 +76,31 @@ object SubtitleSyncMatcher { /** * Score a candidate against a synced reference (a built-in subtitle's cue-visible intervals) - * purely by timing. For each reference interval we take the best temporal overlap fraction - * with any candidate cue and average them. A synced candidate has cues at the same times → - * high overlap; an offset candidate → low. Language-agnostic (no translation needed). + * purely by timing. For each reference interval we measure how much of it is covered by the + * union of the candidate's cues, and average the fractions. A synced candidate has cues at + * the same times → high coverage; an offset candidate → low. Union (not best-single-cue) + * because realtime-collected reference intervals merge back-to-back cues into long windows + * that span several candidate cues. Language-agnostic (no translation needed). */ fun scoreByTiming(cues: List, referenceIntervals: List>): Double { if (cues.isEmpty() || referenceIntervals.isEmpty()) return 0.0 + val sorted = cues.sortedBy { it.startMs } var sum = 0.0 for ((rs, re) in referenceIntervals) { val refDur = (re - rs).coerceAtLeast(1L) - val best = cues.maxOfOrNull { c -> - val overlap = minOf(re, c.endMs) - maxOf(rs, c.startMs) - overlap.toDouble() / refDur - } ?: 0.0 - sum += best.coerceIn(0.0, 1.0) + var covered = 0L + var cursor = rs + for (c in sorted) { + if (c.endMs <= cursor) continue + if (c.startMs >= re) break + val s = maxOf(c.startMs, cursor) + val e = minOf(c.endMs, re) + if (e > s) { + covered += e - s + cursor = e + } + } + sum += (covered.toDouble() / refDur).coerceIn(0.0, 1.0) } return sum / referenceIntervals.size } From a85704ff87437deda9bf7e8e73b98fc418fd4bb9 Mon Sep 17 00:00:00 2001 From: silentbil Date: Sat, 4 Jul 2026 23:19:50 +0300 Subject: [PATCH 03/11] fixes --- .../tv/ui/screens/player/PlayerViewModel.kt | 85 ++++++++++++++----- .../tv/ui/screens/settings/SettingsScreen.kt | 28 +++--- 2 files changed, 83 insertions(+), 30 deletions(-) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt index 002dcfb09..f6c2c0999 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt @@ -1371,7 +1371,7 @@ class PlayerViewModel @Inject constructor( ?: normalizedFallback?.let { bestMatch(it) } if (externalMatch != null) { val current = _uiState.value.selectedSubtitle - if (current == null || !(current.isEmbedded && !externalMatch.isEmbedded)) { + if (!embeddedBlocksExternal(current, externalMatch, normalizedPref)) { translationManager.isEnabled = false aiSourceSubtitle = null _uiState.value = _uiState.value.copy(selectedSubtitle = externalMatch, isAiTranslating = false, isAiAvailable = false, aiTargetLanguageName = "") @@ -1394,7 +1394,7 @@ class PlayerViewModel @Inject constructor( ?: normalizedFallback?.let { bestMatch(it) } if (externalMatch != null) { val current = _uiState.value.selectedSubtitle - if (current == null || !(current.isEmbedded && !externalMatch.isEmbedded)) { + if (!embeddedBlocksExternal(current, externalMatch, normalizedPref)) { translationManager.isEnabled = false if (aiEnabledForLanguage) { // Keep AI available in picker — user can still activate it manually @@ -1408,6 +1408,23 @@ class PlayerViewModel @Inject constructor( } } + /** + * Whether a currently-selected embedded track should block auto-selecting [externalMatch]. + * An embedded track normally wins over externals (guaranteed sync) — but a fallback-language + * embedded (e.g. English selected while the preferred-language addon subs were still + * loading) must not block the preferred-language external that arrives later. + */ + private fun embeddedBlocksExternal( + current: Subtitle?, + externalMatch: Subtitle, + normalizedPref: String + ): Boolean { + if (current == null || !current.isEmbedded || externalMatch.isEmbedded) return false + val externalIsPref = normalizeLanguage(externalMatch.lang) == normalizedPref + val currentIsPref = normalizeLanguage(current.lang) == normalizedPref + return !(externalIsPref && !currentIsPref) + } + private fun findAiSourceSubtitle(subtitles: List): Subtitle? { // Some files label forced tracks "SUBFORCED" without setting SELECTION_FLAG_FORCED fun Subtitle.isEffectivelyForced() = isForced || label.contains("forced", ignoreCase = true) @@ -2474,14 +2491,18 @@ class PlayerViewModel @Inject constructor( if (source != null) { activateAiTranslation() findBestSubtitleMatch( - onNoMatch = { + onNoMatch = { score -> // Staying on the AI fallback — re-activate to upgrade the translation source: // an embedded track may have resolved while the scan ran, and the early // activation could still be translating an external fallback source. Say so — // otherwise the searching indicator just vanishes with no visible outcome. if (_uiState.value.isAiTranslating) { activateAiTranslation() - showMatchToast("No well-synced subtitle found — keeping AI translation") + showMatchToast( + "No well-synced subtitle found" + + (score?.let { " (best ${(it * 100).toInt()}%)" } ?: "") + + " — keeping AI translation" + ) } }, useCache = !isUserAction @@ -2554,17 +2575,22 @@ class PlayerViewModel @Inject constructor( * listens to the AI hearing for a short window, then scores each candidate by how often its * on-screen cue matches what was spoken at that moment (see [SubtitleSyncMatcher]). */ - fun findBestSubtitleMatch(onNoMatch: (() -> Unit)? = null, useCache: Boolean = true) { + fun findBestSubtitleMatch(onNoMatch: ((Double?) -> Unit)? = null, useCache: Boolean = true) { if (_uiState.value.isFindingBestMatch) return val previousSubtitle = _uiState.value.selectedSubtitle findMatchJob?.cancel() findMatchJob = viewModelScope.launch { val targetLang = targetSubtitleLangCode.ifBlank { normalizeLanguage(getDefaultSubtitle()) } val targetLangName = languageCodeToName(targetLang) - val noMatch = onNoMatch - ?: { showMatchToast("No well-synced $targetLangName subtitle found") } + // Include the best (rejected) score so a fast verdict is visibly a real scan result. + val noMatch = onNoMatch ?: { score -> + showMatchToast( + "No well-synced $targetLangName subtitle found" + + (score?.let { " (best ${(it * 100).toInt()}%)" } ?: "") + ) + } if (targetLang.isBlank() || isSubtitleDisabledPreference(targetLang)) { - noMatch() + noMatch(null) return@launch } beginMatch("Finding best subtitle…") @@ -2629,7 +2655,7 @@ class PlayerViewModel @Inject constructor( .take(MATCH_MAX_CANDIDATES) if (candidates.isEmpty()) { endMatch() - noMatch() + noMatch(null) return@launch } @@ -2684,7 +2710,7 @@ class PlayerViewModel @Inject constructor( if (loaded.isEmpty()) { endMatch() restoreSubtitle(previousSubtitle) - noMatch() + noMatch(null) selectLastResort() return@launch } @@ -2715,7 +2741,7 @@ class PlayerViewModel @Inject constructor( } else { // Nothing synced found (or too little dialogue) → let the caller fall back (AI translate). restoreSubtitle(previousSubtitle) - noMatch() + noMatch(best?.second) selectLastResort() } } @@ -2799,16 +2825,29 @@ class PlayerViewModel @Inject constructor( lastRefCount = refs.size lastProgressElapsed = elapsed } + // While paused no realtime evidence can arrive — freeze the give-up clock so a scan + // with partial evidence isn't concluded just because the user paused. (A scan with + // full evidence still exits via the target/early-accept breaks above.) + if (!lastIsPlaying) lastProgressElapsed = elapsed // Interval count alone is not evidence: cues clustered in one moment (a recap, a // titles sequence) can agree with an overall-drifting sub. Only accept once the refs // also cover a minimum stretch of the video. - val spanOk = refs.size >= 2 && - refs.last().second - refs.first().first >= MATCH_MIN_REF_SPAN_MS + val span = if (refs.size >= 2) refs.last().second - refs.first().first else 0L + val spanOk = span >= MATCH_MIN_REF_SPAN_MS if (spanOk && refs.size >= MATCH_TARGET_REF_INTERVALS) break // Early accept: scoring is free, so score incrementally — once some candidate is // already clearly in sync there's no need to keep collecting up to the target. - if (spanOk && refs.size >= MATCH_EARLY_ACCEPT_REFS && - loaded.maxOf { (_, cues) -> SubtitleSyncMatcher.scoreByTiming(cues, refs) } >= MATCH_EARLY_ACCEPT_SCORE + val bestNow = if (refs.size >= MATCH_EARLY_ACCEPT_REFS) { + loaded.maxOf { (_, cues) -> SubtitleSyncMatcher.scoreByTiming(cues, refs) } + } else { + 0.0 + } + if (spanOk && refs.size >= MATCH_EARLY_ACCEPT_REFS && bestNow >= MATCH_EARLY_ACCEPT_SCORE) break + // Fast accept: a decisively-synced candidate doesn't need the full span — the + // cluster false-positive it protects against packs refs into a few seconds, which + // half the span already rules out. Saves ~15s of realtime collection per scan. + if (span >= MATCH_FAST_ACCEPT_SPAN_MS && refs.size >= MATCH_FAST_ACCEPT_REFS && + bestNow >= MATCH_FAST_ACCEPT_SCORE ) break // Give-up timers: a long one while waiting for speech to exist at all (openings can // run minutes without dialogue), and a progress-anchored one afterwards — a silent @@ -2828,8 +2867,9 @@ class PlayerViewModel @Inject constructor( collectingReference = false // Inconclusive after the hard cap (too few refs, or all clustered in one moment) — a pick // would be a guess; return null so the caller stays on the AI-translation fallback. + // Floor matches the fast-accept span so a fast-accepted result isn't discarded here. if (refs.size < MATCH_MIN_REF_INTERVALS || - refs.last().second - refs.first().first < MATCH_MIN_REF_SPAN_MS + refs.last().second - refs.first().first < MATCH_FAST_ACCEPT_SPAN_MS ) return null val srcLabel = when { bufferedCount == refs.size -> "buffer" @@ -2882,6 +2922,8 @@ class PlayerViewModel @Inject constructor( lastSampleCount = samples.size lastProgressElapsed = elapsed } + // Paused playback produces no audio — freeze the give-up clock (as in the built-in path). + if (!lastIsPlaying) lastProgressElapsed = elapsed // Same timer scheme as the built-in path: wait long for speech to exist, then a // progress-anchored budget so mid-scan silence doesn't drain it. Absolute ceiling. val deadline = if (lastSampleCount == 0) { @@ -3807,6 +3849,9 @@ class PlayerViewModel @Inject constructor( private const val MATCH_TARGET_REF_INTERVALS = 8 // built-in mode: stop collecting once we have this many private const val MATCH_EARLY_ACCEPT_REFS = 4 // min refs before the incremental early-accept check private const val MATCH_EARLY_ACCEPT_SCORE = 0.8 // a candidate this well-synced ends collection early + private const val MATCH_FAST_ACCEPT_SPAN_MS = 15_000L // decisive candidates accept at half the span + private const val MATCH_FAST_ACCEPT_REFS = 5 // …with at least this many refs + private const val MATCH_FAST_ACCEPT_SCORE = 0.85 // …scoring at least this private const val MATCH_MIN_REF_SPAN_MS = 30_000L // refs must cover this much video before any accept private const val MATCH_TRACK_SWITCH_SETTLE_MS = 1_500L // let the reference-track switch reach the renderer private const val MATCH_BUFFERED_REF_INTERVALS = 12 // max upcoming cues to read from the buffer per poll @@ -3817,11 +3862,11 @@ class PlayerViewModel @Inject constructor( private const val MATCH_LATENCY_MS = 800L // AI hearing lag: audio time ≈ arrival − this private const val MATCH_TOLERANCE_MS = 1_500L // cue search window around the audio time private const val MATCH_MIN_SIMILARITY = 0.35 // word-overlap needed to count a cue as a hit - // Separate accept thresholds per reference type: timing overlap of a truly synced sub - // scores 0.8+, while a consistently-offset one still reaches 0.5-0.7 — accepting those - // puts a visibly off sub on screen when staying on AI translation would be better. + // Separate accept thresholds per reference type: calibrated against user-verified subs — + // a confirmed-synced sub scored 0.71, a confirmed-offset one 0.65 (union-coverage + // scoring, 12 refs). Below the bar, staying on AI translation beats a visibly-off sub. // Hearing hit-ratios run much lower, so they keep the permissive bar. - private const val MATCH_SUCCESS_THRESHOLD_TIMING = 0.75 + private const val MATCH_SUCCESS_THRESHOLD_TIMING = 0.70 private const val MATCH_SUCCESS_THRESHOLD_HEARING = 0.30 private const val MATCH_MAX_CANDIDATES = 10 // cap downloaded candidates (best release-name matches first) private const val MATCH_CACHE_MAX_ENTRIES = 50 // per-stream remembered matches (oldest evicted) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt index 69ce49909..b0f158d2d 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt @@ -902,7 +902,7 @@ fun SettingsScreen( 28 -> viewModel.setSubtitleAiEnabled(!uiState.subtitleAiEnabled) 29 -> showAiModelDialog = true 30 -> viewModel.setSubtitleAiAutoSelect(!uiState.subtitleAiAutoSelect) - 38 -> viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) + 38 -> if (uiState.subtitleAiEnabled && uiState.subtitleAiAutoSelect) viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) 31 -> viewModel.setSubtitleRemoveHearingImpaired(!uiState.subtitleRemoveHearingImpaired) 32 -> showAiApiKeyDialog = true 33 -> viewModel.startAiKeyServer() @@ -3730,14 +3730,21 @@ private fun MobileSettingsSubPage( isFocused = false, onClick = { viewModel.setSubtitleAiAutoSelect(!uiState.subtitleAiAutoSelect) } ) - MobileSettingsRow( - icon = Icons.Default.AutoAwesome, - title = stringResource(R.string.ai_find_best_match_title), - subtitle = stringResource(R.string.ai_find_best_match_desc), - value = if (uiState.subtitleAiFindBestMatch) "On" else "Off", - isFocused = false, - onClick = { viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) } - ) + // Auto-scan only runs as part of the AI auto-select flow — dim it when that's off. + Box(modifier = Modifier.alpha(if (uiState.subtitleAiAutoSelect) 1f else 0.4f)) { + MobileSettingsRow( + icon = Icons.Default.AutoAwesome, + title = stringResource(R.string.ai_find_best_match_title), + subtitle = stringResource(R.string.ai_find_best_match_desc), + value = if (uiState.subtitleAiFindBestMatch) "On" else "Off", + isFocused = false, + onClick = { + if (uiState.subtitleAiAutoSelect) { + viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) + } + } + ) + } MobileSettingsRow( icon = Icons.Default.Subtitles, title = stringResource(R.string.ai_remove_hi_title), @@ -4861,7 +4868,8 @@ private fun TvGeneralSettingsRows( modifier = Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f) ) 30 -> SettingsToggleRow(stringResource(R.string.ai_auto_select_title), stringResource(R.string.ai_auto_select_desc), subtitleAiAutoSelect, focusedIndex == localIndex, onSubtitleAiAutoSelectToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) - 38 -> SettingsToggleRow(stringResource(R.string.ai_find_best_match_title), stringResource(R.string.ai_find_best_match_desc), subtitleAiFindBestMatch, focusedIndex == localIndex, onSubtitleAiFindBestMatchToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) + // Auto-scan only runs as part of the AI auto-select flow — dim it when that's off. + 38 -> SettingsToggleRow(stringResource(R.string.ai_find_best_match_title), stringResource(R.string.ai_find_best_match_desc), subtitleAiFindBestMatch, focusedIndex == localIndex, onSubtitleAiFindBestMatchToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled && subtitleAiAutoSelect) 1f else 0.4f)) 31 -> SettingsToggleRow(stringResource(R.string.ai_remove_hi_title), stringResource(R.string.ai_remove_hi_desc), subtitleRemoveHearingImpaired, focusedIndex == localIndex, onSubtitleRemoveHearingImpairedToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) 32 -> SettingsRow(Icons.Default.VpnKey, stringResource(R.string.ai_api_key_title), stringResource(R.string.ai_api_key_desc), maskAiApiKey(subtitleAiApiKey, stringResource(R.string.ai_key_not_set)), focusedIndex == localIndex, onSubtitleAiApiKeyClick, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) 33 -> SettingsRow(Icons.Default.QrCode, stringResource(R.string.ai_scan_qr_title), stringResource(R.string.ai_scan_qr_desc), "", focusedIndex == localIndex, onSubtitleAiQrClick, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) From 63d08470ef966783f568d4a8527fd2475b826148 Mon Sep 17 00:00:00 2001 From: silentbil Date: Sun, 5 Jul 2026 21:44:34 +0300 Subject: [PATCH 04/11] aio fixes --- .../tv/data/repository/StreamRepository.kt | 55 +++++++++++++------ .../tv/ui/screens/player/PlayerScreen.kt | 4 +- .../tv/ui/screens/player/PlayerViewModel.kt | 37 +++++++++---- 3 files changed, 67 insertions(+), 29 deletions(-) diff --git a/app/src/main/kotlin/com/arflix/tv/data/repository/StreamRepository.kt b/app/src/main/kotlin/com/arflix/tv/data/repository/StreamRepository.kt index 6665df862..9a5e810a6 100644 --- a/app/src/main/kotlin/com/arflix/tv/data/repository/StreamRepository.kt +++ b/app/src/main/kotlin/com/arflix/tv/data/repository/StreamRepository.kt @@ -40,6 +40,7 @@ import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.callbackFlow import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.first @@ -1393,7 +1394,10 @@ class StreamRepository @Inject constructor( private val ADDON_EXTENDED_EPISODE_TIMEOUT_MS = 45_000L private val ADDON_EXTENDED_SINGLE_STREAM_REQUEST_TIMEOUT_MS = 35_000L // Subtitles should not block playback but need enough time on slow connections. - private val SUBTITLE_TIMEOUT_MS = 6_000L + // Generous because aggregator addons (AIOStreams) fan out to upstreams server-side and can + // take 10s+ cold (or 502 outright, then succeed warm — hence the retry). Fetches run in + // parallel and in the background, so a slow addon delays nothing. + private val SUBTITLE_TIMEOUT_MS = 20_000L // If addons return nothing, allow Xtream VOD lookup to recover playback. private val VOD_LOOKUP_TIMEOUT_MS = 6_000L // If addons already returned streams, keep VOD lookup shorter to avoid UI delay. @@ -2729,21 +2733,24 @@ class StreamRepository @Inject constructor( else -> "" } - subtitleAddons.flatMap { addon -> - runCatching { - withTimeout(SUBTITLE_TIMEOUT_MS) { - val addonUrl = addon.url ?: return@withTimeout emptyList() - val (baseUrl, queryParams) = getAddonBaseUrl(addonUrl) - val url = buildSubtitlesUrl( - baseUrl = baseUrl, - type = type, - id = contentId, - addonQueryParams = queryParams, - videoHash = videoHash, - videoSize = videoSize - ) - val response = streamApi.getSubtitles(url) - response.subtitles?.mapIndexed { index, sub -> + // Query addons in parallel — with a generous timeout, sequential fetches would stack + // each slow addon's latency onto the total. + subtitleAddons.map { addon -> + async { + suspend fun attempt(): List = runCatching { + withTimeout(SUBTITLE_TIMEOUT_MS) { + val addonUrl = addon.url ?: return@withTimeout emptyList() + val (baseUrl, queryParams) = getAddonBaseUrl(addonUrl) + val url = buildSubtitlesUrl( + baseUrl = baseUrl, + type = type, + id = contentId, + addonQueryParams = queryParams, + videoHash = videoHash, + videoSize = videoSize + ) + val response = streamApi.getSubtitles(url) + response.subtitles?.mapIndexed { index, sub -> val normalizedLang = normalizeLanguageCode(sub.lang) val langFallback = sub.lang?.trim()?.lowercase()?.take(2) ?: "und" Subtitle( @@ -2754,9 +2761,21 @@ class StreamRepository @Inject constructor( provider = addon.name ) } ?: emptyList() + } + }.onFailure { + Log.w("StreamRepository", "subtitles fetch failed addon=${addon.name} err=${it.message}") + }.getOrDefault(emptyList()) + + // Aggregator endpoints (AIOStreams) often hang or 502 on the first (cold) hit and + // succeed once their upstream fan-out is warm — one retry recovers those. + var subs = attempt() + if (subs.isEmpty()) { + delay(1_500) + subs = attempt() } - }.getOrDefault(emptyList()) - } + subs + } + }.awaitAll().flatten() } private fun buildSubtitlesUrl( diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt index 06eea2469..27a310e16 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt @@ -5061,7 +5061,9 @@ private fun buildExternalSubtitleConfigurations(subtitles: List): List } private fun subtitleMimeTypeFromUrl(url: String): String { - val cleanUrl = url.substringBefore('?').lowercase() + // Trailing slash before the query is real-world (AIOStreams: ".../sub.vtt/?lang=…") — trim it + // so the extension check still sees ".vtt"; the SRT fallback would silently fail on WEBVTT. + val cleanUrl = url.substringBefore('?').trimEnd('/').lowercase() return when { cleanUrl.endsWith(".vtt") -> MimeTypes.TEXT_VTT cleanUrl.endsWith(".srt") || cleanUrl.endsWith(".srt.gz") -> MimeTypes.APPLICATION_SUBRIP diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt index f6c2c0999..04cc6bad7 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt @@ -1832,20 +1832,37 @@ class PlayerViewModel @Inject constructor( ) } + /** Aggregator addons (AIOStreams) sort results server-side per the user's own web config — + * their streams keep the order the addon returned them in instead of being re-sorted. */ + private fun keepsOwnStreamOrder(stream: StreamSource): Boolean = + stream.addonName.contains("aiostream", ignoreCase = true) || + stream.addonId.contains("aiostream", ignoreCase = true) + private fun sortStreamsByQualityAndSize( streams: List, preferredLanguage: String ): List { - return streams.sortedWith( - compareBy { addonOrderIndex(it) } - .thenByDescending { parseSize(it.size) } - .thenByDescending { qualityScore(it.quality) } - .thenByDescending { playbackPriorityScore(it) } - .thenBy { streamRepository.getPlaybackHostHealthPenalty(it) } - .thenBy { if (it.behaviorHints?.notWebReady == true) 1 else 0 } - .thenByDescending { streamLanguageScore(it, preferredLanguage) } - .thenBy { it.source.lowercase() } - ) + val qualityOrder = compareByDescending> { parseSize(it.value.size) } + .thenByDescending { qualityScore(it.value.quality) } + .thenByDescending { playbackPriorityScore(it.value) } + .thenBy { streamRepository.getPlaybackHostHealthPenalty(it.value) } + .thenBy { if (it.value.behaviorHints?.notWebReady == true) 1 else 0 } + .thenByDescending { streamLanguageScore(it.value, preferredLanguage) } + .thenBy { it.value.source.lowercase() } + return streams.withIndex() + .sortedWith( + // Streams stay grouped by addon order; within a group, self-ordered addons keep + // their original (arrival) order while everything else gets the quality sort. + compareBy> { addonOrderIndex(it.value) } + .then { a, b -> + if (keepsOwnStreamOrder(a.value) && keepsOwnStreamOrder(b.value)) { + a.index.compareTo(b.index) + } else { + qualityOrder.compare(a, b) + } + } + ) + .map { it.value } } fun prewarmStream(stream: StreamSource) { From 0c841f8141110695c21f09128fdc9b12b0d5a89a Mon Sep 17 00:00:00 2001 From: silentbil Date: Sun, 5 Jul 2026 22:12:58 +0300 Subject: [PATCH 05/11] i18n --- app/src/main/res/values-iw/strings.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/src/main/res/values-iw/strings.xml b/app/src/main/res/values-iw/strings.xml index 28649c613..75254aa73 100644 --- a/app/src/main/res/values-iw/strings.xml +++ b/app/src/main/res/values-iw/strings.xml @@ -208,6 +208,8 @@ בחר ספק ומודל לתרגום בחירה אוטומטית של תרגום בינה מלאכותית הפעל אוטומטית כאשר קיים תרגום מובנה כלשהו + חיפוש התאמה לתוסף באמצעות בינה מלאכותית + בחירה אוטומטית של כתובית מסונכרנת מההרחבות שלך הסר טקסט למוגבלי שמיעה הסר סימוני [סוגריים] למוגבלי שמיעה לפני תרגום מפתח API From f7685eb6cfda53a34e2616c3b4c5d44ddd4d441e Mon Sep 17 00:00:00 2001 From: silentbil Date: Mon, 6 Jul 2026 10:55:42 +0300 Subject: [PATCH 06/11] fix --- .../ui/screens/player/SubtitleSyncMatcher.kt | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt index 3d627e1f5..063664e67 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt @@ -74,13 +74,21 @@ object SubtitleSyncMatcher { return hits.toDouble() / samples.size } + /** Reference windows longer than this are merged back-to-back cues, not a single cue. */ + private const val SINGLE_CUE_MAX_MS = 7_000L + /** * Score a candidate against a synced reference (a built-in subtitle's cue-visible intervals) - * purely by timing. For each reference interval we measure how much of it is covered by the - * union of the candidate's cues, and average the fractions. A synced candidate has cues at - * the same times → high coverage; an offset candidate → low. Union (not best-single-cue) - * because realtime-collected reference intervals merge back-to-back cues into long windows - * that span several candidate cues. Language-agnostic (no translation needed). + * purely by timing, averaging a per-window overlap fraction. + * + * Two regimes per reference window: + * - Normal windows (a single cue, ≤ [SINGLE_CUE_MAX_MS]): best overlap by any *one* candidate + * cue. This is what discriminates offsets — in dialogue-dense scenes a shifted sub still has + * *chains* of cues touching any window, but no single cue aligns with it. + * - Long windows (realtime collection merges back-to-back cues): coverage by the union of the + * candidate's cues, since even a perfectly synced sub needs several cues to span them. + * + * Language-agnostic (no translation needed). */ fun scoreByTiming(cues: List, referenceIntervals: List>): Double { if (cues.isEmpty() || referenceIntervals.isEmpty()) return 0.0 @@ -88,19 +96,26 @@ object SubtitleSyncMatcher { var sum = 0.0 for ((rs, re) in referenceIntervals) { val refDur = (re - rs).coerceAtLeast(1L) - var covered = 0L - var cursor = rs - for (c in sorted) { - if (c.endMs <= cursor) continue - if (c.startMs >= re) break - val s = maxOf(c.startMs, cursor) - val e = minOf(c.endMs, re) - if (e > s) { - covered += e - s - cursor = e + val fraction = if (refDur <= SINGLE_CUE_MAX_MS) { + sorted.maxOfOrNull { c -> + (minOf(re, c.endMs) - maxOf(rs, c.startMs)).toDouble() / refDur + } ?: 0.0 + } else { + var covered = 0L + var cursor = rs + for (c in sorted) { + if (c.endMs <= cursor) continue + if (c.startMs >= re) break + val s = maxOf(c.startMs, cursor) + val e = minOf(c.endMs, re) + if (e > s) { + covered += e - s + cursor = e + } } + covered.toDouble() / refDur } - sum += (covered.toDouble() / refDur).coerceIn(0.0, 1.0) + sum += fraction.coerceIn(0.0, 1.0) } return sum / referenceIntervals.size } From 4e84f93bb1192356ce04d034938a15634dc731fa Mon Sep 17 00:00:00 2001 From: silentbil Date: Mon, 6 Jul 2026 11:56:55 +0300 Subject: [PATCH 07/11] fix --- .../player/AiSubtitleRenderersFactory.kt | 4 + .../tv/ui/screens/player/PlayerScreen.kt | 71 ++++++++++++----- .../tv/ui/screens/player/PlayerViewModel.kt | 76 ++++++++++++++----- .../player/SubtitleTranslationManager.kt | 4 + .../tv/ui/screens/settings/SettingsScreen.kt | 34 ++++----- app/src/main/res/values-iw/strings.xml | 4 +- app/src/main/res/values/strings.xml | 4 +- 7 files changed, 135 insertions(+), 62 deletions(-) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt index d268604a4..1d2fdf022 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt @@ -322,6 +322,10 @@ private class SubtitleOffsetRenderer( * while TextRenderer.render() has the subtitle buffer populated. */ fun triggerPreTranslation() { + // No AI translation active → no lookahead. tryPeriodicLookahead already checks this; + // this first-cue trigger fires for any newly selected track (e.g. the reference track + // "find best match" selects with AI off) and must not spend API requests either. + if (!translationManager.isEnabled) return if (preTranslatedUpToUs != Long.MIN_VALUE && preTranslatedUpToUs > currentPositionUs + PREFETCH_TRIGGER_US) { return diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt index 230c4484b..b5144e26e 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt @@ -447,13 +447,16 @@ fun PlayerScreen( )) } .toMutableList() - // When AI is available but no subtitles exist in the target language yet, inject a - // synthetic empty group so the AI option is reachable in the picker. - if (uiState.isAiAvailable && uiState.aiTargetLanguageName.isNotBlank() && - groups.none { (name, _) -> name.equals(uiState.aiTargetLanguageName, ignoreCase = true) }) { - groups.add(0, Pair(uiState.aiTargetLanguageName, emptyList())) + // When no subtitles exist in the target language yet, inject a synthetic empty group so + // the "Find Best Match" entry (AI-independent) and the AI option stay reachable. + val headerGroupName = uiState.matchLanguageName.ifBlank { + if (uiState.isAiAvailable) uiState.aiTargetLanguageName else "" } - // "Find Best Match (AI)" is rendered inside the target-language (e.g. Hebrew) group as its + if (headerGroupName.isNotBlank() && + groups.none { (name, _) -> name.equals(headerGroupName, ignoreCase = true) }) { + groups.add(0, Pair(headerGroupName, emptyList())) + } + // "Find Best Match" is rendered inside the target-language (e.g. Hebrew) group as its // first item, alongside the AI translation option — not as a global picker entry. groups.toList() } @@ -2406,10 +2409,13 @@ fun PlayerScreen( } else -> { val group = subtitleGroups.getOrNull(subtitleLangIndex - 1) + val matchGroup = latestUiState.matchLanguageName.isNotBlank() && + group?.first?.equals(latestUiState.matchLanguageName, ignoreCase = true) == true val aiGroup = latestUiState.isAiAvailable && latestUiState.aiTargetLanguageName.isNotBlank() && group?.first?.equals(latestUiState.aiTargetLanguageName, ignoreCase = true) == true - val trackCount = (group?.second?.size ?: 0) + if (aiGroup) 2 else 0 + val headerCount = (if (matchGroup) 1 else 0) + (if (aiGroup) 1 else 0) + val trackCount = (group?.second?.size ?: 0) + headerCount if (subtitleTrackIndex < trackCount - 1) subtitleTrackIndex++ } } @@ -2474,14 +2480,19 @@ fun PlayerScreen( } } else { val group = subtitleGroups.getOrNull(subtitleLangIndex - 1) + val matchGroup = latestUiState.matchLanguageName.isNotBlank() && + group?.first?.equals(latestUiState.matchLanguageName, ignoreCase = true) == true val aiGroup = latestUiState.isAiAvailable && latestUiState.aiTargetLanguageName.isNotBlank() && group?.first?.equals(latestUiState.aiTargetLanguageName, ignoreCase = true) == true - // In the AI/target-language group, index 0 = Find Best Match, 1 = AI translate, 2+ = subs. - val realIdx = if (aiGroup) subtitleTrackIndex - 2 else subtitleTrackIndex - if (aiGroup && subtitleTrackIndex == 0) { + // Target-language group headers: Find Best Match first (AI-independent), + // then the AI translate entry when AI is available, then the subs. + val aiHeaderIdx = if (matchGroup) 1 else 0 + val headerCount = aiHeaderIdx + (if (aiGroup) 1 else 0) + val realIdx = subtitleTrackIndex - headerCount + if (matchGroup && subtitleTrackIndex == 0) { viewModel.runFindBestMatch() - } else if (aiGroup && subtitleTrackIndex == 1) { + } else if (aiGroup && subtitleTrackIndex == aiHeaderIdx) { if (!latestUiState.isAiTranslating) viewModel.activateAiTranslation() } else { group?.second?.getOrNull(realIdx)?.second @@ -2689,6 +2700,16 @@ fun PlayerScreen( val pipSubScale = if (isInPipMode) 0.4f else 1f setFixedTextSize(android.util.TypedValue.COMPLEX_UNIT_SP, baseSizeSp * (subtitleSizePct / 100f) * pipSubScale) setBottomPaddingFraction((subtitleVerticalPct / 100f).coerceIn(0f, 0.5f)) + // "Find best match" without AI showing selects the built-in reference + // track under the hood to read its timing — hide its raw (e.g. English) + // cues while the scan runs. Display-only: the scan's cue collection + // listens on the player, not this view. With AI translating, the + // on-screen text is the translation, so nothing is hidden. + visibility = if (uiState.isFindingBestMatch && !uiState.isAiTranslating) { + android.view.View.INVISIBLE + } else { + android.view.View.VISIBLE + } } }, modifier = Modifier.fillMaxSize() @@ -3321,6 +3342,7 @@ fun PlayerScreen( isAiTranslating = uiState.isAiTranslating, isAiAvailable = uiState.isAiAvailable, aiTargetLanguageName = uiState.aiTargetLanguageName, + matchLanguageName = uiState.matchLanguageName, audioTracks = audioTracks, selectedAudioIndex = selectedAudioIndex, activeTab = subtitleMenuTab, @@ -4188,6 +4210,7 @@ private fun SubtitleMenu( isAiTranslating: Boolean = false, isAiAvailable: Boolean = false, aiTargetLanguageName: String = "", + matchLanguageName: String = "", isLiveAudioTranslating: Boolean = false, isFindingBestMatch: Boolean = false, audioTracks: List, @@ -4339,8 +4362,12 @@ private fun SubtitleMenu( } } else { val isLiveAudioGroup = selectedGroup.first == "Live Audio" + val isMatchGroup = matchLanguageName.isNotBlank() && + selectedGroup.first.equals(matchLanguageName, ignoreCase = true) val isAiGroup = isAiAvailable && aiTargetLanguageName.isNotBlank() && selectedGroup.first.equals(aiTargetLanguageName, ignoreCase = true) + val aiHeaderIdx = if (isMatchGroup) 1 else 0 + val headerCount = aiHeaderIdx + (if (isAiGroup) 1 else 0) LazyColumn( state = trackListState, modifier = Modifier @@ -4349,26 +4376,28 @@ private fun SubtitleMenu( .padding(start = 8.dp), verticalArrangement = Arrangement.spacedBy(2.dp) ) { - if (isAiGroup) { - // First: "Find Best Match (AI)" — runs the match logic. + if (isMatchGroup) { + // First: "Find Best Match" — timing scan, works without AI. item { TrackMenuItem( label = if (isFindingBestMatch) "Scanning…" else "Find Best Match", - subtitle = "AI", + subtitle = "Auto", subtitleDetail = "Auto-pick the best-synced subtitle", isSelected = isFindingBestMatch, isFocused = subtitlePanelFocus == 1 && subtitleTrackIndex == 0, onClick = { /* D-pad only */ } ) } - // Second: translate the built-in subtitle with AI. + } + if (isAiGroup) { + // Translate the built-in subtitle with AI. item { TrackMenuItem( label = aiTargetLanguageName, subtitle = "AI", subtitleDetail = null, isSelected = isAiTranslating, - isFocused = subtitlePanelFocus == 1 && subtitleTrackIndex == 1, + isFocused = subtitlePanelFocus == 1 && subtitleTrackIndex == aiHeaderIdx, onClick = { /* D-pad only */ } ) } @@ -4392,7 +4421,7 @@ private fun SubtitleMenu( .ifBlank { subtitle.id } .ifBlank { null } } - val itemIdx = if (isAiGroup) idx + 2 else idx + val itemIdx = idx + headerCount TrackMenuItem( label = mainLabel, subtitle = badge, @@ -4603,6 +4632,8 @@ private fun SubtitleMenu( // Grouped by language subtitleGroups.forEach { (langName, indexedSubs) -> val isLiveAudioGroup = langName == "Live Audio" + val isMatchGroup = matchLanguageName.isNotBlank() && + langName.equals(matchLanguageName, ignoreCase = true) val isAiGroup = isAiAvailable && aiTargetLanguageName.isNotBlank() && langName.equals(aiTargetLanguageName, ignoreCase = true) item(key = "mobile_header_$langName") { @@ -4635,11 +4666,11 @@ private fun SubtitleMenu( ) } } - if (isAiGroup) { + if (isMatchGroup) { item(key = "mobile_find_best_match_item") { MobileTrackItem( name = if (isFindingBestMatch) "Scanning…" else "Find Best Match", - description = "AI", + description = "Auto", isSelected = isFindingBestMatch, onClick = { onFindBestMatch(); onClose() } ) @@ -4651,6 +4682,8 @@ private fun SubtitleMenu( .background(Color.White.copy(alpha = 0.06f)) ) } + } + if (isAiGroup) { item(key = "mobile_ai_item") { MobileTrackItem( name = aiTargetLanguageName, diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt index 4c2e844e6..de75b2dd2 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt @@ -135,6 +135,9 @@ data class PlayerUiState( val matchToast: String? = null, // Persistent status shown on screen for the whole duration of a "Find best match" scan. val matchStatusText: String = "", + // Full name of the preferred subtitle language (e.g. "Hebrew") — drives the "Find Best Match" + // menu entry. Independent of AI availability: the timing scan needs no AI/API key. + val matchLanguageName: String = "", // Episode title for TV shows (e.g. "The Devil's Verdict"), populated from TMDB season details val episodeTitle: String? = null, // Plot synopsis from TMDB, used in the pause overlay metadata block @@ -211,6 +214,9 @@ class PlayerViewModel @Inject constructor( // Normalized code of the user's preferred subtitle language (e.g. "he") — the target of both // "find best match" and AI translation. Blank when unset/disabled. @Volatile private var targetSubtitleLangCode = "" + // One auto-run of "find best match" per playback/stream — applyPreferredSubtitle re-runs on + // every subtitle-list update and must not restart a completed scan. + private var autoMatchAttempted = false private var aiApiKey = "" private var aiModel = SubtitleAiModel.GROQ_LLAMA_70B private var aiRemoveHearingImpaired = true @@ -413,6 +419,7 @@ class PlayerViewModel @Inject constructor( primaryStreamResolutionFinal = false hasManualSubtitleSelection = false userPickedSubtitle = false + autoMatchAttempted = false aiSourceSubtitle = null val cachedItem = mediaRepository.getCachedItem(mediaType, mediaId) val cachedLogoUrl = mediaRepository.peekCachedLogoUrl(mediaType, mediaId) @@ -464,7 +471,7 @@ class PlayerViewModel @Inject constructor( .let { if (isSubtitleDisabledPreference(it)) "" else it } val secondarySub = prefs[secondarySubtitleKey()]?.trim().orEmpty() .let { if (isSubtitleDisabledPreference(it)) "" else it } - targetSubtitleLangCode = normalizeLanguage(preferredSub) + setTargetSubtitleLang(normalizeLanguage(preferredSub)) // Load AI subtitle settings aiSubtitleEnabled = prefs[aiEnabledKey] ?: false @@ -1227,10 +1234,19 @@ class PlayerViewModel @Inject constructor( } } + /** Keeps [targetSubtitleLangCode] and the menu-facing [PlayerUiState.matchLanguageName] in sync. */ + private fun setTargetSubtitleLang(code: String) { + targetSubtitleLangCode = code + val name = if (code.isBlank()) "" else languageCodeToName(code) + if (_uiState.value.matchLanguageName != name) { + _uiState.value = _uiState.value.copy(matchLanguageName = name) + } + } + private fun applyPreferredSubtitle(preference: String, subtitles: List, fallbackLanguage: String?) { val normalizedPref = normalizeLanguage(preference) if (normalizedPref.isNotBlank() && !isSubtitleDisabledPreference(preference)) { - targetSubtitleLangCode = normalizedPref + setTargetSubtitleLang(normalizedPref) } // Highest priority: an embedded (muxed) track in the preferred language is inherently in @@ -1410,6 +1426,15 @@ class PlayerViewModel @Inject constructor( } } } + // "Find best match first" auto-runs the scan even without AI auto-select: it verifies + // sync on top of the classic release-name pick. AI translation is deliberately NOT + // auto-activated here — that's what the auto-select toggle controls — and the raw + // reference cues stay hidden while scanning, so on no-match the classic pick above + // simply returns to screen. + if (aiFindBestMatchFirst && !autoMatchAttempted && !hasManualSubtitleSelection) { + autoMatchAttempted = true + findBestSubtitleMatch() + } } } @@ -2208,6 +2233,7 @@ class PlayerViewModel @Inject constructor( cancelFindBestMatch() hasManualSubtitleSelection = false userPickedSubtitle = false + autoMatchAttempted = false aiSourceSubtitle = null translationManager.isEnabled = false @@ -2499,17 +2525,24 @@ class PlayerViewModel @Inject constructor( * found, fall back to the AI translation option when one is available. */ fun runFindBestMatch(isUserAction: Boolean = true) { - // Defense-in-depth: the menu entries are already hidden when the AI feature is off, but - // nothing below should ever run without it. - if (!aiSubtitleEnabled) return + // The timing-based scan is AI-independent — no aiSubtitleEnabled gate. Only the optional + // AI interim below and the hearing fallback (gated at the scoring dispatch) need AI. // A user-triggered rescan means the current (possibly remembered) pick is unwanted — // forget the cached entry and scan fresh; the auto-run keeps using the cache. - if (isUserAction) writeCachedMatch(null) - // Show AI translation immediately (if a source exists) so the user sees subtitles right - // away — and because the AI source (English) track is then selected under the hood, the - // match can read its cue timing without ever putting English on screen. Then run the match - // in the background and upgrade to a synced addon sub if one is found; otherwise stay on AI. - val source = findAiSourceSubtitle(_uiState.value.subtitles) ?: aiSourceSubtitle + if (isUserAction) writeCachedMatch(null) else autoMatchAttempted = true + // With AI auto-select on, show AI translation immediately (if a source exists) so the + // user sees subtitles right away — and because the AI source (English) track is then + // selected under the hood, the match can read its cue timing without ever putting English + // on screen. Then run the match in the background and upgrade to a synced addon sub if + // one is found; otherwise stay on AI. + // With AI auto-select OFF (or the AI feature disabled), the user opted out of automatic + // AI translation entirely: the scan runs "silent" (reference cues hidden, zero AI/API + // usage) and falls back to whatever was selected before. + val source = if (aiSubtitleEnabled && aiSubtitleAutoSelect) { + findAiSourceSubtitle(_uiState.value.subtitles) ?: aiSourceSubtitle + } else { + null + } if (source != null) { activateAiTranslation() findBestSubtitleMatch( @@ -2681,12 +2714,16 @@ class PlayerViewModel @Inject constructor( return@launch } - // Last resort when the scan fails outright and nothing else is showing (no AI - // fallback active, no previous selection to restore): behave like the classic non-AI - // flow and pick the best release-name-scored candidate. Sync is unverified, so it is - // deliberately not remembered in the match cache. + // Last resort when the scan fails outright and no AI fallback is active: behave like + // the classic non-AI flow and pick the best release-name-scored candidate. A current + // selection survives only if it's already in the target language — a fallback-language + // stand-in (e.g. embedded English picked while the addon list was still loading) must + // not outlive a failed scan. Sync is unverified, so it is deliberately not remembered + // in the match cache. fun selectLastResort() { - if (_uiState.value.isAiTranslating || _uiState.value.selectedSubtitle != null) return + if (_uiState.value.isAiTranslating) return + val current = _uiState.value.selectedSubtitle + if (current != null && normalizeLanguage(current.lang) == targetLang) return candidates.firstOrNull()?.let { selectSubtitle(it, isUserAction = false) showMatchToast("Selected ${it.label} (sync unverified)") @@ -2743,9 +2780,10 @@ class PlayerViewModel @Inject constructor( // AI translation is already on screen as the fallback — the hearing path is too // unreliable to risk replacing it, so just stay on AI. _uiState.value.isAiTranslating -> null - // The hearing reference streams audio to the Gemini Live API — a Groq key can't - // open that connection, so don't try (it would just hang until the hard cap). - aiModel != SubtitleAiModel.GEMINI_FLASH_25 -> null + // The hearing reference streams audio to the Gemini Live API — it needs the AI + // feature on, a key, and the Gemini model (a Groq key can't open that connection). + !aiSubtitleEnabled || aiApiKey.isBlank() || + aiModel != SubtitleAiModel.GEMINI_FLASH_25 -> null else -> scoreAgainstHearing(loaded, sourceLabel) } endMatch() diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleTranslationManager.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleTranslationManager.kt index aaed83579..913f88066 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleTranslationManager.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleTranslationManager.kt @@ -140,6 +140,10 @@ class SubtitleTranslationManager( } suspend fun preTranslateWindow(texts: List) { + // The lookahead triggers fire whenever a text track renders cues — including tracks + // selected under the hood by "find best match" with AI translation off. Never spend + // API requests unless translation is actually active. + if (!isEnabled) return val uncached = texts.filter { !cache.containsKey(it) && !inFlight.containsKey(it) } if (uncached.isEmpty()) return uncached.chunked(40).forEach { chunk -> diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt index 9ac5de69f..933bde803 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt @@ -216,8 +216,8 @@ private val tvGeneralSectionIds = setOf( private fun tvGeneralRowsForSection(section: String): List { return when (section) { "language" -> listOf(0, 3) - "subtitles" -> listOf(1, 2, 4, 5, 6, 7, 8, 9) - "ai_subtitles" -> listOf(28, 29, 30, 38, 31, 32, 33) + "subtitles" -> listOf(1, 2, 38, 4, 5, 6, 7, 8, 9) + "ai_subtitles" -> listOf(28, 29, 30, 31, 32, 33) "playback" -> listOf(10, 11, 12, 13, 14, 37, 34, 16, 15, 27) "appearance" -> listOf(17, 18, 20, 21, 24, 23, 22, 36) "profiles" -> listOf(19) @@ -903,7 +903,7 @@ fun SettingsScreen( 28 -> viewModel.setSubtitleAiEnabled(!uiState.subtitleAiEnabled) 29 -> showAiModelDialog = true 30 -> viewModel.setSubtitleAiAutoSelect(!uiState.subtitleAiAutoSelect) - 38 -> if (uiState.subtitleAiEnabled && uiState.subtitleAiAutoSelect) viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) + 38 -> viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) 31 -> viewModel.setSubtitleRemoveHearingImpaired(!uiState.subtitleRemoveHearingImpaired) 32 -> showAiApiKeyDialog = true 33 -> viewModel.startAiKeyServer() @@ -3423,6 +3423,15 @@ private fun MobileSettingsMainPage( isFocused = false, onClick = openSecondarySubtitlePicker ) + // AI-independent: the timing-based match scan needs no API key. + MobileSettingsRow( + icon = Icons.Default.Subtitles, + title = stringResource(R.string.ai_find_best_match_title), + subtitle = stringResource(R.string.ai_find_best_match_desc), + value = if (uiState.subtitleAiFindBestMatch) "On" else "Off", + isFocused = false, + onClick = { viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) } + ) MobileSettingsRow( icon = Icons.Default.Subtitles, title = stringResource(R.string.filter_subtitles), @@ -3731,21 +3740,6 @@ private fun MobileSettingsSubPage( isFocused = false, onClick = { viewModel.setSubtitleAiAutoSelect(!uiState.subtitleAiAutoSelect) } ) - // Auto-scan only runs as part of the AI auto-select flow — dim it when that's off. - Box(modifier = Modifier.alpha(if (uiState.subtitleAiAutoSelect) 1f else 0.4f)) { - MobileSettingsRow( - icon = Icons.Default.AutoAwesome, - title = stringResource(R.string.ai_find_best_match_title), - subtitle = stringResource(R.string.ai_find_best_match_desc), - value = if (uiState.subtitleAiFindBestMatch) "On" else "Off", - isFocused = false, - onClick = { - if (uiState.subtitleAiAutoSelect) { - viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) - } - } - ) - } MobileSettingsRow( icon = Icons.Default.Subtitles, title = stringResource(R.string.ai_remove_hi_title), @@ -4870,8 +4864,8 @@ private fun TvGeneralSettingsRows( modifier = Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f) ) 30 -> SettingsToggleRow(stringResource(R.string.ai_auto_select_title), stringResource(R.string.ai_auto_select_desc), subtitleAiAutoSelect, focusedIndex == localIndex, onSubtitleAiAutoSelectToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) - // Auto-scan only runs as part of the AI auto-select flow — dim it when that's off. - 38 -> SettingsToggleRow(stringResource(R.string.ai_find_best_match_title), stringResource(R.string.ai_find_best_match_desc), subtitleAiFindBestMatch, focusedIndex == localIndex, onSubtitleAiFindBestMatchToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled && subtitleAiAutoSelect) 1f else 0.4f)) + // AI-independent: the timing-based match scan needs no API key. + 38 -> SettingsToggleRow(stringResource(R.string.ai_find_best_match_title), stringResource(R.string.ai_find_best_match_desc), subtitleAiFindBestMatch, focusedIndex == localIndex, onSubtitleAiFindBestMatchToggle, Modifier.settingsFocusSlot(localIndex)) 31 -> SettingsToggleRow(stringResource(R.string.ai_remove_hi_title), stringResource(R.string.ai_remove_hi_desc), subtitleRemoveHearingImpaired, focusedIndex == localIndex, onSubtitleRemoveHearingImpairedToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) 32 -> SettingsRow(Icons.Default.VpnKey, stringResource(R.string.ai_api_key_title), stringResource(R.string.ai_api_key_desc), maskAiApiKey(subtitleAiApiKey, stringResource(R.string.ai_key_not_set)), focusedIndex == localIndex, onSubtitleAiApiKeyClick, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) 33 -> SettingsRow(Icons.Default.QrCode, stringResource(R.string.ai_scan_qr_title), stringResource(R.string.ai_scan_qr_desc), "", focusedIndex == localIndex, onSubtitleAiQrClick, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) diff --git a/app/src/main/res/values-iw/strings.xml b/app/src/main/res/values-iw/strings.xml index 75254aa73..8116ea887 100644 --- a/app/src/main/res/values-iw/strings.xml +++ b/app/src/main/res/values-iw/strings.xml @@ -208,8 +208,8 @@ בחר ספק ומודל לתרגום בחירה אוטומטית של תרגום בינה מלאכותית הפעל אוטומטית כאשר קיים תרגום מובנה כלשהו - חיפוש התאמה לתוסף באמצעות בינה מלאכותית - בחירה אוטומטית של כתובית מסונכרנת מההרחבות שלך + חיפוש אוטומטי של הכתובית המתאימה ביותר + בחירה אוטומטית של הכתובית המסונכרנת ביותר מההרחבות שלך הסר טקסט למוגבלי שמיעה הסר סימוני [סוגריים] למוגבלי שמיעה לפני תרגום מפתח API diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 4b032b4e9..cff06215d 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -235,8 +235,8 @@ Select translation provider and model Auto-Select AI Translation Automatically activate AI when any built in language is available - Use AI to Find Best Addon Match First - auto-pick a synced subtitle from your addons using AI + Auto Find Best Subtitle Match + Auto-pick the best-synced subtitle from your addons Remove Hearing Impaired Text Strip [bracketed] hearing-impaired markers before translation API Key From 41182ab7acd8b62fe5e217000914f84edd8b2125 Mon Sep 17 00:00:00 2001 From: silentbil Date: Mon, 6 Jul 2026 12:54:12 +0300 Subject: [PATCH 08/11] fix --- .../tv/ui/screens/player/PlayerScreen.kt | 16 ++- .../tv/ui/screens/player/PlayerViewModel.kt | 108 +++++++++++++----- .../ui/screens/player/SubtitleSyncMatcher.kt | 16 +-- 3 files changed, 101 insertions(+), 39 deletions(-) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt index b5144e26e..16a048cde 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt @@ -764,10 +764,16 @@ fun PlayerScreen( .setDefaultRequestProperties(baseRequestHeaders) } val mediaCache = remember(context) { PlaybackCacheSingleton.getInstance(context) } - val cacheDataSourceFactory = remember(httpDataSourceFactory, mediaCache) { + // Wrap the OkHttp factory so file:// URIs also work — matched subtitles are served from a + // local cache file (already downloaded by the scan) instead of re-fetching addon servers. + // http/https still routes through the same OkHttp client as before. + val fileCapableDataSourceFactory = remember(httpDataSourceFactory) { + androidx.media3.datasource.DefaultDataSource.Factory(context, httpDataSourceFactory) + } + val cacheDataSourceFactory = remember(fileCapableDataSourceFactory, mediaCache) { CacheDataSource.Factory() .setCache(mediaCache) - .setUpstreamDataSourceFactory(httpDataSourceFactory) + .setUpstreamDataSourceFactory(fileCapableDataSourceFactory) .setFlags(CacheDataSource.FLAG_IGNORE_CACHE_ON_ERROR) } // Non-cached factory for heavy/debrid progressive streams to avoid disk I/O bottleneck @@ -1847,6 +1853,12 @@ fun PlayerScreen( !userSelectedSourceManually && tryAdvanceToNextStream() ) { + // Restart the startup clock immediately: the real reset happens only after + // the next stream resolves (async). Without this, the loop re-evaluates the + // stale clock on the very next iteration and fires a failover burst that + // burns through the entire source list in milliseconds. + streamSelectedTime = System.currentTimeMillis() + startupRecoverAttempted = false continue } startupHardFailureReported = true diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt index de75b2dd2..6c63f4a44 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt @@ -2426,15 +2426,22 @@ class PlayerViewModel @Inject constructor( } val resolvedSelected = if (selected != null) { - finalList.firstOrNull { it.id == selected.id } - ?: finalList.firstOrNull { selected.url.isNotBlank() && it.url == selected.url } - // After MediaItem rebuild groupIndex can change, making generated IDs stale. - // Fall back to lang+label match for embedded subs so the nonce-based - // LaunchedEffect fires with the freshly-indexed version. - ?: if (selected.isEmbedded) finalList.firstOrNull { - it.isEmbedded && it.lang == selected.lang && it.label == selected.label - } else null - ?: selected + if (!selected.isEmbedded && selected.url.startsWith("file:")) { + // A matched subtitle localized to a cache file — keep it as-is. Remapping to + // the addon-list entry (same id) would swap the URL back to the remote server + // and re-trigger a MediaItem rebuild with the slow remote source. + selected + } else { + finalList.firstOrNull { it.id == selected.id } + ?: finalList.firstOrNull { selected.url.isNotBlank() && it.url == selected.url } + // After MediaItem rebuild groupIndex can change, making generated IDs stale. + // Fall back to lang+label match for embedded subs so the nonce-based + // LaunchedEffect fires with the freshly-indexed version. + ?: if (selected.isEmbedded) finalList.firstOrNull { + it.isEmbedded && it.lang == selected.lang && it.label == selected.label + } else null + ?: selected + } } else { null } @@ -2714,22 +2721,6 @@ class PlayerViewModel @Inject constructor( return@launch } - // Last resort when the scan fails outright and no AI fallback is active: behave like - // the classic non-AI flow and pick the best release-name-scored candidate. A current - // selection survives only if it's already in the target language — a fallback-language - // stand-in (e.g. embedded English picked while the addon list was still loading) must - // not outlive a failed scan. Sync is unverified, so it is deliberately not remembered - // in the match cache. - fun selectLastResort() { - if (_uiState.value.isAiTranslating) return - val current = _uiState.value.selectedSubtitle - if (current != null && normalizeLanguage(current.lang) == targetLang) return - candidates.firstOrNull()?.let { - selectSubtitle(it, isUserAction = false) - showMatchToast("Selected ${it.label} (sync unverified)") - } - } - // A previously matched subtitle for this exact stream (same file → same sync) that is // still offered by the addons wins immediately — no need to rescan. Skipped for // user-triggered rescans, which mean the remembered pick is unwanted. @@ -2738,8 +2729,12 @@ class PlayerViewModel @Inject constructor( candidates.firstOrNull { it.id == cached.id && it.provider == cached.provider } } if (remembered != null) { + // Serve from a local cache file so the MediaItem rebuild doesn't stall on a + // slow addon server (download bounded by the matcher's client timeouts). + val local = SubtitleSyncMatcher.loadRaw(remembered.url) + ?.let { localizeSubtitle(remembered, it) } ?: remembered endMatch() - selectSubtitle(remembered, isUserAction = false) + selectSubtitle(local, isUserAction = false) showMatchToast("Matched: ${remembered.label} (remembered)") return@launch } @@ -2762,9 +2757,42 @@ class PlayerViewModel @Inject constructor( ) updateMatchStatus("Finding best subtitle ($sourceLabel)…") - val loaded = candidates.map { sub -> - async { sub to SubtitleSyncMatcher.loadCues(sub.url) } - }.awaitAll().filter { it.second.isNotEmpty() } + // Keep the raw text alongside the parsed cues: the winning subtitle is later served to + // ExoPlayer from a local cache file (already downloaded here) instead of re-fetching + // a possibly slow addon server during the MediaItem rebuild. + val loadedRaw = candidates.map { sub -> + async { sub to SubtitleSyncMatcher.loadRaw(sub.url) } + }.awaitAll() + val rawBySubKey = loadedRaw.mapNotNull { (sub, raw) -> + raw?.let { "${sub.provider}|${sub.id}" to it } + }.toMap() + val loaded = loadedRaw.mapNotNull { (sub, raw) -> + val cues = raw?.let { SubtitleSyncMatcher.parseCues(it) }.orEmpty() + if (cues.isEmpty()) null else sub to cues + } + + /** Select [sub], serving it from a local file when its text was downloaded. */ + fun selectServedLocally(sub: Subtitle) { + val localized = rawBySubKey["${sub.provider}|${sub.id}"] + ?.let { localizeSubtitle(sub, it) } ?: sub + selectSubtitle(localized, isUserAction = false) + } + + // Last resort when the scan fails outright and no AI fallback is active: behave like + // the classic non-AI flow and pick the best release-name-scored candidate. A current + // selection survives only if it's already in the target language — a fallback-language + // stand-in (e.g. embedded English picked while the addon list was still loading) must + // not outlive a failed scan. Sync is unverified, so it is deliberately not remembered + // in the match cache. + fun selectLastResort() { + if (_uiState.value.isAiTranslating) return + val current = _uiState.value.selectedSubtitle + if (current != null && normalizeLanguage(current.lang) == targetLang) return + candidates.firstOrNull()?.let { + selectServedLocally(it) + showMatchToast("Selected ${it.label} (sync unverified)") + } + } if (loaded.isEmpty()) { endMatch() @@ -2795,7 +2823,8 @@ class PlayerViewModel @Inject constructor( } val best = scored?.maxByOrNull { it.second } if (best != null && best.second >= successThreshold) { - selectSubtitle(best.first, isUserAction = false) + selectServedLocally(best.first) + // Cache the original (addon) identity — the local file is per-session transient. writeCachedMatch(best.first) showMatchToast("Matched: ${best.first.label} · ${(best.second * 100).toInt()}% ($sourceLabel)") } else { @@ -3126,6 +3155,25 @@ class PlayerViewModel @Inject constructor( writeCachedMatch(if (cacheable) subtitle else null) } + /** + * Writes downloaded subtitle text to a small local cache file and returns a copy of [sub] + * pointing at it (file:// URI). ExoPlayer then side-loads the matched subtitle instantly + * during the MediaItem rebuild instead of re-downloading it from a possibly slow/flaky addon + * server — which is what used to leave the player stuck buffering after a match. Also + * normalizes the content (UTF-8, gunzipped) as a side effect. + */ + private fun localizeSubtitle(sub: Subtitle, raw: String): Subtitle { + return runCatching { + val dir = java.io.File(context.cacheDir, "matched_subs").apply { mkdirs() } + // Keep the directory bounded; files are ~100KB and keyed stably per subtitle. + dir.listFiles()?.sortedBy { it.lastModified() }?.dropLast(39)?.forEach { it.delete() } + val ext = if (raw.trimStart().startsWith("WEBVTT")) "vtt" else "srt" + val file = java.io.File(dir, "${("${sub.provider}|${sub.id}").hashCode().toUInt()}.$ext") + file.writeText(raw) + sub.copy(url = file.toURI().toString()) + }.getOrDefault(sub) + } + private fun showMatchToast(message: String) { _uiState.value = _uiState.value.copy(matchToast = message) } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt index 063664e67..ef210f3f4 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt @@ -30,25 +30,27 @@ object SubtitleSyncMatcher { .readTimeout(15, TimeUnit.SECONDS) .build() - suspend fun loadCues(url: String): List = withContext(Dispatchers.IO) { + /** Downloads a subtitle and returns its decoded (UTF-8, gunzipped) text, or null on failure. */ + suspend fun loadRaw(url: String): String? = withContext(Dispatchers.IO) { runCatching { val request = Request.Builder().url(url).build() client.newCall(request).execute().use { response -> - if (!response.isSuccessful) return@use emptyList() - val body = response.body ?: return@use emptyList() + if (!response.isSuccessful) return@use null + val body = response.body ?: return@use null val raw = body.bytes() - val text = if (looksGzipped(url, raw)) { + if (looksGzipped(url, raw)) { GZIPInputStream(raw.inputStream()).bufferedReader(Charsets.UTF_8).readText() } else { String(raw, Charsets.UTF_8) } - parseCues(text) } }.onFailure { error -> - Log.w(TAG, "loadCues failed url=$url err=${error.message}") - }.getOrDefault(emptyList()) + Log.w(TAG, "loadRaw failed url=$url err=${error.message}") + }.getOrNull() } + suspend fun loadCues(url: String): List = loadRaw(url)?.let { parseCues(it) } ?: emptyList() + /** * Score a candidate's cues against the collected spoken samples. For each sample we look at * the audio's media time (arrival position minus the AI latency), find cues within a tolerance From 07ba1e048f923974f96281b1f8f496736ada42fd Mon Sep 17 00:00:00 2001 From: silentbil Date: Mon, 6 Jul 2026 19:09:50 +0300 Subject: [PATCH 09/11] fix --- .../tv/data/repository/CloudSyncRepository.kt | 4 +- .../player/AiSubtitleRenderersFactory.kt | 136 +++++++-- .../tv/ui/screens/player/PlayerScreen.kt | 62 ++++- .../tv/ui/screens/player/PlayerViewModel.kt | 263 ++++++++++++++---- .../tv/ui/screens/settings/SettingsScreen.kt | 4 +- .../ui/screens/settings/SettingsViewModel.kt | 4 +- docs/subtitle-auto-match.md | 229 +++++++++++++++ 7 files changed, 606 insertions(+), 96 deletions(-) create mode 100644 docs/subtitle-auto-match.md diff --git a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt index ea8685a8a..18488300b 100644 --- a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt +++ b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt @@ -490,7 +490,7 @@ class CloudSyncRepository @Inject constructor( // Global AI subtitle settings (non-profile-scoped) root.put("subtitleAiEnabled", prefs[subtitleAiEnabledKey] ?: false) root.put("subtitleAiAutoSelect", prefs[subtitleAiAutoSelectKey] ?: false) - root.put("subtitleAiFindBestMatch", prefs[subtitleAiFindBestMatchKey] ?: true) + root.put("subtitleAiFindBestMatch", prefs[subtitleAiFindBestMatchKey] ?: false) root.put("subtitleAiApiKey", prefs[subtitleAiApiKeyKey] ?: "") root.put("subtitleAiModel", prefs[subtitleAiModelKey] ?: "GROQ_LLAMA_70B") root.put("subtitleRemoveHearingImpaired", prefs[subtitleRemoveHearingImpairedKey] ?: true) @@ -1229,7 +1229,7 @@ class CloudSyncRepository @Inject constructor( context.settingsDataStore.edit { prefs -> prefs[subtitleAiEnabledKey] = root.optBoolean("subtitleAiEnabled", false) prefs[subtitleAiAutoSelectKey] = root.optBoolean("subtitleAiAutoSelect", false) - prefs[subtitleAiFindBestMatchKey] = root.optBoolean("subtitleAiFindBestMatch", true) + prefs[subtitleAiFindBestMatchKey] = root.optBoolean("subtitleAiFindBestMatch", false) val apiKey = root.optString("subtitleAiApiKey", "") if (apiKey.isNotBlank()) prefs[subtitleAiApiKeyKey] = apiKey val model = root.optString("subtitleAiModel", "GROQ_LLAMA_70B") diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt index 1d2fdf022..ac8997fed 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt @@ -47,6 +47,19 @@ class AiSubtitleRenderersFactory( return emptyList() } + /** + * Text of the currently-buffered cues of the selected text track. Lets "Find best match" + * verify WHICH track the buffer belongs to (by comparing against the displayed candidate's + * own lines) — timing-based self-detection false-positived on subs cut from the same master. + */ + fun extractBufferedCueTexts(maxCount: Int): List { + for (renderer in offsetRenderers) { + val texts = renderer.extractAllCueTexts() + if (texts.isNotEmpty()) return texts.take(maxCount) + } + return emptyList() + } + override fun buildAudioSink( context: Context, enableFloatOutput: Boolean, @@ -350,7 +363,7 @@ private class SubtitleOffsetRenderer( // ── Reflection-based cue extraction ────────────────────────────────────── - private fun extractAllCueTexts(): List { + fun extractAllCueTexts(): List { val removeHI = translationManager.removeHearingImpaired val texts = mutableSetOf() @@ -413,32 +426,117 @@ private class SubtitleOffsetRenderer( return texts.toList() } + private var bufDiagLogged = 0 + /** Buffered cue intervals (startMs, endMs) for text-carrying cues, from the modern resolver. */ fun extractBufferedIntervals(maxCount: Int): List> { val intervals = ArrayList>() // Cue buffer timestamps are on ExoPlayer's internal stream timeline, which adds a large // base offset; subtract the renderer's stream offset to get 0-based media time. val offsetMs = readStreamOffsetUs() / 1000L + var resolverFound = false + var fieldUsed: String? = null try { - val resolverField = findField(baseRenderer.javaClass, "cuesResolver") ?: return emptyList() - val resolver = resolverField.get(baseRenderer) ?: return emptyList() - for (candidate in listOf("cuesWithTimingList", "cueGroupsByStartTime", "cueGroups", "cueGroupList", "groups")) { - val f = findField(resolver.javaClass, candidate) ?: continue - val v = f.get(resolver) ?: continue - val items: Collection<*> = when (v) { - is Map<*, *> -> v.values - is Collection<*> -> v - else -> continue + val resolverField = findField(baseRenderer.javaClass, "cuesResolver") + val resolver = resolverField?.get(baseRenderer) + resolverFound = resolver != null + if (resolver != null) { + // Media3 has multiple resolver impls chosen per track cue-replacement behavior: + // MergingCuesResolver ("cuesWithTimingList") and ReplacingCuesResolver + // ("cuesWithTimings") — probe both; content differs per rip (MKV SubRip vs VTT). + for (candidate in listOf( + "cuesWithTimingList", "cuesWithTimings", + "cueGroupsByStartTime", "cueGroups", "cueGroupList", "groups" + )) { + val f = findField(resolver.javaClass, candidate) ?: continue + val v = f.get(resolver) ?: continue + val items: Collection<*> = when (v) { + is Map<*, *> -> v.values + is Collection<*> -> v + else -> continue + } + for (item in items) { + val (s, e) = item?.let(::intervalFromCueWrapper) ?: continue + intervals.add((s - offsetMs) to (e - offsetMs)) + } + if (intervals.isNotEmpty()) { + fieldUsed = candidate + break + } } - for (item in items) { - val (s, e) = item?.let(::intervalFromCueWrapper) ?: continue - intervals.add((s - offsetMs) to (e - offsetMs)) + if (intervals.isEmpty()) { + // Unknown impl/field name (new media3 version?) — scan every field for a + // cue-carrying collection, like the text extractor already does. + var cls: Class<*>? = resolver.javaClass + outer@ while (cls != null && cls != Any::class.java) { + for (f in cls.declaredFields) { + try { + f.isAccessible = true + val v = f.get(resolver) ?: continue + val items: Collection<*> = when (v) { + is Map<*, *> -> v.values + is Collection<*> -> v + else -> continue + } + for (item in items) { + val (s, e) = item?.let(::intervalFromCueWrapper) ?: continue + intervals.add((s - offsetMs) to (e - offsetMs)) + } + if (intervals.isNotEmpty()) { + fieldUsed = "fallback:${f.name}" + break@outer + } + } catch (_: Exception) { + } + } + cls = cls.superclass + } } - if (intervals.isNotEmpty()) break } } catch (_: Exception) { } - return intervals.filter { it.first >= 0 }.distinct().sortedBy { it.first }.take(maxCount) + val out = intervals.filter { it.first >= 0 }.distinct().sortedBy { it.first }.take(maxCount) + // Diagnostics: pinpoints whether slow scans starve at the reflection (resolver/field + // missing — e.g. R8), at the offset correction (raw > 0 but out = 0), or downstream. + if (bufDiagLogged < 10) { + bufDiagLogged++ + android.util.Log.i( + "SubMatch", + "bufDiag resolver=$resolverFound field=$fieldUsed raw=${intervals.size} out=${out.size} offsetMs=$offsetMs" + ) + // Extraction found nothing: dump the resolver's real shape once so the probe list can + // be corrected from evidence instead of guessed media3 internals. + if (resolverFound && intervals.isEmpty() && bufDiagLogged == 5) { + runCatching { + val resolver = findField(baseRenderer.javaClass, "cuesResolver")?.get(baseRenderer) + if (resolver != null) { + val desc = buildString { + append(resolver.javaClass.name).append(" {") + var cls: Class<*>? = resolver.javaClass + while (cls != null && cls != Any::class.java) { + for (f in cls.declaredFields) { + runCatching { + f.isAccessible = true + val v = f.get(resolver) + val size = when (v) { + is Collection<*> -> "size=${v.size}" + is Map<*, *> -> "size=${v.size}" + null -> "null" + else -> v.javaClass.simpleName + } + append(" ${f.name}:$size;") + } + } + cls = cls.superclass + } + append(" }") + } + android.util.Log.i("SubMatch", "bufDiag resolverDump $desc") + } + } + } + } + return out } /** The renderer's stream offset (µs) — the base added to buffer sample timestamps. */ @@ -465,7 +563,13 @@ private class SubtitleOffsetRenderer( endUs = startUs + 2_000_000L // CueGroup carries no duration — assume 2s on screen } else { val start = runCatching { findField(obj.javaClass, "startTimeUs")?.getLong(obj) }.getOrNull() ?: return null - val dur = runCatching { findField(obj.javaClass, "durationUs")?.getLong(obj) }.getOrNull() ?: 2_000_000L + // REPLACE-behavior tracks (ReplacingCuesResolver, e.g. MKV SubRip) carry + // durationUs = C.TIME_UNSET (large negative) — each cue lasts until replaced. The + // negative value made endUs < startUs and every item got rejected, silently killing + // the buffered fast path for that whole content category. Fall back to a nominal + // duration: the start timestamp carries the alignment signal. + val dur = runCatching { findField(obj.javaClass, "durationUs")?.getLong(obj) }.getOrNull() + ?.takeIf { it > 0L } ?: 2_000_000L startUs = start endUs = start + dur } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt index 16a048cde..2c674d3c8 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt @@ -819,11 +819,15 @@ fun PlayerScreen( viewModel.bufferedReferenceIntervalsProvider = { max -> aiRenderersFactory.extractBufferedReferenceIntervals(max) } + viewModel.bufferedCueTextsProvider = { max -> + aiRenderersFactory.extractBufferedCueTexts(max) + } } DisposableEffect(aiRenderersFactory) { onDispose { aiRenderersFactory.audioCaptureProcessor?.onChunk = null viewModel.bufferedReferenceIntervalsProvider = null + viewModel.bufferedCueTextsProvider = null viewModel.geminiLiveService.disconnect() } } @@ -915,7 +919,11 @@ fun PlayerScreen( // Feed the selected text track's rendered cues to "Find best match" so it can // read a built-in subtitle's timing (embedded tracks have no URL to parse). override fun onCues(cueGroup: androidx.media3.common.text.CueGroup) { - viewModel.onPlayerCues(cueGroup.cues.isNotEmpty(), cueGroup.presentationTimeUs / 1000L) + viewModel.onPlayerCues( + cueGroup.cues.isNotEmpty(), + cueGroup.presentationTimeUs / 1000L, + cueGroup.cues.firstOrNull()?.text?.toString() + ) } override fun onIsPlayingChanged(playing: Boolean) { @@ -1602,21 +1610,45 @@ fun PlayerScreen( } if (subtitle.isEmbedded && subtitle.groupIndex != null && subtitle.trackIndex != null) { - // For embedded subs, just select the track directly - val groups = exoPlayer.currentTracks.groups - val params = exoPlayer.trackSelectionParameters.buildUpon() - .clearOverridesOfType(C.TRACK_TYPE_TEXT) - .setTrackTypeDisabled(C.TRACK_TYPE_TEXT, false) - if (subtitle.groupIndex in groups.indices && - groups[subtitle.groupIndex].type == C.TRACK_TYPE_TEXT) { - params.setOverrideForType( - androidx.media3.common.TrackSelectionOverride( - groups[subtitle.groupIndex].mediaTrackGroup, - subtitle.trackIndex - ) - ) + // Embedded subs: apply a track override. Track lists refresh asynchronously (e.g. + // right after a MediaItem rebuild), so retry briefly with freshly-resolved indices + // instead of a one-shot: a stale one-shot either selected the WRONG track via the + // preferred-language fallback (find-best-match then scored a candidate against + // itself) or, if it just disabled text, starved the scan of reference cues. + repeat(20) { attempt -> + val groups = exoPlayer.currentTracks.groups + val fresh = latestUiState.subtitles.firstOrNull { + it.isEmbedded && it.id == subtitle.id && + it.groupIndex != null && it.trackIndex != null + } ?: subtitle + val gi = fresh.groupIndex + val ti = fresh.trackIndex + val valid = gi != null && ti != null && gi in groups.indices && + groups[gi].type == C.TRACK_TYPE_TEXT && + ti < groups[gi].mediaTrackGroup.length + val params = exoPlayer.trackSelectionParameters.buildUpon() + .clearOverridesOfType(C.TRACK_TYPE_TEXT) + if (valid) { + params + .setTrackTypeDisabled(C.TRACK_TYPE_TEXT, false) + // Clear the preferred-language fallback: with an explicit override it's + // redundant, and if the override turns invalid it silently selects a + // DIFFERENT track. + .setPreferredTextLanguage(null) + .setOverrideForType( + androidx.media3.common.TrackSelectionOverride( + groups[gi!!].mediaTrackGroup, + ti!! + ) + ) + exoPlayer.trackSelectionParameters = params.build() + return@LaunchedEffect + } + // Not ready yet: showing the wrong track is worse than showing nothing. + params.setTrackTypeDisabled(C.TRACK_TYPE_TEXT, true) + exoPlayer.trackSelectionParameters = params.build() + delay(500) } - exoPlayer.trackSelectionParameters = params.build() return@LaunchedEffect } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt index 6c63f4a44..b6b58d5b3 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt @@ -210,7 +210,7 @@ class PlayerViewModel @Inject constructor( // When true (default), activating AI first tries to auto-pick a synced addon subtitle in the // user's preferred language (Find Best Match); only if nothing matches does it translate the // built-in subtitle. - private var aiFindBestMatchFirst = true + private var aiFindBestMatchFirst = false // Normalized code of the user's preferred subtitle language (e.g. "he") — the target of both // "find best match" and AI translation. Blank when unset/disabled. @Volatile private var targetSubtitleLangCode = "" @@ -285,16 +285,40 @@ class PlayerViewModel @Inject constructor( /** Set by the player screen: reads the selected text track's already-buffered cue intervals. */ @Volatile var bufferedReferenceIntervalsProvider: ((Int) -> List>)? = null + /** Set by the player screen: reads the selected text track's already-buffered cue texts. */ + @Volatile var bufferedCueTextsProvider: ((Int) -> List)? = null + + // Normalized lines of the candidate displayed when the scan started — set for the duration of + // reference collection so BOTH the buffered and the realtime paths can reject self-cues (the + // reference track switch landing late/never would otherwise score a candidate against itself). + @Volatile private var matchPreviousTexts: Set? = null + @Volatile private var refIntervalSelfText = false + /** Rendered-cue callback from the player; records reference intervals during a built-in scan. */ - fun onPlayerCues(hasText: Boolean, timeMs: Long) { + fun onPlayerCues(hasText: Boolean, timeMs: Long, textSample: String? = null) { if (!collectingReference || timeMs < 0) return if (hasText) { - if (refIntervalStart < 0) refIntervalStart = timeMs + if (refIntervalStart < 0) { + refIntervalStart = timeMs + refIntervalSelfText = false + } + val prev = matchPreviousTexts + if (prev != null && textSample != null && + normalizeCueTextForCompare(textSample) in prev + ) { + refIntervalSelfText = true + } } else if (refIntervalStart in 0 until timeMs) { // An interval spanning a forward seek would record a bogus minutes-long "cue"; // real cues never stay on screen this long — drop it. - if (timeMs - refIntervalStart <= MATCH_MAX_REF_INTERVAL_MS) { + val validLength = timeMs - refIntervalStart <= MATCH_MAX_REF_INTERVAL_MS + if (validLength && !refIntervalSelfText) { synchronized(referenceIntervals) { referenceIntervals.add(refIntervalStart to timeMs) } + } else if (refIntervalSelfText) { + android.util.Log.w( + "SubMatch", + "realtime self-cue interval dropped ${refIntervalStart}..$timeMs — reference track switch not landed?" + ) } refIntervalStart = -1L } else if (refIntervalStart >= 0) { @@ -427,6 +451,10 @@ class PlayerViewModel @Inject constructor( isLoading = true, isLoadingStreams = false, selectedSubtitle = null, + // A new video means a new subtitle universe — stale externals from the previous + // title poisoned both the picker and the match scan (references from the new video + // scored against the old video's candidates). + subtitles = emptyList(), isAiTranslating = false, isAiAvailable = false, aiTargetLanguageName = "", @@ -476,7 +504,7 @@ class PlayerViewModel @Inject constructor( // Load AI subtitle settings aiSubtitleEnabled = prefs[aiEnabledKey] ?: false aiSubtitleAutoSelect = prefs[aiAutoSelectKey] ?: false - aiFindBestMatchFirst = prefs[aiFindBestMatchKey] ?: true + aiFindBestMatchFirst = prefs[aiFindBestMatchKey] ?: false aiApiKey = prefs[aiApiKeyKey] ?: "" aiModel = runCatching { SubtitleAiModel.valueOf(prefs[aiModelKey] ?: SubtitleAiModel.GROQ_LLAMA_70B.name) @@ -571,58 +599,85 @@ class PlayerViewModel @Inject constructor( selectedStreamUrl = resolvedProvidedUrl, savedPosition = resumeData.positionMs ) + // NOTE: these background children share the load job — an uncaught exception in + // any of them cancels ALL siblings (that silently killed the subtitle flow). + // Every body is therefore failure-isolated and logged. + fun childFailed(name: String): (Throwable) -> Unit = { e -> + if (e !is kotlinx.coroutines.CancellationException) { + android.util.Log.w("PlayerViewModel", "load child '$name' failed: ${e.message}") + } + } launch { kotlinx.coroutines.delay(1_500L) - populateStreamsForProvidedUrl( - mediaType = mediaType, - mediaId = mediaId, - seasonNumber = seasonNumber, - episodeNumber = episodeNumber, - providedImdbId = providedImdbId, - playbackUrl = resolvedProvidedUrl - ) + runCatching { + populateStreamsForProvidedUrl( + mediaType = mediaType, + mediaId = mediaId, + seasonNumber = seasonNumber, + episodeNumber = episodeNumber, + providedImdbId = providedImdbId, + playbackUrl = resolvedProvidedUrl + ) + }.onFailure(childFailed("populateStreams")) } // Load IPTV and home-server sources from cache in parallel so the // source picker in the player shows alternatives immediately. homeServerAppendJob?.cancel() homeServerAppendJob = launch { - appendHomeServerSourcesInBackground( - mediaType = mediaType, - imdbId = currentImdbId, - seasonNumber = seasonNumber, - episodeNumber = episodeNumber, - timeoutMs = 5_000L - ) + runCatching { + appendHomeServerSourcesInBackground( + mediaType = mediaType, + imdbId = currentImdbId, + seasonNumber = seasonNumber, + episodeNumber = episodeNumber, + timeoutMs = 5_000L + ) + }.onFailure(childFailed("homeServerAppend")) } vodAppendJob?.cancel() vodAppendJob = launch { - appendVodSourceInBackground( - mediaType = mediaType, - imdbId = currentImdbId, - seasonNumber = seasonNumber, - episodeNumber = episodeNumber, - timeoutMs = 15_000L - ) + runCatching { + appendVodSourceInBackground( + mediaType = mediaType, + imdbId = currentImdbId, + seasonNumber = seasonNumber, + episodeNumber = episodeNumber, + timeoutMs = 15_000L + ) + }.onFailure(childFailed("vodAppend")) } // Fetch metadata in background - launch { fetchMediaMetadata(mediaType, mediaId) } + launch { + runCatching { fetchMediaMetadata(mediaType, mediaId) } + .onFailure(childFailed("metadata")) + } // Fetch skip intervals in background (needs IMDB id) launch { - if (mediaType == MediaType.TV && seasonNumber != null && episodeNumber != null) { - val cachedImdbId = currentImdbId ?: mediaRepository.getCachedImdbId(mediaType, mediaId) - val imdbId = cachedImdbId ?: resolveExternalIds(mediaType, mediaId).imdbId - if (!imdbId.isNullOrBlank()) { - currentImdbId = imdbId - if (cachedImdbId == null) mediaRepository.cacheImdbId(mediaType, mediaId, imdbId) - fetchSkipIntervals(imdbId, seasonNumber, episodeNumber) + runCatching { + if (mediaType == MediaType.TV && seasonNumber != null && episodeNumber != null) { + val cachedImdbId = currentImdbId ?: mediaRepository.getCachedImdbId(mediaType, mediaId) + val imdbId = cachedImdbId ?: resolveExternalIds(mediaType, mediaId).imdbId + if (!imdbId.isNullOrBlank()) { + currentImdbId = imdbId + if (cachedImdbId == null) mediaRepository.cacheImdbId(mediaType, mediaId, imdbId) + fetchSkipIntervals(imdbId, seasonNumber, episodeNumber) + } } - } + }.onFailure(childFailed("skipIntervals")) } // Direct-URL playback must still fetch subtitle addons (e.g. OpenSubtitles). - launch { + // On viewModelScope (NOT a child of this load job): sibling children above + // (VOD/home-server/stream appends) occasionally throw, which cancels the whole + // load job and — because this block is registered last — killed the subtitle + // flow before its first line ran: no fetch, no selection, playback with + // subtitles silently Off. loadMedia cancels subtitleRefreshJob, so lifecycle + // stays correct across videos. + subtitleRefreshJob?.cancel() + subtitleRefreshJob = viewModelScope.launch { _uiState.value = _uiState.value.copy(isLoadingSubtitles = true) val cachedImdbId = currentImdbId ?: mediaRepository.getCachedImdbId(mediaType, mediaId) - val imdbId = cachedImdbId ?: resolveExternalIds(mediaType, mediaId).imdbId + val imdbId = cachedImdbId + ?: runCatching { resolveExternalIds(mediaType, mediaId).imdbId }.getOrNull() if (!imdbId.isNullOrBlank()) { currentImdbId = imdbId @@ -652,7 +707,15 @@ class PlayerViewModel @Inject constructor( scheduleSubtitleSelection(currentOriginalLanguage) } else { + // No IMDb id → addon subs can't be fetched, but the selection flow must + // still run: embedded tracks and the AI option exist regardless. Silently + // stopping here left playback with NOTHING selected ("Off") and no scan. + android.util.Log.w( + "SubMatch", + "subtitle fetch skipped: no imdbId for $mediaType/$mediaId — scheduling selection with embedded-only" + ) _uiState.value = _uiState.value.copy(isLoadingSubtitles = false) + scheduleSubtitleSelection(currentOriginalLanguage) } } return@launch @@ -1217,7 +1280,13 @@ class PlayerViewModel @Inject constructor( } private fun scheduleSubtitleSelection(fallbackLanguage: String?) { - if (hasManualSubtitleSelection) return + if (hasManualSubtitleSelection) { + android.util.Log.i( + "SubMatch", + "scheduleSubtitleSelection blocked: manual=true selected=${_uiState.value.selectedSubtitle?.label ?: "OFF"} scanning=${_uiState.value.isFindingBestMatch}" + ) + return + } val currentSel = _uiState.value.selectedSubtitle subtitleSelectionJob?.cancel() subtitleSelectionJob = viewModelScope.launch { @@ -1275,7 +1344,13 @@ class PlayerViewModel @Inject constructor( } } - if (hasManualSubtitleSelection) return + if (hasManualSubtitleSelection) { + android.util.Log.i( + "SubMatch", + "applyPref blocked: manual=true selected=${_uiState.value.selectedSubtitle?.label ?: "OFF"} scanning=${_uiState.value.isFindingBestMatch} subs=${subtitles.size}" + ) + return + } if (isSubtitleDisabledPreference(preference)) { _uiState.value = _uiState.value.copy(selectedSubtitle = null) return @@ -2504,6 +2579,7 @@ class PlayerViewModel @Inject constructor( private fun cancelFindBestMatch() { findMatchJob?.cancel() collectingReference = false + matchPreviousTexts = null if (_uiState.value.isFindingBestMatch || _uiState.value.isLiveAudioTranslating) { geminiLiveService.disconnect() _uiState.value = _uiState.value.copy( @@ -2575,8 +2651,27 @@ class PlayerViewModel @Inject constructor( } } + /** + * Scan-failure fallback: activate AI translation when the feature can run (master toggle on, + * key present, source available). Deliberately NOT gated on auto-select — that setting only + * governs unprompted activation at playback start; a failed scan explicitly asked for the + * best available subtitle, and AI timing (from the built-in track) beats an unverified addon + * pick. Returns true when AI took over. + */ + private fun tryAiFallbackAfterScan(): Boolean { + if (!aiSubtitleEnabled || aiApiKey.isBlank()) return false + if (findAiSourceSubtitle(_uiState.value.subtitles) == null) return false + activateAiTranslation() + showMatchToast("No well-synced subtitle found — using AI translation") + return true + } + /** Existing behavior: translate the built-in/source subtitle to the target language. */ fun activateAiTranslation() { + // Defense-in-depth: every caller is already gated on the AI master toggle (menu entry via + // isAiAvailable, interim via auto-select, fallback via tryAiFallbackAfterScan), but AI + // must never activate — and never spend API requests — when the feature is off. + if (!aiSubtitleEnabled) return // Re-resolve the source on every activation: an early activation (before embedded tracks // resolve) may have cached an external fallback source, and translation inherits the // source's timing — keep the cached one only when nothing better is available now. @@ -2716,8 +2811,14 @@ class PlayerViewModel @Inject constructor( .sortedByDescending { weightedSubtitleScore(streamSrc, it.id) } .take(MATCH_MAX_CANDIDATES) if (candidates.isEmpty()) { + android.util.Log.i( + "SubMatch", + "scan end: no candidates (subs=${subs.size} target=$targetLang) — AI fallback? ai=$aiSubtitleEnabled keyLen=${aiApiKey.length}" + ) endMatch() - noMatch(null) + // No candidates at all in the target language — AI translation is the only + // possible target-language subtitle; fall back to it when available. + if (!_uiState.value.isAiTranslating && !tryAiFallbackAfterScan()) noMatch(null) return@launch } @@ -2786,6 +2887,9 @@ class PlayerViewModel @Inject constructor( // in the match cache. fun selectLastResort() { if (_uiState.value.isAiTranslating) return + // AI translation first (when the feature can run) — verified timing beats any + // unverified addon pick, independent of the auto-select setting. + if (tryAiFallbackAfterScan()) return val current = _uiState.value.selectedSubtitle if (current != null && normalizeLanguage(current.lang) == targetLang) return candidates.firstOrNull()?.let { @@ -2836,6 +2940,10 @@ class PlayerViewModel @Inject constructor( } } + /** Whitespace/tag-insensitive form for comparing renderer cue text against parsed file text. */ + private fun normalizeCueTextForCompare(text: String): String = + text.replace(Regex("<[^>]*>"), " ").replace(Regex("\\s+"), " ").trim() + /** Reference = a built-in track's cue timing (any language). Returns null if too little dialogue. */ private suspend fun scoreAgainstBuiltIn( loaded: List>>, @@ -2851,8 +2959,15 @@ class PlayerViewModel @Inject constructor( val aiAlreadyShowingSource = _uiState.value.isAiTranslating && aiSourceSubtitle?.id == referenceSub.id if (!aiAlreadyShowingSource) { hasManualSubtitleSelection = true + // Re-resolve the reference's track indices from the CURRENT list: a MediaItem rebuild + // (e.g. the previous scan's pick) reassigns embedded group indices, and a stale + // override target silently keeps the wrong track rendering. + val refResolved = _uiState.value.subtitles.firstOrNull { + it.isEmbedded && it.id == referenceSub.id && + it.groupIndex != null && it.trackIndex != null + } ?: referenceSub _uiState.value = _uiState.value.copy( - selectedSubtitle = referenceSub, + selectedSubtitle = refResolved, isAiTranslating = false, subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 ) @@ -2870,16 +2985,21 @@ class PlayerViewModel @Inject constructor( // rendering in real time. Stops as soon as enough evidence accumulates. val candMin = loaded.minOf { it.second.firstOrNull()?.startMs ?: Long.MAX_VALUE } val candMax = loaded.maxOf { it.second.lastOrNull()?.endMs ?: 0L } - // Cues of the candidate that was on screen when the scan started — used to detect stale - // buffer reads that still mirror that track instead of the reference. - val previousCues = previousSubtitle?.let { prev -> + // Normalized cue texts of the candidate that was on screen when the scan started — used + // to detect stale buffer reads that still mirror that track instead of the reference. + val previousTexts: Set? = previousSubtitle?.let { prev -> loaded.firstOrNull { it.first.id == prev.id }?.second + ?.mapTo(HashSet()) { normalizeCueTextForCompare(it.text) } } + matchPreviousTexts = previousTexts collectingReference = true var refs: List> = emptyList() var bufferedCount = 0 var lastRefCount = 0 var lastProgressElapsed = 0L + var lastDiagRaw = -1 + var lastDiagKept = -1 + var lastDiagRealtime = -1 val startedAt = System.currentTimeMillis() while (true) { // Sanity-filter buffered timestamps: they must land within the candidate subtitles' @@ -2887,18 +3007,23 @@ class PlayerViewModel @Inject constructor( val bufferedRaw = runCatching { bufferedReferenceIntervalsProvider?.invoke(MATCH_BUFFERED_REF_INTERVALS) }.getOrNull().orEmpty().filter { it.first in candMin..candMax } - // Self-match guard: if most buffered intervals coincide (to the millisecond range) - // with the previously-displayed candidate's own cues, the buffer still holds that - // track, not the reference — discard the read and rely on realtime this tick. - val selfMatches = if (previousCues == null) 0 else bufferedRaw.count { b -> - previousCues.any { c -> - kotlin.math.abs(c.startMs - b.first) <= 60 && kotlin.math.abs(c.endMs - b.second) <= 60 - } - } - val buffered = if (bufferedRaw.isNotEmpty() && selfMatches * 2 >= bufferedRaw.size) { - emptyList() - } else { - bufferedRaw + // Self-match guard: verify by TEXT which track the buffer belongs to. If the buffered + // cue texts match the previously-displayed candidate's own lines, the reference track + // switch hasn't landed and we'd be scoring the candidate against itself. Timing-based + // detection is unusable here: subs cut from the same master share cue timings across + // languages (a constant-delta check froze scans by discarding legit references). + var diagSampled = 0 + var diagHits = 0 + val buffered = run { + if (previousTexts.isNullOrEmpty() || bufferedRaw.isEmpty()) return@run bufferedRaw + val sampled = runCatching { bufferedCueTextsProvider?.invoke(8) } + .getOrNull().orEmpty() + .map { normalizeCueTextForCompare(it) } + .filter { it.isNotEmpty() } + if (sampled.isEmpty()) return@run bufferedRaw + diagSampled = sampled.size + diagHits = sampled.count { it in previousTexts } + if (diagHits * 2 >= diagSampled) emptyList() else bufferedRaw } val realtime = synchronized(referenceIntervals) { referenceIntervals.toList() } // Realtime intervals that overlap a buffered one describe the same cue — drop them. @@ -2907,6 +3032,20 @@ class PlayerViewModel @Inject constructor( if (buffered.none { b -> r.first < b.second && b.first < r.second }) merged.add(r) } refs = merged.sortedBy { it.first } + // Diagnostics: log whenever the tick's composition changes — pinpoints whether slow + // scans starve at extraction (raw=0), at the guard (raw>0 kept=0), or at realtime. + if (bufferedRaw.size != lastDiagRaw || buffered.size != lastDiagKept || + realtime.size != lastDiagRealtime + ) { + lastDiagRaw = bufferedRaw.size + lastDiagKept = buffered.size + lastDiagRealtime = realtime.size + android.util.Log.i( + "SubMatch", + "tick raw=${bufferedRaw.size} kept=${buffered.size} realtime=${realtime.size} " + + "guardSampled=$diagSampled guardHits=$diagHits refs=${refs.size}" + ) + } bufferedCount = buffered.size val elapsed = System.currentTimeMillis() - startedAt @@ -2949,11 +3088,12 @@ class PlayerViewModel @Inject constructor( if (elapsed >= deadline) break updateMatchStatus( if (refs.isEmpty()) "Searching for a match ($sourceLabel) — waiting for speech…" - else "Searching for a match ($sourceLabel) — comparing subtitles… (${refs.size})" + else "Searching for a match ($sourceLabel) — (${refs.size})" ) delay(300) } collectingReference = false + matchPreviousTexts = null // Inconclusive after the hard cap (too few refs, or all clustered in one moment) — a pick // would be a guess; return null so the caller stays on the AI-translation fallback. // Floor matches the fast-accept span so a fast-accepted result isn't discarded here. @@ -2967,10 +3107,15 @@ class PlayerViewModel @Inject constructor( } loaded.firstOrNull()?.let { (_, cues) -> + // Show the candidate cues NEAREST the first reference window — file-start cues say + // nothing about alignment at the scan position. + val firstRef = refs.firstOrNull()?.first ?: 0L + val nearIdx = cues.indexOfFirst { it.endMs >= firstRef }.coerceAtLeast(0) + val near = cues.subList(nearIdx, minOf(nearIdx + 3, cues.size)) android.util.Log.i( "SubMatch", "align src=$srcLabel buffered=$bufferedCount total=${refs.size} " + - "refs=${refs.take(3)} candCues=${cues.take(3).map { it.startMs to it.endMs }} " + + "refs=${refs.take(3)} candNear=${near.map { it.startMs to it.endMs }} " + "candRange=${cues.firstOrNull()?.startMs}..${cues.lastOrNull()?.endMs}" ) } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt index 933bde803..6a32a1401 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt @@ -4750,7 +4750,7 @@ private fun TvGeneralSettingsRows( onQualityFiltersClick: () -> Unit = {}, subtitleAiEnabled: Boolean = false, subtitleAiAutoSelect: Boolean = false, - subtitleAiFindBestMatch: Boolean = true, + subtitleAiFindBestMatch: Boolean = false, subtitleAiApiKey: String = "", subtitleAiModel: com.arflix.tv.ui.screens.player.SubtitleAiModel = com.arflix.tv.ui.screens.player.SubtitleAiModel.GROQ_LLAMA_70B, subtitleRemoveHearingImpaired: Boolean = true, @@ -4939,7 +4939,7 @@ private fun GeneralSettings( onQualityFiltersClick: () -> Unit = {}, subtitleAiEnabled: Boolean = false, subtitleAiAutoSelect: Boolean = false, - subtitleAiFindBestMatch: Boolean = true, + subtitleAiFindBestMatch: Boolean = false, subtitleAiApiKey: String = "", subtitleAiModel: com.arflix.tv.ui.screens.player.SubtitleAiModel = com.arflix.tv.ui.screens.player.SubtitleAiModel.GROQ_LLAMA_70B, subtitleRemoveHearingImpaired: Boolean = true, diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt index 2d1c7e5b3..d6dab3872 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt @@ -197,7 +197,7 @@ data class SettingsUiState( // AI Subtitles val subtitleAiEnabled: Boolean = false, val subtitleAiAutoSelect: Boolean = false, - val subtitleAiFindBestMatch: Boolean = true, + val subtitleAiFindBestMatch: Boolean = false, val subtitleAiApiKey: String = "", val subtitleAiModel: SubtitleAiModel = SubtitleAiModel.GROQ_LLAMA_70B, val subtitleRemoveHearingImpaired: Boolean = true, @@ -491,7 +491,7 @@ class SettingsViewModel @Inject constructor( val subtitleAiEnabled = prefs[subtitleAiEnabledKey] ?: false val subtitleAiAutoSelect = prefs[subtitleAiAutoSelectKey] ?: false - val subtitleAiFindBestMatch = prefs[subtitleAiFindBestMatchKey] ?: true + val subtitleAiFindBestMatch = prefs[subtitleAiFindBestMatchKey] ?: false val subtitleAiApiKey = prefs[subtitleAiApiKeyKey] ?: "" val subtitleAiModel = runCatching { SubtitleAiModel.valueOf(prefs[subtitleAiModelKey] ?: SubtitleAiModel.GROQ_LLAMA_70B.name) diff --git a/docs/subtitle-auto-match.md b/docs/subtitle-auto-match.md new file mode 100644 index 000000000..2493cb7b9 --- /dev/null +++ b/docs/subtitle-auto-match.md @@ -0,0 +1,229 @@ +# Subtitle Auto-Match ("Find Best Match") — Design & Behavior Reference + +Complete reference for the subtitle auto-matching feature and its interaction with AI subtitle +translation. Written to be self-contained: hand this file to a fresh session/developer for full +context. + +**Code map** + +| Concern | File | +|---|---| +| Scan orchestration, selection flows, caches, settings gating | `app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt` (`findBestSubtitleMatch`, `runFindBestMatch`, `activateAiSubtitle`, `applyPreferredSubtitle`, `MATCH_*` constants) | +| Cue download/parse + timing/word scoring | `.../player/SubtitleSyncMatcher.kt` | +| Subtitle menu, selection application, media item rebuild, startup watchdog | `.../player/PlayerScreen.kt` | +| AI batch translation (Groq/Gemini generateContent) | `.../player/SubtitleTranslationManager.kt`, `SubtitleTranslationService.kt` | +| Live hearing (Gemini Live WS, audio → target-language text) | `.../player/GeminiLiveTranslationService.kt`, `AudioCaptureProcessor.kt` | +| Renderer hooks (buffered-cue reflection, pre-translation, cue offset) | `.../player/AiSubtitleRenderersFactory.kt` | +| Settings UI (toggle lives in **Subtitles** section, not AI) | `.../settings/SettingsScreen.kt` (TV row id **38**), `SettingsViewModel.kt` | +| Cloud backup/restore of settings | `.../data/repository/CloudSyncRepository.kt` | + +--- + +## 1. Settings and what they control + +| Setting | DataStore key | Default | Controls | +|---|---|---|---| +| Preferred subtitle language | `default_subtitle` (profile-scoped) | — | The **target language** of matching and AI translation (`targetSubtitleLangCode`, `matchLanguageName`). | +| Auto Find Best Subtitle Match | `subtitle_ai_find_best_match` | `false` | Auto-runs the scan on playback. **AI-independent** (works with no API key). Off by default — users opt in. Key name kept for backward compat. | +| AI subtitle translation (master) | `subtitle_ai_enabled` | `false` | Enables AI features: translation option in menu, AI interim during scans, hearing fallback. | +| Auto-Select AI Translation | `subtitle_ai_auto_select` | `false` | Allows AI translation to activate **automatically** (incl. as the scan's on-screen interim). | +| Model | `subtitle_ai_model` | `GROQ_LLAMA_70B` | Groq or Gemini for batch translation. **Hearing requires Gemini.** | +| API key | `subtitle_ai_api_key` (global) | — | One key used for both batch translation and Gemini Live. | + +All of the above (except profile-scoped language) sync via `CloudSyncRepository` +(`subtitleAiFindBestMatch` field in the backup JSON). + +### Behavior matrix (all combinations) + +| AI master | Auto-select AI | Find-match | On playback | +|---|---|---|---| +| any | any | **off** | Classic flow only: best release-name-scored sub in preferred language. Manual "Find Best Match" menu entry still available. | +| off | — | on | Classic pick shows immediately → **silent background scan** (subtitles hidden during it) → verified match replaces pick, else pick stays. Zero AI/API usage. | +| on | off | on | Same silent scan (user opted out of auto-AI). AI translate remains a manual menu option. | +| on | on | off | AI translation activates directly (no scan). | +| on | on | on | Scan with **AI translation as on-screen interim**; if no match, AI stays (source re-resolved to prefer embedded). | + +Menu entries (player → subtitle menu, inside the preferred-language group): +- **"Find Best Match"** (badge `Auto`) — always present when a preferred language is set (`matchLanguageName`), AI-independent. +- **"" AI translate** (badge `AI`) — only when AI available (`isAiAvailable`). +- D-pad index math: headers = `(match? 1 : 0) + (ai? 1 : 0)`, subs follow. + +--- + +## 2. The scan pipeline + +```mermaid +flowchart TD + A[Scan requested] --> B{Trigger} + B -->|auto: applyPreferredSubtitle, once per stream
latch: autoMatchAttempted| C + B -->|manual: menu entry
clears remembered cache entry, useCache=false| C + C[Wait for sources
continuous-playback clock: embedded grace 8s,
candidates wait 15s, absolute 90s] --> D{Embedded track in
preferred language?} + D -->|yes| E[Select it — muxed = guaranteed sync
toast: Matched embedded] + D -->|no| F[Candidates: external subs in pref language,
sorted by release-name score, capped at 10] + F -->|none| G[toast: no well-synced subtitle] + F --> H{Remembered match for this exact stream?
only when useCache} + H -->|hit| I[Re-download text → local file → select
toast: remembered] + H -->|miss| J[Download all candidates in parallel
keep raw text + parsed cues] + J --> K{Reference source?} + K -->|any embedded non-forced non-bitmap track,
English preferred| L[Timing scan] + K -->|none + AI on + key + Gemini| M[Hearing scan:
Gemini Live audio→text, word overlap,
threshold 0.30] + K -->|none, otherwise| N[Inconclusive] + L --> O{Best score ≥ 0.70?} + M --> O2{Best ≥ 0.30?} + O -->|yes| P[Select via LOCAL FILE copy
+ write per-stream cache
toast: Matched · N%] + O -->|no| Q[Fallback ladder] + O2 -->|yes| P + O2 -->|no| Q + N --> Q + Q --> R{AI translating on screen?} + R -->|yes| S[Keep AI, re-resolve source
toast: keeping AI translation + best N%] + R -->|no| R2{AI feature available?
master on + key + source
NOT gated on auto-select} + R2 -->|yes| S2[Activate AI translation
toast: using AI translation] + R2 -->|no| T{Current selection already
in preferred language?} + T -->|yes| U[Keep it
toast: best N%] + T -->|no| V[Select top release-name candidate
toast: sync unverified] +``` + +Fallback rationale: AI timing comes from the built-in track (verified), so it beats any +unverified addon pick. Auto-select only governs *unprompted* activation at playback start; a +failed scan explicitly asked for the best available subtitle. Same ladder applies when zero +target-language candidates exist. + +### Timing scan (reference collection loop, 300ms ticks) + +- Merge **buffered upcoming cues** (reflection into the text renderer; typically only 2–12 + visible) with **realtime rendered-cue intervals** (`onPlayerCues`), deduped by overlap. + ⚠ media3 picks a cue resolver per the track's cue-replacement behavior. For REPLACE tracks + (`ReplacingCuesResolver`, e.g. MKV SubRip) the `CuesWithTiming.durationUs` is **C.TIME_UNSET** + (large negative) — each cue lasts until replaced. The interval converter must coerce + non-positive durations to a nominal ~2s, or every item is rejected (`end < start`) and the + buffered path silently returns 0 for that whole content category: scans crawl at realtime pace, + and realtime windows carry a systematic ~600ms render-lag skew that suppresses well-synced subs + to ≈ the 0.70 threshold. `bufDiag`/`resolverDump` log lines show extraction health; realtime + reference intervals ≠ authored cue times — buffered is the trustworthy source. +- Guards on the buffered read: candidate-time-range sanity filter; **self-match guard by TEXT** — + sample up to 8 buffered cue texts and drop the read if ≥half match the previously displayed + candidate's own (normalized) lines. Guard history, do not regress: + 1. exact-coincidence timing (±60ms both edges) — defeated by the reflection's uniform + stream-offset shift; a candidate scored 0.86 against itself; + 2. constant-delta timing (median ±80ms) — false-positived on subs cut from the same master + (Hebrew vs Czech share cue timings), discarding legit references and freezing scans at + realtime pace; + 3. **text comparison (current)** — language-discriminating, immune to both failure modes. + The track switch to the reference also propagates asynchronously: ~1.5s settle delay, target + indices re-resolved from current state, and the embedded override retries up to 10s with fresh + indices (one-shot appliers are fragile after MediaItem rebuilds), disabling text while invalid + instead of falling back to preferred-language (which silently kept the displayed sub rendering). +- Realtime intervals: capped at 20s each (longer = seek artifact), reset on backward seek. + +**Exit conditions** (first hit wins): + +| Condition | Values | +|---|---| +| Target reached | ≥ 8 intervals AND span ≥ 30s | +| Early accept | ≥ 4 intervals, span ≥ 30s, some candidate ≥ 0.8 | +| Fast accept | ≥ 5 intervals, span ≥ 15s, some candidate ≥ 0.85 | +| Give-up (no speech yet) | 4 min | +| Give-up (stagnation) | 90s since last new interval (clock **frozen while paused**) | +| Absolute ceiling | 10 min | + +Post-loop: `< 2` intervals or span `< 15s` ⇒ inconclusive (fall to ladder). + +### Scoring (`SubtitleSyncMatcher.scoreByTiming`) — HYBRID, calibrated + +Per reference window, averaged: +- Window ≤ 7s (a real cue): **best single-cue overlap** — this discriminates offsets; union + coverage here scored *everything* 0.9+ in dense dialogue (offset subs have cue chains touching + any short window). +- Window > 7s (realtime merges back-to-back cues): **union-of-cues coverage** — a synced sub + needs several cues to span it; single-cue unfairly caps at ~0.3. + +Calibration (user-verified, July 2026): synced subs score 0.71–0.98; a confirmed-offset sub +scored 0.65. Accept threshold **0.70** (timing) / **0.30** (hearing). The good/bad boundary is +narrow — do not nudge thresholds or the metric without fresh A/B evidence. + +--- + +## 3. Selection & the local-file trick + +Selecting an **external** subtitle rebuilds the MediaItem (re-prepare). Economics: +- Video re-open: debrid streams are deliberately **not disk-cached** (I/O bottleneck) — usually + fine, occasionally slow/stale. +- Subtitle download during rebuild was the common stall (slow addon proxies) ⇒ matched subs are + served from a **local cache file** (`cacheDir/matched_subs/`, ≤40 files): + the scan already downloaded the text; `localizeSubtitle()` writes UTF-8 (gunzipped) and selects + a `file://` copy (same id). The data source chain wraps OkHttp in `DefaultDataSource` for + file support. The tracks-changed remap keeps `file:` selections as-is (would otherwise swap + back to the remote URL and re-trigger a rebuild). +- **Do NOT preload all subs into the MediaItem** — tried historically and reverted: ExoPlayer + eagerly downloads every side-loaded config at prepare (30+ requests), plus encoding issues. + +Subtitle MIME sniffing strips trailing slashes (`…/sub.vtt/?q=` is VTT — AIOStreams shape); +default is SRT for extensionless URLs. + +## 4. Per-stream remembered cache + +- Key = stream identity, **not** title: `infoHash:fileIdx` → `videoHash` → `filename:size` → + URL hash. Same episode from a different source ⇒ different key ⇒ fresh scan (sync is a property + of the file). +- Value = candidate `provider|id` (not URL — addon URLs are ephemeral). Stored as JSON list in + `subtitle_match_cache_v1` (global DataStore), LRU 50. **Not cloud-synced** (deliberate). +- Written on verified match only (never for unverified fallback picks). +- Manual track pick in the target language **overwrites** the entry; picking anything else + clears it. Manual "Find Best Match" click clears it and scans fresh (`useCache=false`). + +## 5. State resets (bugs lived here) + +- `loadMedia`: resets all selection flags, `autoMatchAttempted`, AI source, **and the subtitles + list** — externals accumulate additively during one video; without clearing, a new video scans + the previous title's candidates (references vs wrong-show subs ⇒ uniform ~0.1 scores while the + correct fresh sub isn't scanned at all). +- Provided-URL loads with **no resolvable IMDb id** must still run `scheduleSubtitleSelection` + (embedded tracks + AI exist without addon subs). The silent else-branch here was one cause of + the "plays with subtitles Off, no scan, nothing" regression. +- The deeper cause: the subtitle fetch/selection block was the **last child coroutine of the + load job** — an uncaught exception in ANY sibling (VOD/home-server/stream appenders) cancels + all siblings, killing the subtitle flow before its first line. It now runs on `viewModelScope` + directly (as `subtitleRefreshJob`, cancelled by `loadMedia`), and each sibling is + failure-isolated with a `load child '' failed` log. Rule: anything subtitle-critical + must not share a Job with flaky background appenders. +- `selectStream` (mid-session source switch — does NOT pass through `loadMedia`): must reset + selection flags, purge **embedded** subtitle entries (per-file!), clear selection with a nonce + bump, cancel a running scan. Missing pieces here caused: wrong-track selection (stale indices), + "selected but not rendering" (no nonce bump), auto-flow never re-running (stale + `hasManualSubtitleSelection`). +- During the scan, subtitles are hidden (`subtitleView.visibility`) when + `isFindingBestMatch && !isAiTranslating` — display-only; cue collection listens on the player. +- Startup watchdog failover must reset `streamSelectedTime` optimistically before `continue`, + or it burns the whole source list in milliseconds (stale-clock burst). + +## 6. AI specifics + +- Batch translation pre-fetch (`triggerPreTranslation`/`preTranslateWindow`) is gated on + `translationManager.isEnabled` — without this it spends API requests whenever any track renders + (the 401-toast-with-AI-off bug). +- AI interim during scans: only when `aiSubtitleEnabled && aiSubtitleAutoSelect`. Source is + re-resolved on every activation (embedded preferred; external only if English) — a stale + external source caused mistimed translations. +- Hearing fallback: Gemini model + key + AI on; aborts on WS ERROR; same progress-anchored + timers as the timing scan. +- Gemini Live target language comes from the preference (`targetLanguageCode`), default `he`. + +## 7. Provider quirks (see also memory: project_aiostreams) + +- **AIOStreams**: subtitles endpoint slow (8–12s) / 502s cold → subtitle fetch is parallel, + 30s timeout, one retry. Its streams are exempt from quality sorting (`keepsOwnStreamOrder`) — + user pre-sorts server-side. Sub URLs end `…/sub.vtt/`. +- **OpenSubtitles v3+ proxy** (`opensubtitles.stremio.homes`): VTT with double `WEBVTT` header, + a branding banner cue at 0:01–0:06, recap lines as NOTE comments. Scanner's parser is tolerant; + keep an eye on ExoPlayer's stricter parser if display anomalies appear. + +## 8. Known-good log workflow + +Debug via `adb logcat` tag `SubMatch`: +- `reference source=…` — scan entered scoring (after source-wait). +- `align src=buffer|realtime|buffer+realtime buffered=N total=M refs=[…] candCues=[…]` — + reference vs first candidate sample; `src=buffer` + score **exactly 1.00** = self-match red flag. +- `[builtin] candidate … score=X` — per-candidate verdicts. +Toast decoder: "Matched · N%" = verified; "(remembered)" = cache hit; "(sync unverified)" = +fallback pick; "No well-synced … (best N%)" = rejection with evidence. From 38e6e894813e28e481806dcfc32b892bbc2cdf94 Mon Sep 17 00:00:00 2001 From: silentbil Date: Mon, 6 Jul 2026 21:54:00 +0300 Subject: [PATCH 10/11] remove logs --- .../player/AiSubtitleRenderersFactory.kt | 63 ++----------------- .../tv/ui/screens/player/PlayerViewModel.kt | 44 ++----------- docs/subtitle-auto-match.md | 15 +++-- 3 files changed, 19 insertions(+), 103 deletions(-) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt index ac8997fed..07a45b632 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt @@ -426,24 +426,18 @@ private class SubtitleOffsetRenderer( return texts.toList() } - private var bufDiagLogged = 0 - /** Buffered cue intervals (startMs, endMs) for text-carrying cues, from the modern resolver. */ fun extractBufferedIntervals(maxCount: Int): List> { val intervals = ArrayList>() // Cue buffer timestamps are on ExoPlayer's internal stream timeline, which adds a large // base offset; subtract the renderer's stream offset to get 0-based media time. val offsetMs = readStreamOffsetUs() / 1000L - var resolverFound = false - var fieldUsed: String? = null try { val resolverField = findField(baseRenderer.javaClass, "cuesResolver") val resolver = resolverField?.get(baseRenderer) - resolverFound = resolver != null if (resolver != null) { - // Media3 has multiple resolver impls chosen per track cue-replacement behavior: - // MergingCuesResolver ("cuesWithTimingList") and ReplacingCuesResolver - // ("cuesWithTimings") — probe both; content differs per rip (MKV SubRip vs VTT). + // Media3 has multiple resolver impls chosen per track cue-replacement behavior + // (MergingCuesResolver, ReplacingCuesResolver) — probe the known field names. for (candidate in listOf( "cuesWithTimingList", "cuesWithTimings", "cueGroupsByStartTime", "cueGroups", "cueGroupList", "groups" @@ -459,10 +453,7 @@ private class SubtitleOffsetRenderer( val (s, e) = item?.let(::intervalFromCueWrapper) ?: continue intervals.add((s - offsetMs) to (e - offsetMs)) } - if (intervals.isNotEmpty()) { - fieldUsed = candidate - break - } + if (intervals.isNotEmpty()) break } if (intervals.isEmpty()) { // Unknown impl/field name (new media3 version?) — scan every field for a @@ -482,10 +473,7 @@ private class SubtitleOffsetRenderer( val (s, e) = item?.let(::intervalFromCueWrapper) ?: continue intervals.add((s - offsetMs) to (e - offsetMs)) } - if (intervals.isNotEmpty()) { - fieldUsed = "fallback:${f.name}" - break@outer - } + if (intervals.isNotEmpty()) break@outer } catch (_: Exception) { } } @@ -495,48 +483,7 @@ private class SubtitleOffsetRenderer( } } catch (_: Exception) { } - val out = intervals.filter { it.first >= 0 }.distinct().sortedBy { it.first }.take(maxCount) - // Diagnostics: pinpoints whether slow scans starve at the reflection (resolver/field - // missing — e.g. R8), at the offset correction (raw > 0 but out = 0), or downstream. - if (bufDiagLogged < 10) { - bufDiagLogged++ - android.util.Log.i( - "SubMatch", - "bufDiag resolver=$resolverFound field=$fieldUsed raw=${intervals.size} out=${out.size} offsetMs=$offsetMs" - ) - // Extraction found nothing: dump the resolver's real shape once so the probe list can - // be corrected from evidence instead of guessed media3 internals. - if (resolverFound && intervals.isEmpty() && bufDiagLogged == 5) { - runCatching { - val resolver = findField(baseRenderer.javaClass, "cuesResolver")?.get(baseRenderer) - if (resolver != null) { - val desc = buildString { - append(resolver.javaClass.name).append(" {") - var cls: Class<*>? = resolver.javaClass - while (cls != null && cls != Any::class.java) { - for (f in cls.declaredFields) { - runCatching { - f.isAccessible = true - val v = f.get(resolver) - val size = when (v) { - is Collection<*> -> "size=${v.size}" - is Map<*, *> -> "size=${v.size}" - null -> "null" - else -> v.javaClass.simpleName - } - append(" ${f.name}:$size;") - } - } - cls = cls.superclass - } - append(" }") - } - android.util.Log.i("SubMatch", "bufDiag resolverDump $desc") - } - } - } - } - return out + return intervals.filter { it.first >= 0 }.distinct().sortedBy { it.first }.take(maxCount) } /** The renderer's stream offset (µs) — the base added to buffer sample timestamps. */ diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt index b6b58d5b3..e3667c971 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt @@ -1280,13 +1280,7 @@ class PlayerViewModel @Inject constructor( } private fun scheduleSubtitleSelection(fallbackLanguage: String?) { - if (hasManualSubtitleSelection) { - android.util.Log.i( - "SubMatch", - "scheduleSubtitleSelection blocked: manual=true selected=${_uiState.value.selectedSubtitle?.label ?: "OFF"} scanning=${_uiState.value.isFindingBestMatch}" - ) - return - } + if (hasManualSubtitleSelection) return val currentSel = _uiState.value.selectedSubtitle subtitleSelectionJob?.cancel() subtitleSelectionJob = viewModelScope.launch { @@ -1344,13 +1338,7 @@ class PlayerViewModel @Inject constructor( } } - if (hasManualSubtitleSelection) { - android.util.Log.i( - "SubMatch", - "applyPref blocked: manual=true selected=${_uiState.value.selectedSubtitle?.label ?: "OFF"} scanning=${_uiState.value.isFindingBestMatch} subs=${subtitles.size}" - ) - return - } + if (hasManualSubtitleSelection) return if (isSubtitleDisabledPreference(preference)) { _uiState.value = _uiState.value.copy(selectedSubtitle = null) return @@ -2811,10 +2799,6 @@ class PlayerViewModel @Inject constructor( .sortedByDescending { weightedSubtitleScore(streamSrc, it.id) } .take(MATCH_MAX_CANDIDATES) if (candidates.isEmpty()) { - android.util.Log.i( - "SubMatch", - "scan end: no candidates (subs=${subs.size} target=$targetLang) — AI fallback? ai=$aiSubtitleEnabled keyLen=${aiApiKey.length}" - ) endMatch() // No candidates at all in the target language — AI translation is the only // possible target-language subtitle; fall back to it when available. @@ -2997,9 +2981,6 @@ class PlayerViewModel @Inject constructor( var bufferedCount = 0 var lastRefCount = 0 var lastProgressElapsed = 0L - var lastDiagRaw = -1 - var lastDiagKept = -1 - var lastDiagRealtime = -1 val startedAt = System.currentTimeMillis() while (true) { // Sanity-filter buffered timestamps: they must land within the candidate subtitles' @@ -3012,8 +2993,6 @@ class PlayerViewModel @Inject constructor( // switch hasn't landed and we'd be scoring the candidate against itself. Timing-based // detection is unusable here: subs cut from the same master share cue timings across // languages (a constant-delta check froze scans by discarding legit references). - var diagSampled = 0 - var diagHits = 0 val buffered = run { if (previousTexts.isNullOrEmpty() || bufferedRaw.isEmpty()) return@run bufferedRaw val sampled = runCatching { bufferedCueTextsProvider?.invoke(8) } @@ -3021,9 +3000,8 @@ class PlayerViewModel @Inject constructor( .map { normalizeCueTextForCompare(it) } .filter { it.isNotEmpty() } if (sampled.isEmpty()) return@run bufferedRaw - diagSampled = sampled.size - diagHits = sampled.count { it in previousTexts } - if (diagHits * 2 >= diagSampled) emptyList() else bufferedRaw + val hits = sampled.count { it in previousTexts } + if (hits * 2 >= sampled.size) emptyList() else bufferedRaw } val realtime = synchronized(referenceIntervals) { referenceIntervals.toList() } // Realtime intervals that overlap a buffered one describe the same cue — drop them. @@ -3032,20 +3010,6 @@ class PlayerViewModel @Inject constructor( if (buffered.none { b -> r.first < b.second && b.first < r.second }) merged.add(r) } refs = merged.sortedBy { it.first } - // Diagnostics: log whenever the tick's composition changes — pinpoints whether slow - // scans starve at extraction (raw=0), at the guard (raw>0 kept=0), or at realtime. - if (bufferedRaw.size != lastDiagRaw || buffered.size != lastDiagKept || - realtime.size != lastDiagRealtime - ) { - lastDiagRaw = bufferedRaw.size - lastDiagKept = buffered.size - lastDiagRealtime = realtime.size - android.util.Log.i( - "SubMatch", - "tick raw=${bufferedRaw.size} kept=${buffered.size} realtime=${realtime.size} " + - "guardSampled=$diagSampled guardHits=$diagHits refs=${refs.size}" - ) - } bufferedCount = buffered.size val elapsed = System.currentTimeMillis() - startedAt diff --git a/docs/subtitle-auto-match.md b/docs/subtitle-auto-match.md index 2493cb7b9..542a14abd 100644 --- a/docs/subtitle-auto-match.md +++ b/docs/subtitle-auto-match.md @@ -99,8 +99,9 @@ target-language candidates exist. non-positive durations to a nominal ~2s, or every item is rejected (`end < start`) and the buffered path silently returns 0 for that whole content category: scans crawl at realtime pace, and realtime windows carry a systematic ~600ms render-lag skew that suppresses well-synced subs - to ≈ the 0.70 threshold. `bufDiag`/`resolverDump` log lines show extraction health; realtime - reference intervals ≠ authored cue times — buffered is the trustworthy source. + to ≈ the 0.70 threshold. Realtime reference intervals ≠ authored cue times — buffered is the + trustworthy source. (When debugging extraction, temporarily log resolver class/field/item + counts inside `extractBufferedIntervals` — that's how both resolver bugs were found.) - Guards on the buffered read: candidate-time-range sanity filter; **self-match guard by TEXT** — sample up to 8 buffered cue texts and drop the read if ≥half match the previously displayed candidate's own (normalized) lines. Guard history, do not regress: @@ -222,8 +223,12 @@ default is SRT for extensionless URLs. Debug via `adb logcat` tag `SubMatch`: - `reference source=…` — scan entered scoring (after source-wait). -- `align src=buffer|realtime|buffer+realtime buffered=N total=M refs=[…] candCues=[…]` — - reference vs first candidate sample; `src=buffer` + score **exactly 1.00** = self-match red flag. -- `[builtin] candidate … score=X` — per-candidate verdicts. +- `align src=buffer|realtime|buffer+realtime buffered=N total=M refs=[…] candNear=[…]` — + reference intervals vs the first candidate's cues nearest the scan window; `src=buffer` + + score **exactly 1.00** = self-match red flag; `buffered=0` = extraction dead (see §2 warning). +- `[builtin] candidate … score=X` / `[hearing] candidate …` — per-candidate verdicts. +- Rare warnings that flag real faults: `realtime self-cue interval dropped` (reference track + switch didn't land), `subtitle fetch skipped: no imdbId`, `load child '' failed`, + `hearing aborted`, `match cache write failed`. Toast decoder: "Matched · N%" = verified; "(remembered)" = cache hit; "(sync unverified)" = fallback pick; "No well-synced … (best N%)" = rejection with evidence. From 3b5dfde3befbdaad2b914c49e9048b768bb7b2b3 Mon Sep 17 00:00:00 2001 From: silentbil Date: Thu, 9 Jul 2026 15:29:05 +0300 Subject: [PATCH 11/11] fix --- .../kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt index e3667c971..2a819cb8c 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt @@ -434,6 +434,10 @@ class PlayerViewModel @Inject constructor( lastWatchHistorySavedPositionSeconds = -1L subtitleRefreshJob?.cancel() subtitleSelectionJob?.cancel() + // Cancel any in-flight "Find best match" scan from the previous title — otherwise it can + // finish on this new session and select/restore a stale subtitle or poison the match cache + // (its stream cache key resolves to the new stream). Same-VM path only, e.g. play-next. + cancelFindBestMatch() vodAppendJob?.cancel() homeServerAppendJob?.cancel() streamPrewarmJob?.cancel()