feat(neuro): key spec 3, fold Hant to Hans and merge Han runs

.github/nightly-notes/2026-10-20-neuro-key-spec-3.md

@@ -0,0 +1,2 @@
+## Improved
+- Recommendations read Traditional and Simplified Chinese titles as the same words (#1341)

app/build.gradle.kts

@@ -404,6 +404,8 @@ dependencies {
     testImplementation(libs.kotlinx.coroutines.test)
     testImplementation(libs.mockk)
     testImplementation(libs.truth)
+    // Han word breaker for tests; the JDK breaker does not split Han runs.
+    testImplementation(libs.icu4j)
     testImplementation(libs.turbine)
     testImplementation(libs.androidx.glance.appwidget.testing)
     testImplementation(libs.hilt.android.testing)

app/src/main/assets/licenses/notices.txt

@@ -63,3 +63,8 @@ Used, trimmed to the most frequent words of each language, in resources/neuro/fr
 Licensed under the Creative Commons Attribution-ShareAlike 4.0 International
 License (https://creativecommons.org/licenses/by-sa/4.0/). The trimmed lists are
 shared under the same license.
+
+Open Chinese Convert (github.com/BYVoid/OpenCC)
+The Traditional to Simplified character table in resources/neuro/fold is made from
+OpenCC's TSCharacters.txt. Copyright (c) the OpenCC authors. Licensed under the
+Apache License, Version 2.0 (https://www.apache.org/licenses/LICENSE-2.0).

app/src/main/java/io/github/aedev/flow/data/recommendation/FlowNeuroEngine.kt

@@ -509,6 +509,8 @@ class FlowNeuroEngine internal constructor(
                     if (maintained !== currentUserBrain) {
                         if (currentUserBrain.schemaVersion < NeuroMaintenance.KEY_SPEC_2_SCHEMA_VERSION) {
                             storage.keepCopyBeforeMigration(currentUserBrain, NeuroMaintenance.KEY_SPEC_2_SCHEMA_VERSION)
+                        } else if (currentUserBrain.schemaVersion < NeuroMaintenance.KEY_SPEC_3_SCHEMA_VERSION) {
+                            storage.keepCopyBeforeMigration(currentUserBrain, NeuroMaintenance.KEY_SPEC_3_SCHEMA_VERSION)
                         }
                         currentUserBrain = maintained
                         storage.save(deepFlowBookkeeping.persistable(currentUserBrain))
@@ -925,7 +927,7 @@ class FlowNeuroEngine internal constructor(
     suspend fun getBlockedTopics(): Set<String> = withBrainLock { currentUserBrain.blockedTopics }
 
     suspend fun addBlockedTopic(topic: String) {
-        val normalized = NeuroText.fold(topic).trim()
+        val normalized = tokenizer.fold(topic).trim()
         if (normalized.isBlank()) return
         withBrainLock {
             val lemma = tokenizer.keyOf(normalized)

app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroChunker.kt

@@ -15,6 +15,7 @@ internal class NeuroChunker(
     private val stopWordsFor: (pieces: List<String>) -> (String) -> Boolean,
     private val normalizeLemma: (String) -> String,
     private val keyOf: (String) -> String,
+    private val breaker: WordBreaker = PlatformWordBreaker,
 ) {
     /** A run of [words] inside one chunk, from word [from] up to but not including [to]. */
     data class Run(
@@ -28,6 +29,10 @@ internal class NeuroChunker(
         val chunks: List<List<String>>,
         val modelPairs: List<String>,
         val formats: Set<String> = emptySet(),
+        /** Merged Han run candidates; promoted by [NeuroPhrases.isMergedRun]. */
+        val merged: List<String> = emptyList(),
+        /** Single Han characters outside the break set, counted for [NeuroPhrases.isMergedRun]. */
+        val singles: Set<String> = emptySet(),
     ) {
         /** Every word in reading order, [NeuroKeySpec.NEVER_ALONE] words included. */
         val words: List<String> get() = chunks.flatten()
@@ -66,9 +71,15 @@ internal class NeuroChunker(
         data object Break : Item
     }
 
-    fun chunk(text: String): Chunks {
+    /** [han]: the Hant fold and merged Han runs ([NeuroKeySpec.HAN_SPEC]); [HanFold.NONE] is spec 2. */
+    fun chunk(
+        text: String,
+        han: HanFold,
+    ): Chunks {
         val items = ArrayList<Item>()
-        val pieces = NeuroText.words(NeuroText.fold(text))
+        val folded = NeuroText.fold(text, han)
+        val spans = NeuroText.pieces(folded, breaker)
+        val pieces = spans.map { it.text }
         val isStopWord = stopWordsFor(pieces)
         pieces.forEach { piece -> split(piece, items) }
         val merged = mergeExpressions(items)
@@ -103,7 +114,75 @@ internal class NeuroChunker(
             }
         }
         close()
-        return Chunks(chunks, modelPairs, formats)
+        val runs = if (han != HanFold.NONE) mergedRuns(folded, spans, isStopWord, han) else MergedRuns.None
+        return Chunks(chunks, modelPairs, formats + runs.formats, runs.keys, runs.singles)
+    }
+
+    private class MergedRuns(
+        val keys: List<String>,
+        val singles: Set<String>,
+        val formats: Set<String>,
+    ) {
+        companion object {
+            val None = MergedRuns(emptyList(), emptySet(), emptySet())
+        }
+    }
+
+    // Stage 6: merge single Han characters (plus an adjacent word) into candidates, split at runBreaks.
+    // Format words inside a run are reported as formats.
+    private fun mergedRuns(
+        folded: String,
+        spans: List<NeuroText.Piece>,
+        isStopWord: (String) -> Boolean,
+        han: HanFold,
+    ): MergedRuns {
+        if (folded.none(NeuroText::isHan)) return MergedRuns.None
+        val language = if (han == HanFold.UNLESS_KANA && folded.any(NeuroText::isKana)) "ja" else "zh"
+        val breaks = NeuroLanguages.runBreaks(language) ?: return MergedRuns.None
+        val keys = LinkedHashSet<String>()
+        val singles = LinkedHashSet<String>()
+        val formats = LinkedHashSet<String>()
+        var group = ArrayList<String>()
+
+        fun valid(candidate: String): Boolean =
+            candidate.length in 2..MAX_MERGED_CHARS &&
+                (2..candidate.length).none { length ->
+                    (0..candidate.length - length).any { start ->
+                        val part = candidate.substring(start, start + length)
+                        isStopWord(part) || NeuroFormats.formatOf(part) != null
+                    }
+                }
+
+        fun flush() {
+            var i = 0
+            while (i < group.size) {
+                if (group[i].length != 1) {
+                    i++
+                    continue
+                }
+                var end = i
+                while (end < group.size && group[end].length == 1) end++
+                val run = group.subList(i, end).joinToString("")
+                val prefix = group.getOrNull(i - 1)?.takeIf { it.length >= 2 }.orEmpty()
+                val suffix = if (run.length == 1) group.getOrNull(end)?.takeIf { it.length >= 2 }.orEmpty() else ""
+                singles += group.subList(i, end)
+                for (length in 2..run.length) {
+                    for (start in 0..run.length - length) NeuroFormats.formatOf(run.substring(start, start + length))?.let { formats += it }
+                }
+                if (valid(run)) keys += run
+                if (prefix.isNotEmpty() && valid(prefix + run)) keys += prefix + run
+                if (suffix.isNotEmpty() && valid(run + suffix)) keys += run + suffix
+                i = end
+            }
+            group = ArrayList()
+        }
+        spans.forEach { span ->
+            val han = span.text.all(NeuroText::isHan)
+            if (!han || !span.joined || (span.text.length == 1 && span.text[0] in breaks)) flush()
+            if (han && !(span.text.length == 1 && span.text[0] in breaks)) group += span.text
+        }
+        flush()
+        return MergedRuns(keys.toList(), singles, formats)
     }
 
     // Stage 2 and 4: one whitespace-separated piece into words and breaks.
@@ -262,6 +341,9 @@ internal class NeuroChunker(
         val KEEP_SUFFIX = setOf('+', '#')
         val YEAR = Regex("^(19|20)\\d\\d$")
 
+        /** Longest merged run. */
+        const val MAX_MERGED_CHARS = 5
+
         fun wordCount(word: String) = word.count { it == ' ' } + 1
     }
 }

app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroFormats.kt

@@ -9,29 +9,30 @@ package io.github.aedev.flow.data.recommendation
 /**
  * What kind of video a title announces ("review", "tutorial", "playalong"). A format word is never a
  * topic: it ends a chunk ([NeuroKeySpec] stage 5) and is learned as a preference of its own (#907).
+ * Chinese surfaces are Hans.
  */
 internal object NeuroFormats {
     /** Format id to the folded words that announce it, in the languages Flow can vouch for. */
     internal val SURFACES: Map<String, List<String>> =
         mapOf(
-            "review" to listOf("review", "reviews", "reseña", "análisis", "análise", "обзор", "inceleme"),
-            "unboxing" to listOf("unboxing", "déballage", "распаковка"),
+            "review" to listOf("review", "reviews", "reseña", "análisis", "análise", "обзор", "inceleme", "评测", "测评", "点评"),
+            "unboxing" to listOf("unboxing", "déballage", "распаковка", "开箱"),
             "reaction" to listOf("reaction", "reactions", "reacting", "reacción", "reação", "réaction", "реакция", "tepki"),
-            "tutorial" to listOf("tutorial", "tutorials", "tuto", "tutoriel", "урок"),
-            "explainer" to listOf("explained", "explainer"),
+            "tutorial" to listOf("tutorial", "tutorials", "tuto", "tutoriel", "урок", "教程", "教学"),
+            "explainer" to listOf("explained", "explainer", "解说", "讲解"),
             "vlog" to listOf("vlog", "vlogs"),
-            "podcast" to listOf("podcast", "podcasts", "подкаст"),
-            "live" to listOf("live", "livestream"),
-            "compilation" to listOf("compilation", "compilación", "compilação", "подборка"),
+            "podcast" to listOf("podcast", "podcasts", "подкаст", "播客"),
+            "live" to listOf("live", "livestream", "直播"),
+            "compilation" to listOf("compilation", "compilación", "compilação", "подборка", "合集", "混剪"),
             "playalong" to listOf("playalong"),
-            "cover" to listOf("cover", "covers", "кавер"),
-            "remix" to listOf("remix", "remixes", "ремикс"),
-            "gameplay" to listOf("gameplay", "геймплей"),
-            "walkthrough" to listOf("walkthrough", "playthrough", "прохождение"),
-            "highlights" to listOf("highlights", "resumen", "хайлайты"),
-            "trailer" to listOf("trailer", "teaser", "tráiler", "трейлер", "fragman"),
-            "interview" to listOf("interview", "interviews", "entrevista", "интервью", "röportaj"),
-            "documentary" to listOf("documentary", "documental", "documentário", "documentaire", "belgesel"),
+            "cover" to listOf("cover", "covers", "кавер", "翻唱"),
+            "remix" to listOf("remix", "remixes", "ремикс", "混音"),
+            "gameplay" to listOf("gameplay", "геймплей", "实况"),
+            "walkthrough" to listOf("walkthrough", "playthrough", "прохождение", "攻略", "通关"),
+            "highlights" to listOf("highlights", "resumen", "хайлайты", "集锦"),
+            "trailer" to listOf("trailer", "teaser", "tráiler", "трейлер", "fragman", "预告", "预告片"),
+            "interview" to listOf("interview", "interviews", "entrevista", "интервью", "röportaj", "采访", "访谈", "专访"),
+            "documentary" to listOf("documentary", "documental", "documentário", "documentaire", "belgesel", "纪录片"),
             "asmr" to listOf("asmr"),
             "tier list" to listOf("tier list"),
             "versus" to listOf("versus", "vs"),

app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroKeyMigration.kt

@@ -79,6 +79,26 @@ internal object NeuroKeyMigration {
         )
     }
 
+    /** Schema 21: re-key by [NeuroKeySpec] 3; Hant spelling kept as [DisplayForm]; no fold for `ja` profiles. */
+    fun toKeySpec3(
+        brain: UserBrain,
+        tokenizer: NeuroTokenizer,
+    ): UserBrain {
+        val rekey = Rekey(tokenizer, emptyMap())
+        val forms = rekey.merged(brain.displayForms) { a, b -> if (a.votes >= b.votes) a else b }.toMutableMap()
+        val linked = (brain.topicAffinities.keys + brain.tagAffinities.keys).flatMap { it.split('|') }
+        val keys = brain.globalVector.topics.keys + brain.topicEvidence.keys + linked
+        keys.mapTo(HashSet(), NeuroScoring::stripDomainTag).forEach { base ->
+            val folded = rekey.keysFor(base).singleOrNull() ?: return@forEach
+            if (folded != base && folded !in forms) forms[folded] = DisplayForm(base)
+        }
+        return run(brain, tokenizer, TopicEvidence::mergedWith, dropChancePairs = false, dropOnboardingEdges = false).copy(
+            displayForms = forms,
+            idfWordFrequency = rekey.merged(brain.idfWordFrequency, Int::plus),
+            blockedTopics = brain.blockedTopics.mapTo(LinkedHashSet()) { rekey.keysFor(it).singleOrNull() ?: it },
+        )
+    }
+
     /** The keys [tokenizer] reads a stored key as: one, several when it splits, none when it goes. */
     fun keyFunction(tokenizer: NeuroTokenizer): (String) -> List<String> = Rekey(tokenizer, emptyMap())::keysFor
 
@@ -170,19 +190,22 @@ internal object NeuroKeyMigration {
     ) {
         private val cache = HashMap<String, List<String>>()
 
+        private val foldHan = tokenizer.hanFolds
+
         fun keysFor(key: String): List<String> =
             cache.getOrPut(key) {
                 // A word-level rejection pattern ("~guitar") keeps its mark.
                 if (key.startsWith(WORD_PATTERN)) return@getOrPut keysFor(key.removePrefix(WORD_PATTERN)).map { WORD_PATTERN + it }
                 val tag = key.substringAfter(':', "")
                 val base = if (tag.isEmpty()) key else key.substringBefore(':')
                 pickForms[key]?.let { return@getOrPut it }
+                val read = if (foldHan) NeuroText.simplified(base) else base
                 // A key left with no word ("m/v") goes.
                 val parts =
-                    if (needsRekey(base)) {
-                        tokenizer.chunk(base).chunks.map { it.joinToString(" ") }
+                    if (needsRekey(read)) {
+                        tokenizer.chunk(read).chunks.map { it.joinToString(" ") }
                     } else {
-                        listOf(base.split(' ').joinToString(" ", transform = tokenizer::keyOf))
+                        listOf(read.split(' ').joinToString(" ", transform = tokenizer::keyOf))
                     }
                 if (tag.isEmpty()) parts else parts.map { "$it:$tag" }
             }

app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroKeySpec.kt

@@ -7,11 +7,13 @@
 package io.github.aedev.flow.data.recommendation
 
 /**
- * Key spec 2: how text becomes the keys the engine learns, ranks and syncs by. Flow Desktop
+ * Key spec 3: how text becomes the keys the engine learns, ranks and syncs by. Flow Desktop
  * implements the same stages, and both pass app/src/test/resources/neuro/key_spec_golden.json.
  *
  * 1. Fold ([NeuroText.fold]): NFKC, small capitals, lowercase. ’ ‘ ʼ ` ´ read as ', Turkish İ as i,
- *    and the emoji joiners U+FE0F and U+200D are dropped.
+ *    and the emoji joiners U+FE0F and U+200D are dropped. Hant folds to Hans
+ *    (`resources/neuro/fold/hant-hans.txt`, 1:1) unless the video holds kana. Keys are Hans;
+ *    [NeuroDisplayForms] keeps the source spelling.
  * 2. Split at whitespace and punctuation; Chinese, Japanese, Thai, Lao, Khmer and Burmese by
  *    dictionary. [KEEP_WHOLE] words ("c++") stay whole, and a dot between letters splits a word
  *    unless the word is listed. A single quote is dropped rather than splitting anything.
@@ -24,20 +26,29 @@ package io.github.aedev.flow.data.recommendation
  *    bare numbers, words under three letters and punctuation end a chunk. A number right after a
  *    word makes a model pair ("pixel 10").
  * 6. Candidates: every word, and every run of two or three words inside one chunk, never across a
- *    boundary. [NEVER_ALONE] words only count inside a run.
+ *    boundary. [NEVER_ALONE] words only count inside a run. Single Han characters from the breaker
+ *    are merged into one candidate with adjacent single characters, a preceding word (大谷 + 翔 + 平) or
+ *    a following word after a lone character (李 + 永乐). Runs end at `resources/neuro/runbreak`
+ *    characters, are 2 to 5 characters, and contain no stop or format word. No break list, no merge.
  * 7. Promotion: words and runs count toward document frequency once per video. A run becomes a key
  *    only when [NeuroPhrases] finds it is a phrase, a creator tag holds it, or it is a compound or
- *    catalog topic.
+ *    catalog topic. A merged Han run becomes one per [NeuroPhrases.isMergedRun]; single characters
+ *    count toward document frequency.
  * 8. Stems ([NeuroStemmer]): each word's lemma is stemmed once, by its script and the viewer's
  *    language. The lemma stays when it is in `resources/neuro/keys/unstemmed.txt`, holds anything
  *    but letters, or would stem to under three letters, to a stem that stems again, or to a word
- *    the lemma map rewrites. Spec 1 kept the lemma.
+ *    the lemma map rewrites. Spec 1 kept the lemma. Chinese is not stemmed.
+ *
+ * Spec 3 changes stages 1, 6 and 7 for Chinese.
  *
  * A key is one to three words joined by a space and at most [MAX_KEY_LENGTH] characters. It never
  * holds "|" or ":", except one trailing domain tag ("metal:music").
  */
 internal object NeuroKeySpec {
-    const val VERSION = 2
+    const val VERSION = 3
+
+    /** First spec that folds Hant to Hans and merges Han runs. */
+    const val HAN_SPEC = 3
     const val MAX_RUN_WORDS = 3
     const val MAX_KEY_LENGTH = 40
 

app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroLanguages.kt

@@ -101,17 +101,36 @@ internal object NeuroLanguages {
             else -> language
         }
 
+    /** Only the `zh` lists fold Han; `ja` and `ko` keep their kanji and hanja. */
+    fun listFold(language: String): HanFold = if (language == "zh") HanFold.UNLESS_KANA else HanFold.NONE
+
     /** [language]'s stop words, folded, or none when it has no list. Read once per process. */
     fun stopList(language: String): Set<String> {
         if (language !in STOP_LIST_LANGUAGES) return emptySet()
         return stopLists.getOrPut(language) {
             val stream = NeuroLanguages::class.java.getResourceAsStream("/neuro/stop/$language.txt") ?: return@getOrPut emptySet()
             stream.bufferedReader(Charsets.UTF_8).useLines { lines ->
-                lines.filter { it.isNotBlank() && !it.startsWith("#") }.mapTo(HashSet()) { NeuroText.fold(it.trim()) }
+                lines.filter { it.isNotBlank() && !it.startsWith("#") }.mapTo(HashSet()) { NeuroText.fold(it.trim(), listFold(language)) }
             }
         }
     }
 
+    private val runBreakSets = ConcurrentHashMap<String, Set<Char>>()
+
+    /** Merged-run break characters of [language] (`resources/neuro/runbreak`), or null: no merge. */
+    fun runBreaks(language: String): Set<Char>? {
+        val set =
+            runBreakSets.getOrPut(language) {
+                val stream = NeuroLanguages::class.java.getResourceAsStream("/neuro/runbreak/$language.txt") ?: return@getOrPut emptySet()
+                stream.bufferedReader(Charsets.UTF_8).useLines { lines ->
+                    lines.filter { it.isNotBlank() && !it.startsWith("#") }.mapTo(HashSet()) {
+                        NeuroText.fold(it.trim(), listFold(language)).single()
+                    }
+                }
+            }
+        return set.takeIf { it.isNotEmpty() }
+    }
+
     private val TOPICAL_LANGUAGES = listOf("zh")
 
     /**