diff --git a/.github/nightly-notes/2026-10-20-neuro-key-spec-3.md b/.github/nightly-notes/2026-10-20-neuro-key-spec-3.md new file mode 100644 index 0000000000..5d150fe3e0 --- /dev/null +++ b/.github/nightly-notes/2026-10-20-neuro-key-spec-3.md @@ -0,0 +1,2 @@ +## Improved +- Recommendations read Traditional and Simplified Chinese titles as the same words (#1341) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 78eca9e0be..fd0fb2d4bb 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -404,6 +404,8 @@ dependencies { testImplementation(libs.kotlinx.coroutines.test) testImplementation(libs.mockk) testImplementation(libs.truth) + // Han word breaker for tests; the JDK breaker does not split Han runs. + testImplementation(libs.icu4j) testImplementation(libs.turbine) testImplementation(libs.androidx.glance.appwidget.testing) testImplementation(libs.hilt.android.testing) diff --git a/app/src/main/assets/licenses/notices.txt b/app/src/main/assets/licenses/notices.txt index ae445e45d5..16118544cb 100644 --- a/app/src/main/assets/licenses/notices.txt +++ b/app/src/main/assets/licenses/notices.txt @@ -63,3 +63,8 @@ Used, trimmed to the most frequent words of each language, in resources/neuro/fr Licensed under the Creative Commons Attribution-ShareAlike 4.0 International License (https://creativecommons.org/licenses/by-sa/4.0/). The trimmed lists are shared under the same license. + +Open Chinese Convert (github.com/BYVoid/OpenCC) +The Traditional to Simplified character table in resources/neuro/fold is made from +OpenCC's TSCharacters.txt. Copyright (c) the OpenCC authors. Licensed under the +Apache License, Version 2.0 (https://www.apache.org/licenses/LICENSE-2.0). diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/FlowNeuroEngine.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/FlowNeuroEngine.kt index 4b08bbad86..9ebe200eb4 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/FlowNeuroEngine.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/FlowNeuroEngine.kt @@ -509,6 +509,8 @@ class FlowNeuroEngine internal constructor( if (maintained !== currentUserBrain) { if (currentUserBrain.schemaVersion < NeuroMaintenance.KEY_SPEC_2_SCHEMA_VERSION) { storage.keepCopyBeforeMigration(currentUserBrain, NeuroMaintenance.KEY_SPEC_2_SCHEMA_VERSION) + } else if (currentUserBrain.schemaVersion < NeuroMaintenance.KEY_SPEC_3_SCHEMA_VERSION) { + storage.keepCopyBeforeMigration(currentUserBrain, NeuroMaintenance.KEY_SPEC_3_SCHEMA_VERSION) } currentUserBrain = maintained storage.save(deepFlowBookkeeping.persistable(currentUserBrain)) @@ -925,7 +927,7 @@ class FlowNeuroEngine internal constructor( suspend fun getBlockedTopics(): Set = withBrainLock { currentUserBrain.blockedTopics } suspend fun addBlockedTopic(topic: String) { - val normalized = NeuroText.fold(topic).trim() + val normalized = tokenizer.fold(topic).trim() if (normalized.isBlank()) return withBrainLock { val lemma = tokenizer.keyOf(normalized) diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroChunker.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroChunker.kt index 92af216956..ed0402f945 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroChunker.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroChunker.kt @@ -15,6 +15,7 @@ internal class NeuroChunker( private val stopWordsFor: (pieces: List) -> (String) -> Boolean, private val normalizeLemma: (String) -> String, private val keyOf: (String) -> String, + private val breaker: WordBreaker = PlatformWordBreaker, ) { /** A run of [words] inside one chunk, from word [from] up to but not including [to]. */ data class Run( @@ -28,6 +29,10 @@ internal class NeuroChunker( val chunks: List>, val modelPairs: List, val formats: Set = emptySet(), + /** Merged Han run candidates; promoted by [NeuroPhrases.isMergedRun]. */ + val merged: List = emptyList(), + /** Single Han characters outside the break set, counted for [NeuroPhrases.isMergedRun]. */ + val singles: Set = emptySet(), ) { /** Every word in reading order, [NeuroKeySpec.NEVER_ALONE] words included. */ val words: List get() = chunks.flatten() @@ -66,9 +71,15 @@ internal class NeuroChunker( data object Break : Item } - fun chunk(text: String): Chunks { + /** [han]: the Hant fold and merged Han runs ([NeuroKeySpec.HAN_SPEC]); [HanFold.NONE] is spec 2. */ + fun chunk( + text: String, + han: HanFold, + ): Chunks { val items = ArrayList() - val pieces = NeuroText.words(NeuroText.fold(text)) + val folded = NeuroText.fold(text, han) + val spans = NeuroText.pieces(folded, breaker) + val pieces = spans.map { it.text } val isStopWord = stopWordsFor(pieces) pieces.forEach { piece -> split(piece, items) } val merged = mergeExpressions(items) @@ -103,7 +114,75 @@ internal class NeuroChunker( } } close() - return Chunks(chunks, modelPairs, formats) + val runs = if (han != HanFold.NONE) mergedRuns(folded, spans, isStopWord, han) else MergedRuns.None + return Chunks(chunks, modelPairs, formats + runs.formats, runs.keys, runs.singles) + } + + private class MergedRuns( + val keys: List, + val singles: Set, + val formats: Set, + ) { + companion object { + val None = MergedRuns(emptyList(), emptySet(), emptySet()) + } + } + + // Stage 6: merge single Han characters (plus an adjacent word) into candidates, split at runBreaks. + // Format words inside a run are reported as formats. + private fun mergedRuns( + folded: String, + spans: List, + isStopWord: (String) -> Boolean, + han: HanFold, + ): MergedRuns { + if (folded.none(NeuroText::isHan)) return MergedRuns.None + val language = if (han == HanFold.UNLESS_KANA && folded.any(NeuroText::isKana)) "ja" else "zh" + val breaks = NeuroLanguages.runBreaks(language) ?: return MergedRuns.None + val keys = LinkedHashSet() + val singles = LinkedHashSet() + val formats = LinkedHashSet() + var group = ArrayList() + + fun valid(candidate: String): Boolean = + candidate.length in 2..MAX_MERGED_CHARS && + (2..candidate.length).none { length -> + (0..candidate.length - length).any { start -> + val part = candidate.substring(start, start + length) + isStopWord(part) || NeuroFormats.formatOf(part) != null + } + } + + fun flush() { + var i = 0 + while (i < group.size) { + if (group[i].length != 1) { + i++ + continue + } + var end = i + while (end < group.size && group[end].length == 1) end++ + val run = group.subList(i, end).joinToString("") + val prefix = group.getOrNull(i - 1)?.takeIf { it.length >= 2 }.orEmpty() + val suffix = if (run.length == 1) group.getOrNull(end)?.takeIf { it.length >= 2 }.orEmpty() else "" + singles += group.subList(i, end) + for (length in 2..run.length) { + for (start in 0..run.length - length) NeuroFormats.formatOf(run.substring(start, start + length))?.let { formats += it } + } + if (valid(run)) keys += run + if (prefix.isNotEmpty() && valid(prefix + run)) keys += prefix + run + if (suffix.isNotEmpty() && valid(run + suffix)) keys += run + suffix + i = end + } + group = ArrayList() + } + spans.forEach { span -> + val han = span.text.all(NeuroText::isHan) + if (!han || !span.joined || (span.text.length == 1 && span.text[0] in breaks)) flush() + if (han && !(span.text.length == 1 && span.text[0] in breaks)) group += span.text + } + flush() + return MergedRuns(keys.toList(), singles, formats) } // Stage 2 and 4: one whitespace-separated piece into words and breaks. @@ -262,6 +341,9 @@ internal class NeuroChunker( val KEEP_SUFFIX = setOf('+', '#') val YEAR = Regex("^(19|20)\\d\\d$") + /** Longest merged run. */ + const val MAX_MERGED_CHARS = 5 + fun wordCount(word: String) = word.count { it == ' ' } + 1 } } diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroFormats.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroFormats.kt index 8343d62f0d..c90ccf8ce6 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroFormats.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroFormats.kt @@ -9,29 +9,30 @@ package io.github.aedev.flow.data.recommendation /** * What kind of video a title announces ("review", "tutorial", "playalong"). A format word is never a * topic: it ends a chunk ([NeuroKeySpec] stage 5) and is learned as a preference of its own (#907). + * Chinese surfaces are Hans. */ internal object NeuroFormats { /** Format id to the folded words that announce it, in the languages Flow can vouch for. */ internal val SURFACES: Map> = mapOf( - "review" to listOf("review", "reviews", "reseña", "análisis", "análise", "обзор", "inceleme"), - "unboxing" to listOf("unboxing", "déballage", "распаковка"), + "review" to listOf("review", "reviews", "reseña", "análisis", "análise", "обзор", "inceleme", "评测", "测评", "点评"), + "unboxing" to listOf("unboxing", "déballage", "распаковка", "开箱"), "reaction" to listOf("reaction", "reactions", "reacting", "reacción", "reação", "réaction", "реакция", "tepki"), - "tutorial" to listOf("tutorial", "tutorials", "tuto", "tutoriel", "урок"), - "explainer" to listOf("explained", "explainer"), + "tutorial" to listOf("tutorial", "tutorials", "tuto", "tutoriel", "урок", "教程", "教学"), + "explainer" to listOf("explained", "explainer", "解说", "讲解"), "vlog" to listOf("vlog", "vlogs"), - "podcast" to listOf("podcast", "podcasts", "подкаст"), - "live" to listOf("live", "livestream"), - "compilation" to listOf("compilation", "compilación", "compilação", "подборка"), + "podcast" to listOf("podcast", "podcasts", "подкаст", "播客"), + "live" to listOf("live", "livestream", "直播"), + "compilation" to listOf("compilation", "compilación", "compilação", "подборка", "合集", "混剪"), "playalong" to listOf("playalong"), - "cover" to listOf("cover", "covers", "кавер"), - "remix" to listOf("remix", "remixes", "ремикс"), - "gameplay" to listOf("gameplay", "геймплей"), - "walkthrough" to listOf("walkthrough", "playthrough", "прохождение"), - "highlights" to listOf("highlights", "resumen", "хайлайты"), - "trailer" to listOf("trailer", "teaser", "tráiler", "трейлер", "fragman"), - "interview" to listOf("interview", "interviews", "entrevista", "интервью", "röportaj"), - "documentary" to listOf("documentary", "documental", "documentário", "documentaire", "belgesel"), + "cover" to listOf("cover", "covers", "кавер", "翻唱"), + "remix" to listOf("remix", "remixes", "ремикс", "混音"), + "gameplay" to listOf("gameplay", "геймплей", "实况"), + "walkthrough" to listOf("walkthrough", "playthrough", "прохождение", "攻略", "通关"), + "highlights" to listOf("highlights", "resumen", "хайлайты", "集锦"), + "trailer" to listOf("trailer", "teaser", "tráiler", "трейлер", "fragman", "预告", "预告片"), + "interview" to listOf("interview", "interviews", "entrevista", "интервью", "röportaj", "采访", "访谈", "专访"), + "documentary" to listOf("documentary", "documental", "documentário", "documentaire", "belgesel", "纪录片"), "asmr" to listOf("asmr"), "tier list" to listOf("tier list"), "versus" to listOf("versus", "vs"), diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroKeyMigration.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroKeyMigration.kt index 60f3449d5b..fe8a8098c2 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroKeyMigration.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroKeyMigration.kt @@ -79,6 +79,26 @@ internal object NeuroKeyMigration { ) } + /** Schema 21: re-key by [NeuroKeySpec] 3; Hant spelling kept as [DisplayForm]; no fold for `ja` profiles. */ + fun toKeySpec3( + brain: UserBrain, + tokenizer: NeuroTokenizer, + ): UserBrain { + val rekey = Rekey(tokenizer, emptyMap()) + val forms = rekey.merged(brain.displayForms) { a, b -> if (a.votes >= b.votes) a else b }.toMutableMap() + val linked = (brain.topicAffinities.keys + brain.tagAffinities.keys).flatMap { it.split('|') } + val keys = brain.globalVector.topics.keys + brain.topicEvidence.keys + linked + keys.mapTo(HashSet(), NeuroScoring::stripDomainTag).forEach { base -> + val folded = rekey.keysFor(base).singleOrNull() ?: return@forEach + if (folded != base && folded !in forms) forms[folded] = DisplayForm(base) + } + return run(brain, tokenizer, TopicEvidence::mergedWith, dropChancePairs = false, dropOnboardingEdges = false).copy( + displayForms = forms, + idfWordFrequency = rekey.merged(brain.idfWordFrequency, Int::plus), + blockedTopics = brain.blockedTopics.mapTo(LinkedHashSet()) { rekey.keysFor(it).singleOrNull() ?: it }, + ) + } + /** The keys [tokenizer] reads a stored key as: one, several when it splits, none when it goes. */ fun keyFunction(tokenizer: NeuroTokenizer): (String) -> List = Rekey(tokenizer, emptyMap())::keysFor @@ -170,6 +190,8 @@ internal object NeuroKeyMigration { ) { private val cache = HashMap>() + private val foldHan = tokenizer.hanFolds + fun keysFor(key: String): List = cache.getOrPut(key) { // A word-level rejection pattern ("~guitar") keeps its mark. @@ -177,12 +199,13 @@ internal object NeuroKeyMigration { val tag = key.substringAfter(':', "") val base = if (tag.isEmpty()) key else key.substringBefore(':') pickForms[key]?.let { return@getOrPut it } + val read = if (foldHan) NeuroText.simplified(base) else base // A key left with no word ("m/v") goes. val parts = - if (needsRekey(base)) { - tokenizer.chunk(base).chunks.map { it.joinToString(" ") } + if (needsRekey(read)) { + tokenizer.chunk(read).chunks.map { it.joinToString(" ") } } else { - listOf(base.split(' ').joinToString(" ", transform = tokenizer::keyOf)) + listOf(read.split(' ').joinToString(" ", transform = tokenizer::keyOf)) } if (tag.isEmpty()) parts else parts.map { "$it:$tag" } } diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroKeySpec.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroKeySpec.kt index afa9016dbb..da52ecec22 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroKeySpec.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroKeySpec.kt @@ -7,11 +7,13 @@ package io.github.aedev.flow.data.recommendation /** - * Key spec 2: how text becomes the keys the engine learns, ranks and syncs by. Flow Desktop + * Key spec 3: how text becomes the keys the engine learns, ranks and syncs by. Flow Desktop * implements the same stages, and both pass app/src/test/resources/neuro/key_spec_golden.json. * * 1. Fold ([NeuroText.fold]): NFKC, small capitals, lowercase. ’ ‘ ʼ ` ´ read as ', Turkish İ as i, - * and the emoji joiners U+FE0F and U+200D are dropped. + * and the emoji joiners U+FE0F and U+200D are dropped. Hant folds to Hans + * (`resources/neuro/fold/hant-hans.txt`, 1:1) unless the video holds kana. Keys are Hans; + * [NeuroDisplayForms] keeps the source spelling. * 2. Split at whitespace and punctuation; Chinese, Japanese, Thai, Lao, Khmer and Burmese by * dictionary. [KEEP_WHOLE] words ("c++") stay whole, and a dot between letters splits a word * unless the word is listed. A single quote is dropped rather than splitting anything. @@ -24,20 +26,29 @@ package io.github.aedev.flow.data.recommendation * bare numbers, words under three letters and punctuation end a chunk. A number right after a * word makes a model pair ("pixel 10"). * 6. Candidates: every word, and every run of two or three words inside one chunk, never across a - * boundary. [NEVER_ALONE] words only count inside a run. + * boundary. [NEVER_ALONE] words only count inside a run. Single Han characters from the breaker + * are merged into one candidate with adjacent single characters, a preceding word (大谷 + 翔 + 平) or + * a following word after a lone character (李 + 永乐). Runs end at `resources/neuro/runbreak` + * characters, are 2 to 5 characters, and contain no stop or format word. No break list, no merge. * 7. Promotion: words and runs count toward document frequency once per video. A run becomes a key * only when [NeuroPhrases] finds it is a phrase, a creator tag holds it, or it is a compound or - * catalog topic. + * catalog topic. A merged Han run becomes one per [NeuroPhrases.isMergedRun]; single characters + * count toward document frequency. * 8. Stems ([NeuroStemmer]): each word's lemma is stemmed once, by its script and the viewer's * language. The lemma stays when it is in `resources/neuro/keys/unstemmed.txt`, holds anything * but letters, or would stem to under three letters, to a stem that stems again, or to a word - * the lemma map rewrites. Spec 1 kept the lemma. + * the lemma map rewrites. Spec 1 kept the lemma. Chinese is not stemmed. + * + * Spec 3 changes stages 1, 6 and 7 for Chinese. * * A key is one to three words joined by a space and at most [MAX_KEY_LENGTH] characters. It never * holds "|" or ":", except one trailing domain tag ("metal:music"). */ internal object NeuroKeySpec { - const val VERSION = 2 + const val VERSION = 3 + + /** First spec that folds Hant to Hans and merges Han runs. */ + const val HAN_SPEC = 3 const val MAX_RUN_WORDS = 3 const val MAX_KEY_LENGTH = 40 diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroLanguages.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroLanguages.kt index 9ec348f4c3..c4086a5cdb 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroLanguages.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroLanguages.kt @@ -101,17 +101,36 @@ internal object NeuroLanguages { else -> language } + /** Only the `zh` lists fold Han; `ja` and `ko` keep their kanji and hanja. */ + fun listFold(language: String): HanFold = if (language == "zh") HanFold.UNLESS_KANA else HanFold.NONE + /** [language]'s stop words, folded, or none when it has no list. Read once per process. */ fun stopList(language: String): Set { if (language !in STOP_LIST_LANGUAGES) return emptySet() return stopLists.getOrPut(language) { val stream = NeuroLanguages::class.java.getResourceAsStream("/neuro/stop/$language.txt") ?: return@getOrPut emptySet() stream.bufferedReader(Charsets.UTF_8).useLines { lines -> - lines.filter { it.isNotBlank() && !it.startsWith("#") }.mapTo(HashSet()) { NeuroText.fold(it.trim()) } + lines.filter { it.isNotBlank() && !it.startsWith("#") }.mapTo(HashSet()) { NeuroText.fold(it.trim(), listFold(language)) } } } } + private val runBreakSets = ConcurrentHashMap>() + + /** Merged-run break characters of [language] (`resources/neuro/runbreak`), or null: no merge. */ + fun runBreaks(language: String): Set? { + val set = + runBreakSets.getOrPut(language) { + val stream = NeuroLanguages::class.java.getResourceAsStream("/neuro/runbreak/$language.txt") ?: return@getOrPut emptySet() + stream.bufferedReader(Charsets.UTF_8).useLines { lines -> + lines.filter { it.isNotBlank() && !it.startsWith("#") }.mapTo(HashSet()) { + NeuroText.fold(it.trim(), listFold(language)).single() + } + } + } + return set.takeIf { it.isNotEmpty() } + } + private val TOPICAL_LANGUAGES = listOf("zh") /** diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroMaintenance.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroMaintenance.kt index a047b97919..c8a03d829f 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroMaintenance.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroMaintenance.kt @@ -35,6 +35,8 @@ package io.github.aedev.flow.data.recommendation * V19 drops keys that are stop words in the viewer's language ([NeuroLanguages]), such as "para" or "les". * * V20 stems every key by [NeuroKeySpec] 2 ([NeuroKeyMigration.toKeySpec2]). + * + * V21 folds Hant keys to Hans ([NeuroKeyMigration.toKeySpec3]). */ internal object NeuroMaintenance { /** Stop lists for the viewer's language ([NeuroLanguages]): keys that are now stop words go. */ @@ -43,7 +45,10 @@ internal object NeuroMaintenance { /** Keys stemmed by [NeuroKeySpec] 2 ([NeuroKeyMigration.toKeySpec2]). */ const val KEY_SPEC_2_SCHEMA_VERSION = 20 - const val TARGET_SCHEMA_VERSION = KEY_SPEC_2_SCHEMA_VERSION + /** [NeuroKeyMigration.toKeySpec3]. */ + const val KEY_SPEC_3_SCHEMA_VERSION = 21 + + const val TARGET_SCHEMA_VERSION = KEY_SPEC_3_SCHEMA_VERSION private const val V15_SCHEMA_VERSION = 15 private const val V16_SCHEMA_VERSION = 16 @@ -72,8 +77,9 @@ internal object NeuroMaintenance { if (updated.schemaVersion < KEY_SPEC_2_SCHEMA_VERSION) { // Stop words are spelled whole; once stemmed ("youtube" to "youtub") the list no longer finds them. updated = dropNoise(updated, earlier) - updated = NeuroKeyMigration.toKeySpec2(updated, tokenizer) + updated = NeuroKeyMigration.toKeySpec2(updated, tokenizer.atKeySpec(2)) } + if (updated.schemaVersion < KEY_SPEC_3_SCHEMA_VERSION) updated = NeuroKeyMigration.toKeySpec3(updated, tokenizer) updated = dropBlockedTopics(updated, tokenizer) updated = dropNoise(updated, tokenizer) if (isHollow(updated)) updated = rehydrateFromChannelProfiles(updated, tokenizer) @@ -84,7 +90,7 @@ internal object NeuroMaintenance { if (key.all { it.code < 0x80 }) { key } else { - key.split('|').map(NeuroText::fold).let { parts -> + key.split('|').map { NeuroText.fold(it, HanFold.NONE) }.let { parts -> if (parts.size == 2) NeuroScoring.makeAffinityKey(parts[0], parts[1]) else parts.joinToString("|") } } @@ -136,7 +142,7 @@ internal object NeuroMaintenance { tokenizer: NeuroTokenizer, withPhraseWords: Boolean = false, ): Set { - val lemmas = brain.blockedTopics.mapTo(HashSet()) { tokenizer.keyOf(NeuroText.fold(it).trim()) } + val lemmas = brain.blockedTopics.mapTo(HashSet()) { tokenizer.keyOf(tokenizer.fold(it).trim()) } if (withPhraseWords) { // Blocking "code technology" hides that phrase, never the viewer's own pick "Coding". val picked = brain.preferredTopics.flatMapTo(HashSet(), tokenizer::keysOf) diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroPhrases.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroPhrases.kt index 9d1e8603aa..28cc0dd392 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroPhrases.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroPhrases.kt @@ -47,4 +47,18 @@ internal object NeuroPhrases { if ((idf.wordFrequency[bigram] ?: 0) < MIN_COUNT) return false return (npmi(bigram, idf) ?: return false) >= MIN_NPMI } + + /** [MIN_COUNT] documents, and document frequency of [run] at least [MIN_MERGED_SHARE] of its rarest single character. */ + fun isMergedRun( + run: String, + idf: IdfSnapshot, + ): Boolean { + if (idf.totalDocs < MIN_DOCUMENTS) return false + val joint = idf.wordFrequency[run] ?: return false + if (joint < MIN_COUNT) return false + val rarest = run.mapNotNull { idf.wordFrequency[it.toString()] }.minOrNull()?.let { maxOf(it, joint) } ?: return false + return joint >= MIN_MERGED_SHARE * rarest + } + + const val MIN_MERGED_SHARE = 0.5 } diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroScoring.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroScoring.kt index 9b8e394787..86e7568b63 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroScoring.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroScoring.kt @@ -238,6 +238,15 @@ internal object NeuroScoring { "build", "design", "travel", + // Hans keys. + "新闻", + "音乐", + "游戏", + "科技", + "电影", + "教育", + "美食", + "旅游", ) // ── Time Decay Engine ── @@ -704,7 +713,7 @@ internal object NeuroScoring { val tokens = mutableSetOf() fun addTerm(term: String) { - val lower = term.lowercase().trim() + val lower = NeuroText.fold(term, HanFold.ALWAYS).trim() if (lower.isEmpty()) return if (lower.contains(' ')) { phrases += lower @@ -733,8 +742,8 @@ internal object NeuroScoring { normalizeLemma: (String) -> String, ): Boolean { if (matchers.isEmpty()) return false - val titleLower = NeuroText.fold(title) - val channelLower = NeuroText.fold(channelName) + val titleLower = NeuroText.fold(title, HanFold.ALWAYS) + val channelLower = NeuroText.fold(channelName, HanFold.ALWAYS) if (matchers.phrases.any { titleLower.contains(it) || channelLower.contains(it) }) return true if (matchers.tokens.isEmpty()) return false return sequenceOf(titleLower, channelLower) diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroSpecificity.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroSpecificity.kt index f7f9876702..ce243bb24c 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroSpecificity.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroSpecificity.kt @@ -41,7 +41,14 @@ internal class NeuroSpecificity( lines .mapNotNull { line -> val space = line.lastIndexOf(' ') - if (space <= 0) null else NeuroText.fold(line.substring(0, space)) to line.substring(space + 1).toInt() + if (space <= + 0 + ) { + null + } else { + NeuroText.fold(line.substring(0, space), NeuroLanguages.listFold(language)) to + line.substring(space + 1).toInt() + } }.toMap() } } diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroText.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroText.kt index eff9547afb..6d3c509175 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroText.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroText.kt @@ -10,6 +10,33 @@ import java.text.BreakIterator import java.text.Normalizer import java.util.Locale +/** Dictionary word breaker: consecutive tokens of a chunk, punctuation included. */ +internal fun interface WordBreaker { + fun split(chunk: String): List +} + +/** `java.text.BreakIterator`; ICU on Android. */ +internal object PlatformWordBreaker : WordBreaker { + private val wordBreaks = ThreadLocal.withInitial { BreakIterator.getWordInstance(Locale.ROOT) } + + override fun split(chunk: String): List { + val breaks = wordBreaks.get() + breaks.setText(chunk) + val tokens = ArrayList() + var start = breaks.first() + var end = breaks.next() + while (end != BreakIterator.DONE) { + tokens += chunk.substring(start, end) + start = end + end = breaks.next() + } + return tokens + } +} + +/** Han fold of [NeuroText.fold]: [UNLESS_KANA] skips strings with kana, [ALWAYS] ignores kana. */ +internal enum class HanFold { NONE, UNLESS_KANA, ALWAYS } + /** * How the engine reads raw text before it becomes topics: styled letters fold to plain ones, short * topic names survive, and scripts written without spaces are split into words (#907). @@ -66,13 +93,28 @@ internal object NeuroText { // Turkish dotted capital I, the emoji variation selector and joiner, and the Unicode hyphens. private val Marks = setOf('\u0130', '\uFE0F', '\u200D', '\u2010', '\u2011') - private val wordBreaks = ThreadLocal.withInitial { BreakIterator.getWordInstance(Locale.ROOT) } + /** `resources/neuro/fold/hant-hans.txt`, from `scripts/neuro/gen_hant_hans.py`. */ + private val HantToHans: Map by lazy { + val stream = NeuroText::class.java.getResourceAsStream("/neuro/fold/hant-hans.txt") ?: return@lazy emptyMap() + stream.bufferedReader(Charsets.UTF_8).useLines { lines -> + lines.filter { it.length == 2 && !it.startsWith("#") }.associate { it[0] to it[1] } + } + } + + /** [joined]: no character between this word and the previous one. */ + data class Piece( + val text: String, + val joined: Boolean, + ) /** * Lowercase plain letters: "𝙋𝙃𝙊𝙉𝙆", "ｐｈｏｎｋ", "ⓟⓗⓞⓝⓚ" and "ᴘʜᴏɴᴋ" all read "phonk". Every apostrophe - * reads ', Turkish İ reads i, and emoji joiners are dropped ([NeuroKeySpec] stage 1). + * reads ', Turkish İ reads i, and emoji joiners are dropped ([NeuroKeySpec] stage 1). [han]: [HanFold]. */ - fun fold(text: String): String { + fun fold( + text: String, + han: HanFold = HanFold.UNLESS_KANA, + ): String { if (text.all { it.code < 0x80 }) return if ('`' in text) text.lowercase().replace('`', '\'') else text.lowercase() // Before NFKC, which turns ´ into a space and a combining accent. val marks = @@ -98,17 +140,49 @@ internal object NeuroText { .replace("\u0E4D\u0E32", "\u0E33") .replace("\u0ECD\u0EB2", "\u0EB3") val plain = if (normalized.any { it in SmallCaps }) normalized.map { SmallCaps[it] ?: it }.joinToString("") else normalized - return plain.lowercase() + val lower = plain.lowercase() + return if (han == HanFold.NONE) lower else simplified(lower, ignoreKana = han == HanFold.ALWAYS) + } + + fun hasKana(text: String): Boolean = text.any(::isKana) + + /** Hant to Hans per character; unchanged when [text] has kana unless [ignoreKana]. */ + fun simplified( + text: String, + ignoreKana: Boolean = false, + ): String { + val table = HantToHans + val first = text.indexOfFirst { it in table } + if (first < 0 || (!ignoreKana && hasKana(text))) return text + val out = StringBuilder(text.length).append(text, 0, first) + for (i in first until text.length) out.append(table[text[i]] ?: text[i]) + return out.toString() } + fun isKana(c: Char): Boolean = c in '\u3040'..'\u30FF' || c in '\u31F0'..'\u31FF' || c in '\uFF66'..'\uFF9F' + + // Every Han block starts at U+2E80; below it the script lookup is skipped. + fun isHan(c: Char): Boolean = c.code >= 0x2E80 && Character.UnicodeScript.of(c.code) == Character.UnicodeScript.HAN + /** - * The words of folded text. Chinese, Japanese, Thai, Lao, Khmer and Burmese runs are split by the - * platform's dictionary word breaker, since those scripts put no spaces between words. + * The words of folded text; unspaced scripts are split by [breaker]. */ - fun words(folded: String): List = - folded.split(Whitespace).flatMap { chunk -> - if (chunk.any(::isUnspaced)) segment(chunk) else listOf(chunk) + fun words( + folded: String, + breaker: WordBreaker = PlatformWordBreaker, + ): List = pieces(folded, breaker).map { it.text } + + /** [words] with adjacency. */ + fun pieces( + folded: String, + breaker: WordBreaker = PlatformWordBreaker, + ): List { + val out = ArrayList() + folded.split(Whitespace).forEach { chunk -> + if (chunk.any(::isUnspaced)) segment(chunk, breaker, out) else out += Piece(chunk, false) } + return out + } /** A letter, digit or combining mark: the marks spell vowels and tones in Thai and Devanagari words. */ fun isWordChar(c: Char): Boolean = @@ -132,18 +206,19 @@ internal object NeuroText { // Hiragana-only words are almost always particles and inflections, not subjects. private fun isGrammarKana(word: String): Boolean = word.all { Character.UnicodeScript.of(it.code) == Character.UnicodeScript.HIRAGANA } - private fun segment(chunk: String): List { - val breaks = wordBreaks.get() - breaks.setText(chunk) - val words = ArrayList() - var start = breaks.first() - var end = breaks.next() - while (end != BreakIterator.DONE) { - val word = chunk.substring(start, end) - if (word.any(Char::isLetterOrDigit)) words += word - start = end - end = breaks.next() + private fun segment( + chunk: String, + breaker: WordBreaker, + out: MutableList, + ) { + var joined = false + breaker.split(chunk).forEach { token -> + if (token.any(Char::isLetterOrDigit)) { + out += Piece(token, joined) + joined = true + } else { + joined = false + } } - return words } } diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroTokenizer.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroTokenizer.kt index a835563533..c2fabf2491 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroTokenizer.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroTokenizer.kt @@ -23,7 +23,10 @@ import kotlin.math.* * * Stateless utility — can be shared across all other components. */ -internal class NeuroTokenizer { +internal class NeuroTokenizer( + /** Dictionary breaker for unspaced scripts. */ + private val wordBreaker: WordBreaker = PlatformWordBreaker, +) { companion object { val WHITESPACE_REGEX = Regex("\\s+") @@ -853,7 +856,7 @@ internal class NeuroTokenizer { /** A tokenizer that reads keys by [spec] for the same viewer. */ fun atKeySpec(spec: Int): NeuroTokenizer = - NeuroTokenizer().also { + NeuroTokenizer(wordBreaker).also { it.keySpec = spec it.languageProfile = languageProfile it.keyLanguage = keyLanguage @@ -870,7 +873,7 @@ internal class NeuroTokenizer { compoundTerms.forEach { addAll(it.split(' ')) } NeuroTopicCatalog.TOPIC_CATEGORIES.forEach { category -> category.topics.forEach { topic -> - addAll(NeuroText.words(NeuroText.fold(topic)).map { it.trim { c -> !NeuroText.isWordChar(c) } }) + addAll(NeuroText.words(fold(topic), wordBreaker).map { it.trim { c -> !NeuroText.isWordChar(c) } }) } } } @@ -924,7 +927,26 @@ internal class NeuroTokenizer { return if (normalizeLemma(stem) == stem) stem else lemma } - private val chunker by lazy { NeuroChunker(::stopWordsFor, ::normalizeLemma, ::keyOf) } + private val chunker by lazy { + NeuroChunker(::stopWordsFor, ::normalizeLemma, ::keyOf, wordBreaker) + } + + /** [NeuroKeySpec.HAN_SPEC] and a non-`ja` viewer: keys fold Hant and merge Han runs. */ + val hanFolds: Boolean get() = keySpec >= NeuroKeySpec.HAN_SPEC && languageProfile != "ja" + + private val defaultHan: HanFold get() = if (hanFolds) HanFold.UNLESS_KANA else HanFold.NONE + + /** One fold for every field of [video]; kana in the title, channel name or tags turns it off. */ + fun hanFoldOf(video: Video): HanFold = if (hanFolds && !hasKana(video)) HanFold.ALWAYS else HanFold.NONE + + private fun hasKana(video: Video): Boolean = + NeuroText.hasKana(video.title) || NeuroText.hasKana(video.channelName) || + video.tags.take(TAG_MAX_INGEST).any(NeuroText::hasKana) + + fun fold( + text: String, + han: HanFold = defaultHan, + ): String = NeuroText.fold(text, han) /** Multi-word catalog topics, keyed the way a title spells them. */ private val catalogPhrases: Set by lazy { @@ -932,7 +954,7 @@ internal class NeuroTokenizer { .flatMap { it.topics } .mapNotNull { topic -> chunker - .chunk(topic) + .chunk(topic, defaultHan) .chunks .singleOrNull() ?.takeIf { it.size >= 2 } @@ -948,9 +970,15 @@ internal class NeuroTokenizer { } /** [text] read by [NeuroKeySpec]: its chunks of words that belong together. */ - fun chunk(text: String): NeuroChunker.Chunks = chunker.chunk(text) + fun chunk( + text: String, + han: HanFold = defaultHan, + ): NeuroChunker.Chunks = chunker.chunk(text, han) - fun tokenize(text: String): List = chunker.chunk(text).words + fun tokenize( + text: String, + han: HanFold = defaultHan, + ): List = chunker.chunk(text, han).words /** * The keys a short label stands for, such as an onboarding pick: one per chunk, so "Lo-Fi" reads @@ -958,10 +986,10 @@ internal class NeuroTokenizer { */ fun keysOf(label: String): List = chunker - .chunk(label) + .chunk(label, defaultHan) .chunks .map { it.joinToString(" ") } - .ifEmpty { listOfNotNull(keyOf(NeuroText.fold(label).trim()).takeIf { it.isNotBlank() }) } + .ifEmpty { listOfNotNull(keyOf(fold(label).trim()).takeIf { it.isNotBlank() }) } fun tokenizeForSimilarity(text: String): Set = tokenize(text).toSet() @@ -1007,12 +1035,13 @@ internal class NeuroTokenizer { channelTopicProfile: Map? = null, ): ContentVector { val topics = mutableMapOf() + val han = hanFoldOf(video) - val title = chunker.chunk(video.title) + val title = chunker.chunk(video.title, han) val titleWords = title.words // Use smart channel tokenization to filter names and branding - val chWords = tokenizeChannelName(video.channelName) + val chWords = tokenizeChannelName(video.channelName, han) // Channel keywords — filtered for actual topic words only chWords.forEach { @@ -1020,9 +1049,9 @@ internal class NeuroTokenizer { } // Collect all words for domain disambiguation context BEFORE processing tokens - val fullContext = buildContextWordList(video) + val fullContext = buildContextWordList(video, han) val tags = topicTags(video) - val tagChunks = tags.take(TAG_MAX_INGEST).map { chunker.chunk(it) } + val tagChunks = tags.take(TAG_MAX_INGEST).map { chunker.chunk(it, han) } val tagPhrases = tagChunks.flatMapTo(HashSet()) { tagPhrases(it) } val formats = title.formats + tagChunks.flatMap { it.formats } @@ -1035,6 +1064,11 @@ internal class NeuroTokenizer { for (i in run.from until run.to) claimed += position(run.chunk, i) } } + title.merged.forEach { run -> + if (NeuroPhrases.isMergedRun(run, idfSnapshot) || run in tagPhrases) { + topics[run] = calculateIdfWeight(run, BIGRAM_PRIORITY_WEIGHT, idfSnapshot) + } + } title.chunks.forEachIndexed { chunkIndex, chunk -> chunk.forEachIndexed { index, word -> if (position(chunkIndex, index) in claimed || word in NeuroKeySpec.NEVER_ALONE) return@forEachIndexed @@ -1057,6 +1091,7 @@ internal class NeuroTokenizer { extractDescriptionKeywords( video.description, idfSnapshot, + han, ) descriptionTopics.forEach { (word, weight) -> // Also disambiguate description words using the same full context @@ -1089,6 +1124,7 @@ internal class NeuroTokenizer { titleWords, descWords, idfSnapshot, + han, ) tagTopics.forEach { (word, weight) -> topics[word] = (topics[word] ?: 0.0) + weight @@ -1232,6 +1268,7 @@ internal class NeuroTokenizer { fun extractDescriptionKeywords( description: String?, idfSnapshot: IdfSnapshot, + han: HanFold = defaultHan, ): Map { if (description.isNullOrBlank() || description.length < DESCRIPTION_MIN_LENGTH) { return emptyMap() @@ -1252,7 +1289,7 @@ internal class NeuroTokenizer { if (contentLines.isBlank()) return emptyMap() - val words = tokenize(contentLines).take(DESCRIPTION_TAKE_WORDS) + val words = tokenize(contentLines, han).take(DESCRIPTION_TAKE_WORDS) val result = mutableMapOf() words.forEach { word -> result[word] = ( @@ -1276,6 +1313,7 @@ internal class NeuroTokenizer { titleWords: List, descriptionWords: List, idfSnapshot: IdfSnapshot, + han: HanFold = defaultHan, ): Map { if (tags.isEmpty()) return emptyMap() @@ -1286,7 +1324,7 @@ internal class NeuroTokenizer { val tagContextWords = tags.take(TAG_MAX_INGEST).flatMap { t -> NeuroText - .fold(t) + .fold(t, han) .split(WHITESPACE_REGEX) .map { it.trim { c -> !c.isLetterOrDigit() } } .filter { it.length > 1 } @@ -1294,7 +1332,7 @@ internal class NeuroTokenizer { val fullDisambiguationContext = tagContextWords + titleWords + descriptionWords tags.take(TAG_MAX_INGEST).forEach { rawTag -> - val cleaned = NeuroText.fold(rawTag).trim() + val cleaned = fold(rawTag, han).trim() if ((cleaned.length < TAG_MIN_LENGTH && cleaned !in NeuroText.SHORT_TOPICS) || cleaned.length > TAG_MAX_LENGTH @@ -1306,7 +1344,7 @@ internal class NeuroTokenizer { if (cleaned in stopWords) return@forEach if (YEAR_TAG_REGEX.matches(cleaned)) return@forEach - val chunked = chunker.chunk(cleaned) + val chunked = chunker.chunk(cleaned, han) val tagTokens = chunked.words if (tagTokens.isEmpty()) return@forEach @@ -1376,11 +1414,13 @@ internal class NeuroTokenizer { video: Video, vector: ContentVector, ): Set { - val title = chunker.chunk(video.title) + val title = chunker.chunk(video.title, hanFoldOf(video)) return HashSet(vector.topics.keys).apply { addAll(title.words) title.runs().forEach { add(it.key) } addAll(title.modelPairs) + addAll(title.merged) + addAll(title.singles) } } @@ -1406,14 +1446,15 @@ internal class NeuroTokenizer { video: Video, idfSnapshot: IdfSnapshot, ): Set { - val title = chunker.chunk(video.title) + val han = hanFoldOf(video) + val title = chunker.chunk(video.title, han) val titleWords = title.words.toSet() - val tags = topicTags(video).take(TAG_MAX_INGEST).map { chunker.chunk(it) } + val tags = topicTags(video).take(TAG_MAX_INGEST).map { chunker.chunk(it, han) } val tagWords = tags.flatMapTo(HashSet()) { it.words } val descriptionWords = video.description .takeIf { it.isNotBlank() } - ?.let { tokenize(it.lines().take(DESCRIPTION_TAKE_LINES).joinToString(" ")).toSet() } + ?.let { tokenize(it.lines().take(DESCRIPTION_TAKE_LINES).joinToString(" "), han).toSet() } .orEmpty() val confirmedTags = tags.filter { tag -> tag.words.any { it in titleWords || it in descriptionWords } } val tagPhrases = tags.flatMapTo(HashSet(), ::tagPhrases) @@ -2937,8 +2978,11 @@ internal class NeuroTokenizer { * "History Matters" → ["history"] * "央視新聞" → ["新聞"] (央視 = a brand, 新聞 = news) */ - fun tokenizeChannelName(channelName: String): List { - val rawOriginal = nameWords(channelName) + fun tokenizeChannelName( + channelName: String, + han: HanFold = defaultHan, + ): List { + val rawOriginal = nameWords(channelName, han) if (rawOriginal.isEmpty()) return emptyList() @@ -2968,9 +3012,12 @@ internal class NeuroTokenizer { private fun isChannelTopic(word: String): Boolean = word in alwaysTopical || word in NeuroLanguages.channelTopics /** Words of a channel name; scripts written without spaces are split by the word breaker. */ - private fun nameWords(name: String): List = + private fun nameWords( + name: String, + han: HanFold, + ): List = NeuroText - .words(NeuroText.fold(name)) + .words(fold(name, han), wordBreaker) .map { word -> word.trim { !NeuroText.isWordChar(it) } } .filter { it.length > 1 } @@ -2981,24 +3028,27 @@ internal class NeuroTokenizer { * Does NOT lemmatize — we want raw word stems for prefix matching * against domain context words. */ - private fun buildContextWordList(video: Video): List { + private fun buildContextWordList( + video: Video, + han: HanFold, + ): List { val words = mutableListOf() NeuroText - .fold(video.title) + .fold(video.title, han) .split(WHITESPACE_REGEX) .map { it.trim { c -> !c.isLetterOrDigit() } } .filter { it.length > 1 } .forEach { words.add(it) } - words.addAll(nameWords(video.channelName)) + words.addAll(nameWords(video.channelName, han)) if (!video.description.isNullOrBlank()) { video.description .lines() .take(3) .joinToString(" ") - .let(NeuroText::fold) + .let { NeuroText.fold(it, han) } .split(WHITESPACE_REGEX) .map { it.trim { c -> !c.isLetterOrDigit() } } .filter { it.length > 2 } @@ -3010,7 +3060,7 @@ internal class NeuroTokenizer { if (tags.isNotEmpty()) { tags.take(TAG_MAX_INGEST).forEach { tag -> NeuroText - .fold(tag) + .fold(tag, han) .split(WHITESPACE_REGEX) .map { it.trim { c -> !c.isLetterOrDigit() } } .filter { it.length > 1 } diff --git a/app/src/main/resources/neuro/fold/hant-hans.txt b/app/src/main/resources/neuro/fold/hant-hans.txt new file mode 100644 index 0000000000..8f29f347df --- /dev/null +++ b/app/src/main/resources/neuro/fold/hant-hans.txt @@ -0,0 +1,2966 @@ +# OpenCC ba09ffb8353f TSCharacters.txt (Apache-2.0) | scripts/neuro/gen_hant_hans.py | traditional, simplified +㑯㑔 +㑳㑇 +㑶㐹 +㓨刾 +㘚㘎 +㜄㚯 +㜏㛣 +㠏㟆 +㥮㤘 +㩜㨫 +㩳㧐 +㩵擜 +䁻䀥 +䃮鿎 +䊷䌶 +䋙䌺 +䋚䌻 +䋹䌿 +䋻䌾 +䍦䍠 +䎱䎬 +䙡䙌 +䜀䜧 +䝼䞍 +䥇䦂 +䥑鿏 +䥱䥾 +䦛䦶 +䦟䦷 +䯀䯅 +䰾鲃 +䱷䲣 +䱽䲝 +䲁鳚 +䲘鳤 +䴉鹮 +丟丢 +並并 +乾干 +亂乱 +亙亘 +亞亚 +佇伫 +佈布 +佔占 +併并 +來来 +侖仑 +侶侣 +侷局 +俁俣 +係系 +俔伣 +俠侠 +俥伡 +俬私 +倀伥 +倆俩 +倈俫 +倉仓 +個个 +們们 +倖幸 +倫伦 +倲㑈 +偉伟 +偑㐽 +側侧 +偵侦 +偽伪 +傌㐷 +傑杰 +傖伧 +傘伞 +備备 +傢家 +傭佣 +傯偬 +傳传 +傴伛 +債债 +傷伤 +傾倾 +僂偻 +僅仅 +僉佥 +僑侨 +僕仆 +僞伪 +僥侥 +僨偾 +僱雇 +價价 +儀仪 +儁俊 +儂侬 +億亿 +儈侩 +儉俭 +儎傤 +儐傧 +儔俦 +儕侪 +儘尽 +償偿 +優优 +儲储 +儷俪 +儸㑩 +儺傩 +儻傥 +儼俨 +兇凶 +兌兑 +兒儿 +兗兖 +內内 +兩两 +冊册 +冑胄 +冪幂 +凈净 +凍冻 +凜凛 +凱凯 +別别 +刪删 +剄刭 +則则 +剋克 +剎刹 +剗刬 +剛刚 +剝剥 +剮剐 +剴剀 +創创 +剷铲 +劃划 +劄札 +劇剧 +劉刘 +劊刽 +劌刿 +劍剑 +劏㓥 +劑剂 +劚㔉 +勁劲 +動动 +務务 +勛勋 +勝胜 +勞劳 +勢势 +勩勚 +勱劢 +勳勋 +勵励 +勸劝 +勻匀 +匭匦 +匯汇 +匱匮 +區区 +協协 +卹恤 +卻却 +卽即 +厙厍 +厠厕 +厤历 +厭厌 +厲厉 +厴厣 +參参 +叄叁 +叢丛 +吒咤 +吳吴 +吶呐 +呂吕 +咼呙 +員员 +唄呗 +唸念 +問问 +啓启 +啞哑 +啟启 +啢唡 +喎㖞 +喚唤 +喪丧 +喫吃 +喬乔 +單单 +喲哟 +嗆呛 +嗇啬 +嗊唝 +嗎吗 +嗚呜 +嗩唢 +嗶哔 +嘆叹 +嘍喽 +嘓啯 +嘔呕 +嘖啧 +嘗尝 +嘜唛 +嘩哗 +嘮唠 +嘯啸 +嘰叽 +嘵哓 +嘸呒 +嘽啴 +噁恶 +噓嘘 +噚㖊 +噝咝 +噠哒 +噥哝 +噦哕 +噯嗳 +噲哙 +噴喷 +噸吨 +噹当 +嚀咛 +嚇吓 +嚌哜 +嚐尝 +嚕噜 +嚙啮 +嚥咽 +嚦呖 +嚨咙 +嚮向 +嚲亸 +嚳喾 +嚴严 +嚶嘤 +囀啭 +囁嗫 +囂嚣 +囅冁 +囈呓 +囉啰 +囌苏 +囑嘱 +囪囱 +圇囵 +國国 +圍围 +園园 +圓圆 +圖图 +團团 +垻坝 +埡垭 +埰采 +執执 +堅坚 +堊垩 +堖垴 +堝埚 +堯尧 +報报 +場场 +塊块 +塋茔 +塏垲 +塒埘 +塗涂 +塚冢 +塢坞 +塤埙 +塵尘 +塹堑 +墊垫 +墜坠 +墮堕 +墰坛 +墳坟 +墶垯 +墻墙 +墾垦 +壇坛 +壋垱 +壎埙 +壓压 +壘垒 +壙圹 +壚垆 +壜坛 +壞坏 +壟垄 +壠垅 +壢坜 +壩坝 +壪塆 +壯壮 +壺壶 +壼壸 +壽寿 +夠够 +夢梦 +夥伙 +夾夹 +奐奂 +奧奥 +奩奁 +奪夺 +奬奖 +奮奋 +奼姹 +妝妆 +姍姗 +姦奸 +娛娱 +婁娄 +婦妇 +婭娅 +媧娲 +媯妫 +媰㛀 +媼媪 +媽妈 +嫋袅 +嫗妪 +嫵妩 +嫺娴 +嫻娴 +嫿婳 +嬀妫 +嬃媭 +嬈娆 +嬋婵 +嬌娇 +嬙嫱 +嬡嫒 +嬤嬷 +嬪嫔 +嬰婴 +嬸婶 +孃娘 +孋㛤 +孌娈 +孫孙 +學学 +孿孪 +宮宫 +寀采 +寢寝 +實实 +寧宁 +審审 +寫写 +寬宽 +寵宠 +寶宝 +將将 +專专 +尋寻 +對对 +導导 +尷尴 +屆届 +屍尸 +屓屃 +屜屉 +屢屡 +層层 +屨屦 +屬属 +岡冈 +峯峰 +峴岘 +島岛 +峽峡 +崍崃 +崑昆 +崗岗 +崙仑 +崢峥 +崬岽 +嵐岚 +嵗岁 +嵾㟥 +嶁嵝 +嶄崭 +嶇岖 +嶔嵚 +嶗崂 +嶠峤 +嶢峣 +嶧峄 +嶨峃 +嶮崄 +嶸嵘 +嶺岭 +嶼屿 +嶽岳 +巋岿 +巒峦 +巔巅 +巖岩 +巰巯 +巹卺 +帥帅 +師师 +帳帐 +帶带 +幀帧 +幃帏 +幓㡎 +幗帼 +幘帻 +幟帜 +幣币 +幫帮 +幬帱 +幷并 +幹干 +幾几 +庫库 +廁厕 +廂厢 +廄厩 +廈厦 +廎庼 +廕荫 +廚厨 +廝厮 +廟庙 +廠厂 +廡庑 +廢废 +廣广 +廩廪 +廬庐 +廳厅 +弒弑 +弔吊 +弳弪 +張张 +強强 +彆别 +彈弹 +彌弥 +彎弯 +彔录 +彙汇 +彠彟 +彥彦 +彫雕 +彲彨 +彿佛 +後后 +徑径 +從从 +徠徕 +復复 +徵征 +徹彻 +恆恒 +恥耻 +悅悦 +悞悮 +悵怅 +悶闷 +悽凄 +惡恶 +惱恼 +惲恽 +惻恻 +愛爱 +愜惬 +愨悫 +愴怆 +愷恺 +愾忾 +慄栗 +態态 +慍愠 +慘惨 +慚惭 +慟恸 +慣惯 +慤悫 +慪怄 +慫怂 +慮虑 +慳悭 +慶庆 +慺㥪 +慼戚 +慾欲 +憂忧 +憊惫 +憐怜 +憑凭 +憒愦 +憖慭 +憚惮 +憤愤 +憫悯 +憮怃 +憲宪 +憶忆 +懇恳 +應应 +懌怿 +懍懔 +懞蒙 +懟怼 +懣懑 +懤㤽 +懨恹 +懲惩 +懶懒 +懷怀 +懸悬 +懺忏 +懼惧 +懾慑 +戀恋 +戇戆 +戔戋 +戧戗 +戩戬 +戰战 +戱戯 +戲戏 +戶户 +扞捍 +拋抛 +拚拼 +挩捝 +挱挲 +挾挟 +捨舍 +捫扪 +捱挨 +捲卷 +掃扫 +掄抡 +掆㧏 +掗挜 +掙挣 +掛挂 +採采 +揀拣 +揚扬 +換换 +揮挥 +揯搄 +損损 +搖摇 +搗捣 +搧扇 +搵揾 +搶抢 +摑掴 +摜掼 +摟搂 +摯挚 +摳抠 +摶抟 +摺折 +摻掺 +撈捞 +撏挦 +撐撑 +撓挠 +撝㧑 +撟挢 +撣掸 +撥拨 +撫抚 +撲扑 +撳揿 +撻挞 +撾挝 +撿捡 +擁拥 +擄掳 +擇择 +擊击 +擋挡 +擓㧟 +擔担 +據据 +擠挤 +擡抬 +擣捣 +擬拟 +擯摈 +擰拧 +擱搁 +擲掷 +擴扩 +擷撷 +擺摆 +擻擞 +擼撸 +擽㧰 +擾扰 +攄摅 +攆撵 +攏拢 +攔拦 +攖撄 +攙搀 +攛撺 +攜携 +攝摄 +攢攒 +攣挛 +攤摊 +攪搅 +攬揽 +敎教 +敓敚 +敗败 +敘叙 +敵敌 +數数 +斂敛 +斃毙 +斆敩 +斕斓 +斬斩 +斷断 +於于 +旂旗 +旣既 +昇升 +時时 +晉晋 +晝昼 +暈晕 +暉晖 +暘旸 +暢畅 +暫暂 +曄晔 +曆历 +曇昙 +曉晓 +曏向 +曖暧 +曠旷 +曨昽 +曬晒 +書书 +會会 +朧胧 +朮术 +東东 +枴拐 +柵栅 +柺拐 +査查 +桿杆 +梔栀 +梘枧 +條条 +梟枭 +梲棁 +棄弃 +棊棋 +棖枨 +棗枣 +棟栋 +棡㭎 +棧栈 +棲栖 +棶梾 +椏桠 +椲㭏 +楊杨 +楓枫 +楨桢 +業业 +極极 +榘矩 +榦干 +榪杩 +榮荣 +榲榅 +榿桤 +構构 +槍枪 +槓杠 +槤梿 +槧椠 +槨椁 +槮椮 +槳桨 +槶椢 +槼椝 +樁桩 +樂乐 +樅枞 +樑梁 +樓楼 +標标 +樞枢 +樢㭤 +樣样 +樧榝 +樫㭴 +樳桪 +樸朴 +樹树 +樺桦 +樿椫 +橈桡 +橋桥 +機机 +橢椭 +橫横 +檁檩 +檉柽 +檔档 +檜桧 +檟槚 +檢检 +檣樯 +檮梼 +檯台 +檳槟 +檸柠 +檻槛 +櫃柜 +櫓橹 +櫚榈 +櫛栉 +櫝椟 +櫞橼 +櫟栎 +櫥橱 +櫧槠 +櫨栌 +櫪枥 +櫫橥 +櫬榇 +櫱蘖 +櫳栊 +櫸榉 +櫻樱 +欄栏 +欅榉 +權权 +欏椤 +欒栾 +欖榄 +欞棂 +欽钦 +歎叹 +歐欧 +歟欤 +歡欢 +歲岁 +歷历 +歸归 +歿殁 +殘残 +殞殒 +殤殇 +殨㱮 +殫殚 +殭僵 +殮殓 +殯殡 +殰㱩 +殲歼 +殺杀 +殻壳 +殼壳 +毀毁 +毆殴 +毿毵 +氂牦 +氈毡 +氌氇 +氣气 +氫氢 +氬氩 +氳氲 +氾泛 +汎泛 +汙污 +決决 +沒没 +沖冲 +況况 +泝溯 +洩泄 +洶汹 +浹浃 +涇泾 +涗涚 +涼凉 +淒凄 +淚泪 +淥渌 +淨净 +淩凌 +淪沦 +淵渊 +淶涞 +淺浅 +渙涣 +減减 +渢沨 +渦涡 +測测 +渾浑 +湊凑 +湞浈 +湧涌 +湯汤 +溈沩 +準准 +溝沟 +溫温 +溮浉 +溳涢 +溼湿 +滄沧 +滅灭 +滌涤 +滎荥 +滙汇 +滬沪 +滯滞 +滲渗 +滷卤 +滸浒 +滻浐 +滾滚 +滿满 +漁渔 +漊溇 +漚沤 +漢汉 +漣涟 +漬渍 +漲涨 +漵溆 +漸渐 +漿浆 +潁颍 +潑泼 +潔洁 +潙沩 +潚㴋 +潛潜 +潤润 +潯浔 +潰溃 +潷滗 +潿涠 +澀涩 +澆浇 +澇涝 +澐沄 +澗涧 +澠渑 +澤泽 +澦滪 +澩泶 +澮浍 +澱淀 +澾㳠 +濁浊 +濃浓 +濄㳡 +濕湿 +濘泞 +濚溁 +濛蒙 +濜浕 +濟济 +濤涛 +濧㳔 +濫滥 +濰潍 +濱滨 +濺溅 +濼泺 +濾滤 +瀂澛 +瀅滢 +瀆渎 +瀇㲿 +瀉泻 +瀋沈 +瀏浏 +瀕濒 +瀘泸 +瀝沥 +瀟潇 +瀠潆 +瀦潴 +瀧泷 +瀨濑 +瀰弥 +瀲潋 +瀾澜 +灃沣 +灄滠 +灑洒 +灕漓 +灘滩 +灝灏 +灡㳕 +灣湾 +灤滦 +灧滟 +灩滟 +災灾 +為为 +烏乌 +烴烃 +無无 +煉炼 +煒炜 +煙烟 +煢茕 +煥焕 +煩烦 +煬炀 +煱㶽 +熅煴 +熒荧 +熗炝 +熱热 +熲颎 +熾炽 +燁烨 +燈灯 +燉炖 +燒烧 +燙烫 +燜焖 +營营 +燦灿 +燬毁 +燭烛 +燴烩 +燶㶶 +燻熏 +燼烬 +燾焘 +爍烁 +爐炉 +爛烂 +爭争 +爲为 +爺爷 +爾尔 +牀床 +牆墙 +牘牍 +牴抵 +牽牵 +犖荦 +犛牦 +犢犊 +犧牺 +狀状 +狹狭 +狽狈 +猙狰 +猶犹 +猻狲 +獁犸 +獃呆 +獄狱 +獅狮 +獎奖 +獨独 +獪狯 +獫猃 +獮狝 +獰狞 +獱㺍 +獲获 +獵猎 +獷犷 +獸兽 +獺獭 +獻献 +獼猕 +玀猡 +現现 +琱雕 +琺珐 +琿珲 +瑋玮 +瑒玚 +瑣琐 +瑤瑶 +瑩莹 +瑪玛 +瑲玱 +璉琏 +璡琎 +璣玑 +璦瑷 +璫珰 +璯㻅 +環环 +璵玙 +璸瑸 +璽玺 +璿璇 +瓊琼 +瓏珑 +瓔璎 +瓚瓒 +甌瓯 +甕瓮 +產产 +産产 +畝亩 +畢毕 +畫画 +異异 +畵画 +當当 +疇畴 +疊叠 +痙痉 +痠酸 +痾疴 +瘂痖 +瘋疯 +瘍疡 +瘓痪 +瘞瘗 +瘡疮 +瘧疟 +瘮瘆 +瘲疭 +瘺瘘 +瘻瘘 +療疗 +癆痨 +癇痫 +癉瘅 +癒愈 +癘疠 +癟瘪 +癡痴 +癢痒 +癤疖 +癥症 +癧疬 +癩癞 +癬癣 +癭瘿 +癮瘾 +癰痈 +癱瘫 +癲癫 +發发 +皁皂 +皚皑 +皰疱 +皸皲 +皺皱 +盃杯 +盜盗 +盞盏 +盡尽 +監监 +盤盘 +盧卢 +盪荡 +眞真 +眥眦 +眾众 +睏困 +睜睁 +睞睐 +瞘眍 +瞜䁖 +瞞瞒 +瞶瞆 +瞼睑 +矇蒙 +矓眬 +矚瞩 +矯矫 +硃朱 +硜硁 +硤硖 +硨砗 +硯砚 +碕埼 +碩硕 +碭砀 +碸砜 +確确 +碼码 +碽䂵 +磑硙 +磚砖 +磠硵 +磣碜 +磧碛 +磯矶 +磽硗 +磾䃅 +礄硚 +礎础 +礙碍 +礦矿 +礪砺 +礫砾 +礬矾 +礱砻 +祕秘 +祿禄 +禍祸 +禎祯 +禕祎 +禡祃 +禦御 +禪禅 +禮礼 +禰祢 +禱祷 +禿秃 +秈籼 +稅税 +稈秆 +稏䅉 +稜棱 +稟禀 +種种 +稱称 +穀谷 +穇䅟 +穌稣 +積积 +穎颖 +穠秾 +穡穑 +穢秽 +穩稳 +穫获 +穭穞 +窩窝 +窪洼 +窮穷 +窯窑 +窵窎 +窶窭 +窺窥 +竄窜 +竅窍 +竇窦 +竈灶 +竊窃 +竪竖 +競竞 +筆笔 +筍笋 +筧笕 +筴䇲 +箇个 +箋笺 +箏筝 +箚札 +節节 +範范 +築筑 +篋箧 +篔筼 +篠筿 +篤笃 +篩筛 +篳筚 +簀箦 +簍篓 +簑蓑 +簞箪 +簡简 +簣篑 +簫箫 +簹筜 +簽签 +簾帘 +籃篮 +籌筹 +籔䉤 +籙箓 +籛篯 +籜箨 +籟籁 +籠笼 +籤签 +籩笾 +籪簖 +籬篱 +籮箩 +籲吁 +粵粤 +糉粽 +糝糁 +糞粪 +糧粮 +糰团 +糲粝 +糴籴 +糶粜 +糹纟 +糾纠 +紀纪 +紂纣 +約约 +紅红 +紆纡 +紇纥 +紈纨 +紉纫 +紋纹 +納纳 +紐纽 +紓纾 +純纯 +紕纰 +紖纼 +紗纱 +紘纮 +紙纸 +級级 +紛纷 +紜纭 +紝纴 +紡纺 +紬䌷 +紮扎 +細细 +紱绂 +紲绁 +紳绅 +紵纻 +紹绍 +紺绀 +紼绋 +紿绐 +絀绌 +終终 +絃弦 +組组 +絅䌹 +絆绊 +絎绗 +結结 +絕绝 +絛绦 +絝绔 +絞绞 +絡络 +絢绚 +給给 +絨绒 +絰绖 +統统 +絲丝 +絳绛 +絶绝 +絹绢 +綁绑 +綃绡 +綆绠 +綈绨 +綉绣 +綌绤 +綏绥 +綐䌼 +綑捆 +經经 +綜综 +綞缍 +綠绿 +綢绸 +綣绻 +綫线 +綬绶 +維维 +綯绹 +綰绾 +綱纲 +網网 +綳绷 +綴缀 +綵彩 +綸纶 +綹绺 +綺绮 +綻绽 +綽绰 +綾绫 +綿绵 +緄绲 +緇缁 +緊紧 +緋绯 +緑绿 +緒绪 +緓绬 +緔绱 +緗缃 +緘缄 +緙缂 +線线 +緝缉 +緞缎 +締缔 +緡缗 +緣缘 +緦缌 +編编 +緩缓 +緬缅 +緯纬 +緱缑 +緲缈 +練练 +緶缏 +緹缇 +緻致 +緼缊 +縈萦 +縉缙 +縊缢 +縋缒 +縐绉 +縑缣 +縕缊 +縗缞 +縛缚 +縝缜 +縞缟 +縟缛 +縣县 +縧绦 +縫缝 +縭缡 +縮缩 +縱纵 +縲缧 +縳䌸 +縴纤 +縵缦 +縶絷 +縷缕 +縹缥 +總总 +績绩 +繃绷 +繅缫 +繆缪 +繒缯 +織织 +繕缮 +繚缭 +繞绕 +繡绣 +繢缋 +繩绳 +繪绘 +繫系 +繭茧 +繮缰 +繯缳 +繰缲 +繳缴 +繸䍁 +繹绎 +繼继 +繽缤 +繾缱 +繿䍀 +纇颣 +纈缬 +纊纩 +續续 +纍累 +纏缠 +纓缨 +纔才 +纖纤 +纘缵 +纜缆 +缽钵 +罃䓨 +罈坛 +罌罂 +罎坛 +罰罚 +罵骂 +罷罢 +羅罗 +羆罴 +羈羁 +羋芈 +羣群 +羥羟 +羨羡 +義义 +羶膻 +習习 +翫玩 +翬翚 +翹翘 +翽翙 +耬耧 +耮耢 +聖圣 +聞闻 +聯联 +聰聪 +聲声 +聳耸 +聵聩 +聶聂 +職职 +聹聍 +聽听 +聾聋 +肅肃 +脅胁 +脈脉 +脛胫 +脣唇 +脩修 +脫脱 +脹胀 +腎肾 +腖胨 +腡脶 +腦脑 +腫肿 +腳脚 +腸肠 +膃腽 +膕腘 +膚肤 +膞䏝 +膠胶 +膩腻 +膽胆 +膾脍 +膿脓 +臉脸 +臍脐 +臏膑 +臘腊 +臚胪 +臟脏 +臠脔 +臢臜 +臥卧 +臨临 +臺台 +與与 +興兴 +舉举 +舊旧 +舖铺 +舘馆 +艙舱 +艤舣 +艦舰 +艫舻 +艱艰 +艷艳 +芻刍 +苧苎 +茲兹 +荊荆 +莊庄 +莖茎 +莢荚 +莧苋 +華华 +菴庵 +菸烟 +萇苌 +萊莱 +萬万 +萴荝 +萵莴 +葉叶 +葒荭 +葤荮 +葦苇 +葯药 +葷荤 +蒐搜 +蒓莼 +蒔莳 +蒕蒀 +蒞莅 +蒼苍 +蓀荪 +蓆席 +蓋盖 +蓮莲 +蓯苁 +蓴莼 +蓽荜 +蔔卜 +蔘参 +蔞蒌 +蔣蒋 +蔥葱 +蔦茑 +蔭荫 +蕁荨 +蕆蒇 +蕎荞 +蕒荬 +蕓芸 +蕕莸 +蕘荛 +蕢蒉 +蕩荡 +蕪芜 +蕭萧 +蕷蓣 +薀蕰 +薈荟 +薊蓟 +薌芗 +薑姜 +薔蔷 +薘荙 +薟莶 +薦荐 +薩萨 +薴苎 +薵䓓 +薹苔 +薺荠 +藍蓝 +藎荩 +藝艺 +藥药 +藪薮 +藭䓖 +藴蕴 +藶苈 +藹蔼 +藺蔺 +蘀萚 +蘄蕲 +蘆芦 +蘇苏 +蘊蕴 +蘋苹 +蘚藓 +蘞蔹 +蘢茏 +蘭兰 +蘺蓠 +蘿萝 +虆蔂 +處处 +虛虚 +虜虏 +號号 +虧亏 +虯虬 +蛺蛱 +蛻蜕 +蜆蚬 +蝕蚀 +蝟猬 +蝦虾 +蝨虱 +蝸蜗 +螄蛳 +螞蚂 +螢萤 +螮䗖 +螻蝼 +螿螀 +蟄蛰 +蟈蝈 +蟎螨 +蟣虮 +蟬蝉 +蟯蛲 +蟲虫 +蟶蛏 +蟻蚁 +蠁蚃 +蠅蝇 +蠆虿 +蠍蝎 +蠐蛴 +蠑蝾 +蠔蚝 +蠟蜡 +蠣蛎 +蠨蟏 +蠱蛊 +蠶蚕 +蠻蛮 +衆众 +衊蔑 +術术 +衕同 +衚胡 +衛卫 +衝冲 +袞衮 +袷夹 +裊袅 +裏里 +補补 +裝装 +裡里 +製制 +複复 +褌裈 +褘袆 +褲裤 +褳裢 +褸褛 +褻亵 +襇裥 +襉裥 +襏袯 +襖袄 +襝裣 +襠裆 +襤褴 +襪袜 +襬摆 +襯衬 +襲袭 +襴襕 +覈核 +見见 +覎觃 +規规 +覓觅 +視视 +覘觇 +覡觋 +覥觍 +覦觎 +親亲 +覬觊 +覯觏 +覲觐 +覷觑 +覺觉 +覽览 +覿觌 +觀观 +觴觞 +觶觯 +觸触 +訁讠 +訂订 +訃讣 +計计 +訊讯 +訌讧 +討讨 +訐讦 +訒讱 +訓训 +訕讪 +訖讫 +託托 +記记 +訛讹 +訝讶 +訟讼 +訢䜣 +訣诀 +訥讷 +訩讻 +訪访 +設设 +許许 +訴诉 +訶诃 +診诊 +註注 +証证 +詁诂 +詆诋 +詎讵 +詐诈 +詒诒 +詔诏 +評评 +詖诐 +詗诇 +詘诎 +詛诅 +詞词 +詠咏 +詡诩 +詢询 +詣诣 +試试 +詩诗 +詫诧 +詬诟 +詭诡 +詮诠 +詰诘 +話话 +該该 +詳详 +詵诜 +詼诙 +詿诖 +誄诔 +誅诛 +誆诓 +誇夸 +誌志 +認认 +誑诳 +誒诶 +誕诞 +誘诱 +誚诮 +語语 +誠诚 +誡诫 +誣诬 +誤误 +誥诰 +誦诵 +誨诲 +說说 +説说 +誰谁 +課课 +誶谇 +誹诽 +誼谊 +誾訚 +調调 +諂谄 +諄谆 +談谈 +諉诿 +請请 +諍诤 +諏诹 +諑诼 +諒谅 +論论 +諗谂 +諛谀 +諜谍 +諝谞 +諞谝 +諡谥 +諢诨 +諤谔 +諦谛 +諧谐 +諫谏 +諭谕 +諮咨 +諱讳 +諳谙 +諶谌 +諷讽 +諸诸 +諺谚 +諼谖 +諾诺 +謀谋 +謁谒 +謂谓 +謄誊 +謅诌 +謊谎 +謎谜 +謐谧 +謔谑 +謖谡 +謗谤 +謙谦 +謚谥 +講讲 +謝谢 +謠谣 +謡谣 +謨谟 +謫谪 +謬谬 +謭谫 +謳讴 +謹谨 +謾谩 +譁哗 +證证 +譎谲 +譏讥 +譖谮 +識识 +譙谯 +譚谭 +譜谱 +譟噪 +譫谵 +譭毁 +譯译 +議议 +譴谴 +護护 +譸诪 +譽誉 +譾谫 +讀读 +讅谉 +變变 +讋詟 +讌䜩 +讎雠 +讒谗 +讓让 +讕谰 +讖谶 +讚赞 +讜谠 +讞谳 +谿溪 +豈岂 +豎竖 +豐丰 +豔艳 +豬猪 +豶豮 +貍狸 +貓猫 +貙䝙 +貝贝 +貞贞 +貟贠 +負负 +財财 +貢贡 +貧贫 +貨货 +販贩 +貪贪 +貫贯 +責责 +貯贮 +貰贳 +貲赀 +貳贰 +貴贵 +貶贬 +買买 +貸贷 +貺贶 +費费 +貼贴 +貽贻 +貿贸 +賀贺 +賁贲 +賂赂 +賃赁 +賄贿 +賅赅 +資资 +賈贾 +賊贼 +賑赈 +賒赊 +賓宾 +賕赇 +賙赒 +賚赉 +賜赐 +賞赏 +賠赔 +賡赓 +賢贤 +賣卖 +賤贱 +賦赋 +賧赕 +質质 +賫赍 +賬账 +賭赌 +賰䞐 +賴赖 +賵赗 +賺赚 +賻赙 +購购 +賽赛 +賾赜 +贄贽 +贅赘 +贇赟 +贈赠 +贊赞 +贋赝 +贍赡 +贏赢 +贐赆 +贓赃 +贔赑 +贖赎 +贗赝 +贛赣 +贜赃 +赬赪 +趕赶 +趙赵 +趨趋 +趲趱 +跡迹 +踐践 +踰逾 +踴踊 +蹌跄 +蹕跸 +蹟迹 +蹠跖 +蹣蹒 +蹤踪 +蹺跷 +躂跶 +躉趸 +躊踌 +躋跻 +躍跃 +躎䟢 +躑踯 +躒跞 +躓踬 +躕蹰 +躚跹 +躡蹑 +躥蹿 +躦躜 +躪躏 +軀躯 +車车 +軋轧 +軌轨 +軍军 +軑轪 +軒轩 +軔轫 +軛轭 +軟软 +軤轷 +軫轸 +軲轱 +軸轴 +軹轵 +軺轺 +軻轲 +軼轶 +軾轼 +較较 +輅辂 +輇辁 +輈辀 +載载 +輊轾 +輒辄 +輓挽 +輔辅 +輕轻 +輛辆 +輜辎 +輝辉 +輞辋 +輟辍 +輥辊 +輦辇 +輩辈 +輪轮 +輬辌 +輯辑 +輳辏 +輸输 +輻辐 +輼辒 +輾辗 +輿舆 +轀辒 +轂毂 +轄辖 +轅辕 +轆辘 +轉转 +轍辙 +轎轿 +轔辚 +轟轰 +轡辔 +轢轹 +轤轳 +辦办 +辭辞 +辮辫 +辯辩 +農农 +迴回 +逕径 +這这 +連连 +週周 +進进 +遊游 +運运 +過过 +達达 +違违 +遙遥 +遜逊 +遞递 +遠远 +遡溯 +適适 +遲迟 +遶绕 +遷迁 +選选 +遺遗 +遼辽 +邁迈 +還还 +邇迩 +邊边 +邏逻 +邐逦 +郟郏 +郵邮 +鄆郓 +鄉乡 +鄒邹 +鄔邬 +鄖郧 +鄧邓 +鄭郑 +鄰邻 +鄲郸 +鄴邺 +鄶郐 +鄺邝 +酇酂 +酈郦 +醃腌 +醖酝 +醜丑 +醞酝 +醟蒏 +醣糖 +醫医 +醬酱 +醱酦 +釀酿 +釁衅 +釃酾 +釅酽 +釋释 +釐厘 +釒钅 +釓钆 +釔钇 +釕钌 +釗钊 +釘钉 +釙钋 +針针 +釣钓 +釤钐 +釦扣 +釧钏 +釩钒 +釵钗 +釷钍 +釹钕 +釺钎 +釾䥺 +鈀钯 +鈁钫 +鈃钘 +鈄钭 +鈅钥 +鈈钚 +鈉钠 +鈍钝 +鈎钩 +鈐钤 +鈑钣 +鈒钑 +鈔钞 +鈕钮 +鈞钧 +鈡钟 +鈣钙 +鈥钬 +鈦钛 +鈧钪 +鈮铌 +鈰铈 +鈳钶 +鈴铃 +鈷钴 +鈸钹 +鈹铍 +鈺钰 +鈽钸 +鈾铀 +鈿钿 +鉀钾 +鉅巨 +鉆钻 +鉈铊 +鉉铉 +鉋铇 +鉍铋 +鉑铂 +鉕钷 +鉗钳 +鉚铆 +鉛铅 +鉞钺 +鉢钵 +鉤钩 +鉦钲 +鉬钼 +鉭钽 +鉳锫 +鉶铏 +鉸铰 +鉺铒 +鉻铬 +鉿铪 +銀银 +銃铳 +銅铜 +銍铚 +銑铣 +銓铨 +銖铢 +銘铭 +銚铫 +銛铦 +銜衔 +銠铑 +銣铷 +銥铱 +銦铟 +銨铵 +銩铥 +銪铕 +銫铯 +銬铐 +銱铞 +銳锐 +銷销 +銹锈 +銻锑 +銼锉 +鋁铝 +鋃锒 +鋅锌 +鋇钡 +鋌铤 +鋏铗 +鋒锋 +鋙铻 +鋝锊 +鋟锓 +鋣铘 +鋤锄 +鋥锃 +鋦锔 +鋨锇 +鋩铓 +鋪铺 +鋭锐 +鋮铖 +鋯锆 +鋰锂 +鋱铽 +鋶锍 +鋸锯 +鋼钢 +錁锞 +錄录 +錆锖 +錇锫 +錈锩 +錏铔 +錐锥 +錒锕 +錕锟 +錘锤 +錙锱 +錚铮 +錛锛 +錟锬 +錠锭 +錡锜 +錢钱 +錦锦 +錨锚 +錩锠 +錫锡 +錮锢 +錯错 +録录 +錳锰 +錶表 +錸铼 +錼镎 +鍀锝 +鍁锨 +鍃锪 +鍅钫 +鍆钔 +鍇锴 +鍈锳 +鍊炼 +鍋锅 +鍍镀 +鍔锷 +鍘铡 +鍚钖 +鍛锻 +鍠锽 +鍤锸 +鍥锲 +鍩锘 +鍬锹 +鍰锾 +鍵键 +鍶锶 +鍺锗 +鍼针 +鍾钟 +鎂镁 +鎄锿 +鎇镅 +鎊镑 +鎌镰 +鎔镕 +鎖锁 +鎘镉 +鎚锤 +鎛镈 +鎡镃 +鎢钨 +鎣蓥 +鎦镏 +鎧铠 +鎩铩 +鎪锼 +鎬镐 +鎭镇 +鎮镇 +鎰镒 +鎲镋 +鎳镍 +鎵镓 +鎶鿔 +鎸镌 +鎿镎 +鏃镞 +鏇旋 +鏈链 +鏌镆 +鏍镙 +鏐镠 +鏑镝 +鏗铿 +鏘锵 +鏜镗 +鏝镘 +鏞镛 +鏟铲 +鏡镜 +鏢镖 +鏤镂 +鏨錾 +鏰镚 +鏵铧 +鏷镤 +鏹镪 +鏺䥽 +鏽锈 +鐃铙 +鐋铴 +鐐镣 +鐒铹 +鐓镦 +鐔镡 +鐘钟 +鐙镫 +鐝镢 +鐠镨 +鐥䦅 +鐦锎 +鐧锏 +鐨镄 +鐫镌 +鐮镰 +鐯䦃 +鐲镯 +鐳镭 +鐵铁 +鐶镮 +鐸铎 +鐺铛 +鐿镱 +鑄铸 +鑊镬 +鑌镔 +鑑鉴 +鑒鉴 +鑔镲 +鑕锧 +鑞镴 +鑠铄 +鑣镳 +鑥镥 +鑭镧 +鑰钥 +鑱镵 +鑲镶 +鑷镊 +鑹镩 +鑼锣 +鑽钻 +鑾銮 +鑿凿 +钁镢 +钂镋 +長长 +門门 +閂闩 +閃闪 +閆闫 +閈闬 +閉闭 +開开 +閌闶 +閎闳 +閏闰 +閑闲 +閒闲 +間间 +閔闵 +閘闸 +閡阂 +閣阁 +閤合 +閥阀 +閨闺 +閩闽 +閫阃 +閬阆 +閭闾 +閱阅 +閲阅 +閶阊 +閹阉 +閻阎 +閼阏 +閽阍 +閾阈 +閿阌 +闃阒 +闆板 +闇暗 +闈闱 +闊阔 +闋阕 +闌阑 +闍阇 +闐阗 +闒阘 +闓闿 +闔阖 +闕阙 +闖闯 +關关 +闞阚 +闠阓 +闡阐 +闢辟 +闤阛 +闥闼 +陘陉 +陝陕 +陞升 +陣阵 +陰阴 +陳陈 +陸陆 +陽阳 +隉陧 +隊队 +階阶 +隕陨 +際际 +隨随 +險险 +隯陦 +隱隐 +隴陇 +隸隶 +隻只 +雋隽 +雖虽 +雙双 +雛雏 +雜杂 +雞鸡 +離离 +難难 +雲云 +電电 +霑沾 +霢霡 +霧雾 +霽霁 +靂雳 +靄霭 +靆叇 +靈灵 +靉叆 +靚靓 +靜静 +靝靔 +靦腼 +靨靥 +鞏巩 +鞝绱 +鞦秋 +鞽鞒 +韁缰 +韃鞑 +韆千 +韉鞯 +韋韦 +韌韧 +韍韨 +韓韩 +韙韪 +韜韬 +韝鞲 +韞韫 +韻韵 +響响 +頁页 +頂顶 +頃顷 +項项 +順顺 +頇顸 +須须 +頊顼 +頌颂 +頎颀 +頏颃 +預预 +頑顽 +頒颁 +頓顿 +頗颇 +領领 +頜颌 +頡颉 +頤颐 +頦颏 +頭头 +頮颒 +頰颊 +頲颋 +頴颕 +頷颔 +頸颈 +頹颓 +頻频 +頽颓 +顆颗 +題题 +額额 +顎颚 +顏颜 +顒颙 +顓颛 +顔颜 +願愿 +顙颡 +顛颠 +類类 +顢颟 +顥颢 +顧顾 +顫颤 +顬颥 +顯显 +顰颦 +顱颅 +顳颞 +顴颧 +風风 +颭飐 +颮飑 +颯飒 +颱台 +颳刮 +颶飓 +颸飔 +颺飏 +颻飖 +颼飕 +飀飗 +飄飘 +飆飙 +飈飚 +飛飞 +飠饣 +飢饥 +飣饤 +飥饦 +飩饨 +飪饪 +飫饫 +飭饬 +飯饭 +飱飧 +飲饮 +飴饴 +飼饲 +飽饱 +飾饰 +飿饳 +餃饺 +餄饸 +餅饼 +餈糍 +餉饷 +養养 +餌饵 +餎饹 +餏饻 +餑饽 +餒馁 +餓饿 +餕馂 +餖饾 +餘余 +餚肴 +餛馄 +餜馃 +餞饯 +餡馅 +館馆 +餬糊 +餱糇 +餳饧 +餵喂 +餶馉 +餷馇 +餺馎 +餼饩 +餾馏 +餿馊 +饁馌 +饃馍 +饅馒 +饈馐 +饉馑 +饊馓 +饋馈 +饌馔 +饑饥 +饒饶 +饗飨 +饜餍 +饞馋 +饢馕 +馬马 +馭驭 +馮冯 +馱驮 +馳驰 +馴驯 +馹驲 +駁驳 +駐驻 +駑驽 +駒驹 +駔驵 +駕驾 +駘骀 +駙驸 +駛驶 +駝驼 +駟驷 +駡骂 +駢骈 +駭骇 +駰骃 +駱骆 +駸骎 +駿骏 +騁骋 +騂骍 +騅骓 +騌骔 +騍骒 +騎骑 +騏骐 +騖骛 +騙骗 +騤骙 +騧䯄 +騫骞 +騭骘 +騮骝 +騰腾 +騶驺 +騷骚 +騸骟 +騾骡 +驀蓦 +驁骜 +驂骖 +驃骠 +驄骢 +驅驱 +驊骅 +驌骕 +驍骁 +驏骣 +驕骄 +驗验 +驚惊 +驛驿 +驟骤 +驢驴 +驤骧 +驥骥 +驦骦 +驪骊 +驫骉 +骯肮 +髏髅 +髒脏 +體体 +髕髌 +髖髋 +髮发 +鬆松 +鬍胡 +鬚须 +鬢鬓 +鬥斗 +鬧闹 +鬨哄 +鬩阋 +鬮阄 +鬱郁 +鬹鬶 +魎魉 +魘魇 +魚鱼 +魛鱽 +魢鱾 +魨鲀 +魯鲁 +魴鲂 +魷鱿 +魺鲄 +鮁鲅 +鮃鲆 +鮊鲌 +鮋鲉 +鮍鲏 +鮎鲇 +鮐鲐 +鮑鲍 +鮒鲋 +鮓鲊 +鮚鲒 +鮜鲘 +鮝鲞 +鮞鲕 +鮣䲟 +鮦鲖 +鮪鲔 +鮫鲛 +鮭鲑 +鮮鲜 +鮳鲓 +鮶鲪 +鮺鲝 +鯀鲧 +鯁鲠 +鯇鲩 +鯉鲤 +鯊鲨 +鯒鲬 +鯔鲻 +鯕鲯 +鯖鲭 +鯗鲞 +鯛鲷 +鯝鲴 +鯡鲱 +鯢鲵 +鯤鲲 +鯧鲳 +鯨鲸 +鯪鲮 +鯫鲰 +鯰鲶 +鯴鲺 +鯷鳀 +鯽鲫 +鯿鳊 +鰁鳈 +鰂鲗 +鰃鳂 +鰆䲠 +鰈鲽 +鰉鳇 +鰌䲡 +鰍鳅 +鰏鲾 +鰐鳄 +鰒鳆 +鰓鳃 +鰛鳁 +鰜鳒 +鰟鳑 +鰠鳋 +鰣鲥 +鰥鳏 +鰧䲢 +鰨鳎 +鰩鳐 +鰭鳍 +鰮鳁 +鰱鲢 +鰲鳌 +鰳鳓 +鰵鳘 +鰷鲦 +鰹鲣 +鰺鲹 +鰻鳗 +鰼鳛 +鰾鳔 +鱂鳉 +鱅鳙 +鱈鳕 +鱉鳖 +鱒鳟 +鱔鳝 +鱖鳜 +鱗鳞 +鱘鲟 +鱝鲼 +鱟鲎 +鱠鲙 +鱣鳣 +鱤鳡 +鱧鳢 +鱨鲿 +鱭鲚 +鱯鳠 +鱷鳄 +鱸鲈 +鱺鲡 +鳥鸟 +鳧凫 +鳩鸠 +鳬凫 +鳲鸤 +鳳凤 +鳴鸣 +鳶鸢 +鳾䴓 +鴆鸩 +鴇鸨 +鴉鸦 +鴒鸰 +鴕鸵 +鴛鸳 +鴝鸲 +鴞鸮 +鴟鸱 +鴣鸪 +鴦鸯 +鴨鸭 +鴯鸸 +鴰鸹 +鴴鸻 +鴷䴕 +鴻鸿 +鴿鸽 +鵁䴔 +鵂鸺 +鵃鸼 +鵐鹀 +鵑鹃 +鵒鹆 +鵓鹁 +鵜鹈 +鵝鹅 +鵠鹄 +鵡鹉 +鵪鹌 +鵬鹏 +鵮鹐 +鵯鹎 +鵰雕 +鵲鹊 +鵷鹓 +鵾鹍 +鶄䴖 +鶇鸫 +鶉鹑 +鶊鹒 +鶓鹋 +鶖鹙 +鶘鹕 +鶚鹗 +鶡鹖 +鶥鹛 +鶩鹜 +鶪䴗 +鶬鸧 +鶯莺 +鶲鹟 +鶴鹤 +鶹鹠 +鶺鹡 +鶻鹘 +鶼鹣 +鶿鹚 +鷀鹚 +鷁鹢 +鷂鹞 +鷄鸡 +鷉䴘 +鷊鹝 +鷓鹧 +鷖鹥 +鷗鸥 +鷙鸷 +鷚鹨 +鷥鸶 +鷦鹪 +鷫鹔 +鷯鹩 +鷲鹫 +鷳鹇 +鷴鹇 +鷸鹬 +鷹鹰 +鷺鹭 +鷽鸴 +鸂㶉 +鸇鹯 +鸊䴙 +鸌鹱 +鸏鹲 +鸕鸬 +鸘鹴 +鸚鹦 +鸛鹳 +鸝鹂 +鸞鸾 +鹵卤 +鹹咸 +鹺鹾 +鹼碱 +鹽盐 +麗丽 +麥麦 +麩麸 +麪面 +麫面 +麯曲 +麴曲 +麵面 +麼么 +麽么 +黃黄 +黌黉 +點点 +黨党 +黲黪 +黴霉 +黶黡 +黷黩 +黽黾 +黿鼋 +鼂鼌 +鼉鼍 +鼕冬 +鼴鼹 +齊齐 +齋斋 +齎赍 +齏齑 +齒齿 +齔龀 +齕龁 +齗龂 +齙龅 +齜龇 +齟龃 +齠龆 +齡龄 +齣出 +齦龈 +齧啮 +齪龊 +齬龉 +齲龋 +齶腭 +齷龌 +龍龙 +龎厐 +龐庞 +龑䶮 +龔龚 +龕龛 +龜龟 +鿁䜤 +鿓鿒 diff --git a/app/src/main/resources/neuro/freq/zh.txt b/app/src/main/resources/neuro/freq/zh.txt new file mode 100644 index 0000000000..1871bba3df --- /dev/null +++ b/app/src/main/resources/neuro/freq/zh.txt @@ -0,0 +1,1737 @@ +我们 70 +什么 68 +知道 67 +他们 66 +一个 65 +没有 65 +不是 65 +you 64 +可以 64 +你们 64 +现在 64 +the 64 +这个 64 +怎么 63 +就是 63 +如果 63 +这里 63 +这样 63 +to 62 +自己 62 +不会 62 +告诉 62 +为什么 62 +不能 62 +真的 62 +先生 62 +只是 61 +不要 61 +已经 61 +需要 61 +it 61 +因为 61 +谢谢 61 +这么 61 +可能 61 +但是 61 +那个 61 +所以 61 +喜欢 61 +我要 60 +觉得 60 +还有 60 +时候 60 +东西 60 +一起 60 +and 60 +应该 60 +我們 60 +这些 60 +that 60 +孩子 60 +看到 60 +一下 60 +那么 60 +of 59 +还是 59 +问题 59 +工作 59 +一样 59 +开始 59 +然后 59 +看看 59 +地方 59 +一定 59 +希望 59 +朋友 59 +今天 59 +in 59 +事情 59 +一直 59 +所有 59 +we 59 +找到 59 +什麼 59 +当然 58 +回来 58 +不想 58 +is 58 +那些 58 +明白 58 +的话 58 +爸爸 58 +what 58 +离开 58 +一切 58 +时间 58 +相信 58 +chffffff 58 +妈妈 58 +想要 58 +发生 58 +起来 58 +me 58 +真是 58 +你好 58 +抱歉 58 +必须 58 +认为 58 +一点 58 +非常 58 +那里 58 +对不起 58 +也许 58 +大家 58 +哪里 58 +有人 58 +意思 58 +准备 57 +出来 57 +this 57 +不过 57 +一些 57 +为了 57 +女人 57 +he 57 +pos 57 +这儿 57 +只有 57 +过来 57 +一次 57 +得到 57 +很多 57 +on 57 +任何 57 +这种 57 +for 57 +感觉 57 +是不是 57 +以为 57 +fn 57 +而且 57 +my 57 +your 57 +生活 57 +男人 57 +之前 57 +发现 57 +可是 57 +不用 56 +re 56 +名字 56 +have 56 +没错 56 +was 56 +no 56 +do 56 +世界 56 +记得 56 +be 56 +不错 56 +肯定 56 +最好 56 +not 56 +不行 56 +等等 56 +出去 56 +亲爱 56 +can 56 +are 56 +don 56 +他們 56 +担心 56 +如何 56 +到底 56 +有点 56 +最后 56 +小姐 56 +上帝 56 +认识 56 +know 56 +正在 56 +那样 56 +父亲 56 +高兴 56 +明天 56 +重要 56 +警察 56 +只要 56 +干什么 56 +好像 56 +再见 55 +我能 55 +继续 55 +今晚 55 +all 55 +他妈的 55 +with 55 +回家 55 +它们 55 +多少 55 +电话 55 +听到 55 +just 55 +but 55 +医生 55 +要是 55 +女孩 55 +so 55 +马上 55 +过去 55 +以前 55 +they 55 +儿子 55 +哪儿 55 +说话 55 +看起来 55 +fnmicrosoft 55 +一個 55 +yahei 55 +完全 55 +该死 55 +there 55 +美国 55 +怎样 55 +回去 55 +其他 55 +永远 55 +家伙 55 +之后 55 +怎麼 55 +看见 55 +或者 55 +小心 55 +fnsimhei 55 +别人 55 +成为 55 +不到 54 +here 54 +情况 54 +get 54 +伙计 54 +一天 54 +见到 54 +晚上 54 +总是 54 +好好 54 +如此 54 +了解 54 +like 54 +兄弟 54 +没什么 54 +我来 54 +结束 54 +机会 54 +安全 54 +拜托 54 +决定 54 +确定 54 +里面 54 +还好 54 +无法 54 +小时 54 +清楚 54 +有些 54 +這個 54 +帮助 54 +长官 54 +out 54 +ll 54 +公司 54 +up 54 +打电话 54 +房子 54 +根本 54 +办法 54 +go 54 +不管 54 +混蛋 54 +关于 54 +right 54 +she 54 +计划 54 +唯一 54 +来说 54 +女儿 54 +one 54 +宝贝 54 +about 54 +其实 54 +漂亮 54 +這樣 54 +if 54 +人们 54 +真正 54 +甚至 54 +那边 54 +不好 54 +him 54 +进来 53 +夫人 53 +at 53 +愿意 53 +麻烦 53 +now 53 +就要 53 +打算 53 +故事 53 +一种 53 +害怕 53 +房间 53 +那儿 53 +衣服 53 +母亲 53 +got 53 +下来 53 +听说 53 +不了 53 +注意 53 +改变 53 +how 53 +这次 53 +没关系 53 +接受 53 +妻子 53 +失去 53 +回到 53 +以后 53 +是否 53 +her 53 +生命 53 +她们 53 +进去 53 +女士 53 +yeah 53 +原因 53 +刚才 53 +保护 53 +选择 53 +几个 53 +关系 53 +从来 53 +will 53 +当时 53 +出现 53 +结婚 53 +至少 53 +oh 53 +成功 53 +come 53 +奇怪 53 +虽然 53 +太好了 53 +身上 53 +或许 53 +你們 53 +保证 53 +外面 53 +ve 53 +丈夫 53 +want 53 +国家 53 +整个 53 +特别 53 +那种 53 +不同 53 +很快 53 +变成 53 +难道 52 +学校 52 +一件 52 +样子 52 +电影 52 +think 52 +危险 52 +开心 52 +别的 52 +才能 52 +进行 52 +照片 52 +刚刚 52 +而已 52 +完成 52 +方式 52 +from 52 +做到 52 +who 52 +因為 52 +声音 52 +第一次 52 +看来 52 +只能 52 +現在 52 +太太 52 +下去 52 +绝对 52 +处理 52 +欢迎 52 +眼睛 52 +通过 52 +行动 52 +感到 52 +最近 52 +as 52 +各位 52 +这边 52 +did 52 +不好意思 52 +作为 52 +see 52 +good 52 +容易 52 +沒有 52 +直到 52 +比赛 52 +像是 52 +his 52 +let 52 +更好 52 +why 52 +有趣 52 +就算 52 +活着 52 +不會 52 +努力 52 +帮忙 52 +留下 52 +同意 52 +参加 52 +well 52 +死亡 52 +上面 52 +结果 52 +消息 52 +要求 52 +理解 52 +曾经 52 +打开 52 +收到 52 +照顾 52 +人类 52 +主意 52 +保持 52 +okay 52 +存在 52 +晚安 52 +确实 52 +全部 52 +謝謝 52 +讨厌 52 +gonna 52 +an 52 +突然 52 +多久 52 +老婆 52 +不再 52 +when 52 +back 52 +任何人 52 +开玩笑 52 +放在 52 +那個 52 +进入 52 +父母 52 +总统 52 +伤害 52 +時候 52 +回答 52 +一位 52 +老师 52 +除了 52 +black 52 +咱们 52 +自由 51 +可爱 51 +us 51 +抓住 51 +简单 51 +time 51 +身体 51 +想法 51 +早上 51 +這麼 51 +能够 51 +昨晚 51 +哥哥 51 +我爱你 51 +them 51 +秘密 51 +休息 51 +战争 51 +其中 51 +医院 51 +每天 51 +之间 51 +大概 51 +控制 51 +小孩 51 +带来 51 +停止 51 +因此 51 +证明 51 +约翰 51 +飞机 51 +解释 51 +律师 51 +终于 51 +翻译 51 +调查 51 +告訴 51 +为何 51 +留在 51 +糟糕 51 +任务 51 +武器 51 +家里 51 +来自 51 +后面 51 +小子 51 +going 51 +来到 51 +解决 51 +would 51 +家人 51 +坐下 51 +where 51 +直接 51 +放弃 51 +加油 51 +方法 51 +游戏 51 +即使 51 +fnarial 51 +take 51 +were 51 +our 51 +年轻 51 +大人 51 +使用 51 +or 51 +老板 51 +想想 51 +坐在 51 +目标 51 +need 51 +变得 51 +可怕 51 +拿到 51 +考虑 51 +个人 51 +开枪 51 +been 51 +睡觉 51 +分钟 51 +想到 51 +等待 51 +音乐 51 +再说 51 +聪明 51 +yes 51 +姑娘 51 +地球 51 +感谢 51 +命令 51 +阻止 51 +一只 51 +消失 51 +喜歡 51 +冷静 51 +似乎 51 +联系 51 +家庭 51 +痛苦 51 +城市 51 +力量 51 +对于 51 +未来 51 +男孩 51 +博士 51 +姐姐 51 +疯狂 51 +报告 51 +放下 51 +昨天 51 +那天 51 +還有 51 +完美 51 +答应 51 +证据 51 +本来 51 +负责 51 +检查 51 +could 51 +已經 51 +一条 51 +man 51 +look 51 +厉害 51 +拥有 51 +正常 51 +并且 51 +否则 51 +then 50 +以及 50 +重新 50 +忘记 50 +had 50 +这位 50 +研究 50 +附近 50 +不然 50 +方正 50 +字幕 50 +加入 50 +反正 50 +不该 50 +政府 50 +快乐 50 +身边 50 +down 50 +遇到 50 +some 50 +這裡 50 +伟大 50 +咖啡 50 +tell 50 +say 50 +仍然 50 +开车 50 +代表 50 +尸体 50 +比较 50 +一年 50 +紧张 50 +监狱 50 +way 50 +位置 50 +随便 50 +安排 50 +表演 50 +除非 50 +屁股 50 +這些 50 +by 50 +老兄 50 +支持 50 +必要 50 +人生 50 +正确 50 +跟着 50 +杀死 50 +more 50 +交给 50 +叔叔 50 +還是 50 +食物 50 +事实 50 +简直 50 +安静 50 +妹妹 50 +something 50 +美丽 50 +再次 50 +案子 50 +下面 50 +hey 50 +make 50 +不够 50 +可怜 50 +首先 50 +另外 50 +提供 50 +记住 50 +听见 50 +国王 50 +didn 50 +面前 50 +几乎 50 +日子 50 +那麼 50 +舒服 50 +原来 50 +再也 50 +纽约 50 +早就 50 +really 50 +有关 50 +放开 50 +信息 50 +星期 50 +never 50 +不得不 50 +发誓 50 +钥匙 50 +错误 50 +放松 50 +约会 50 +打扰 50 +手机 50 +很大 50 +到处 50 +攻击 50 +一份 50 +第一 50 +建议 50 +日本 50 +理由 50 +一樣 50 +跳舞 50 +生气 50 +信任 50 +差不多 50 +原谅 50 +has 50 +经常 50 +的确 50 +之一 50 +幸福 50 +吃饭 50 +正是 50 +不必 50 +最大 50 +people 50 +幸运 50 +我家 50 +問題 50 +東西 50 +而是 50 +动物 50 +弟弟 50 +道歉 50 +严重 50 +人家 50 +坚持 50 +想象 50 +礼物 50 +每次 50 +将军 50 +ok 50 +剩下 50 +有事 50 +老爸 50 +系统 50 +全都 50 +组织 50 +敌人 50 +看上去 50 +應該 49 +老天 49 +认真 49 +关心 49 +中国 49 +生意 49 +上去 49 +值得 49 +搞定 49 +寻找 49 +派对 49 +pro 49 +许多 49 +学生 49 +查理 49 +下次 49 +通知 49 +救命 49 +表现 49 +利用 49 +受伤 49 +后来 49 +精神 49 +居然 49 +受到 49 +am 49 +大学 49 +那时 49 +既然 49 +几天 49 +有时 49 +一张 49 +too 49 +真相 49 +不知 49 +警官 49 +love 49 +属于 49 +fncronos 49 +over 49 +可不是 49 +一名 49 +怀疑 49 +笨蛋 49 +战斗 49 +承认 49 +法律 49 +简体 49 +记录 49 +见面 49 +能力 49 +足够 49 +同样 49 +白痴 49 +拒绝 49 +影响 49 +交易 49 +回來 49 +开门 49 +或是 49 +教授 49 +享受 49 +時間 49 +喝酒 49 +好多 49 +開始 49 +only 49 +走开 49 +合作 49 +女朋友 49 +好吃 49 +事件 49 +出发 49 +酒吧 49 +多么 49 +不如 49 +客人 49 +梦想 49 +英雄 49 +little 49 +杀人 49 +行为 49 +头发 49 +适合 49 +恭喜 49 +上次 49 +地狱 49 +不在乎 49 +哪个 49 +实在 49 +路上 49 +接近 49 +不准 49 +银行 49 +介绍 49 +一旦 49 +味道 49 +感受 49 +身份 49 +off 49 +意外 49 +一般 49 +停下 49 +意义 49 +来看 49 +学习 49 +同时 49 +不起 49 +呼吸 49 +经历 49 +凶手 49 +目前 49 +威胁 49 +没法 49 +should 49 +英国 49 +老大 49 +愚蠢 49 +算了 49 +真实 49 +sorry 49 +婚礼 49 +在家 49 +允许 49 +because 49 +军队 49 +said 49 +电视 49 +please 49 +一家 49 +灵魂 49 +究竟 49 +无论 49 +谋杀 49 +立刻 49 +试图 49 +客气 49 +文件 49 +give 49 +慢慢 49 +前面 49 +奶奶 49 +sir 49 +答案 49 +新闻 49 +多谢 49 +可不 49 +very 49 +any 49 +一遍 49 +权利 49 +做出 49 +历史 49 +two 49 +好了吗 49 +意识 49 +下午 49 +尝试 49 +乔治 49 +经过 49 +获得 49 +部分 49 +當然 49 +带走 49 +竟然 49 +失败 49 +一生 49 +黑暗 49 +中心 49 +麦克 49 +警告 49 +大哥 49 +显然 49 +根据 49 +不過 49 +自杀 49 +subhead 49 +训练 49 +thing 49 +令人 49 +好人 48 +迟到 48 +mean 48 +恐怕 48 +一杯 48 +没用 48 +thank 48 +讨论 48 +爆炸 48 +美好 48 +放心 48 +长大 48 +微软 48 +辛苦 48 +尽管 48 +美元 48 +their 48 +help 48 +收拾 48 +包括 48 +面对 48 +公寓 48 +邀请 48 +生日 48 +法官 48 +方向 48 +一块 48 +法国 48 +一半 48 +find 48 +确认 48 +玩笑 48 +doing 48 +病人 48 +目的 48 +公主 48 +mr 48 +even 48 +sure 48 +these 48 +最终 48 +士兵 48 +其它 48 +反对 48 +傻瓜 48 +into 48 +拿走 48 +对方 48 +警方 48 +不介意 48 +赶快 48 +失望 48 +出生 48 +酒店 48 +德国 48 +上班 48 +uh 48 +覺得 48 +造成 48 +破坏 48 +到达 48 +巴黎 48 +不见 48 +男朋友 48 +想起 48 +老公 48 +教堂 48 +起來 48 +做好 48 +拯救 48 +still 48 +媽媽 48 +一句 48 +子弹 48 +节目 48 +玛丽 48 +方面 48 +life 48 +安娜 48 +晚餐 48 +越来越 48 +巨大 48 +伦敦 48 +啤酒 48 +好看 48 +感情 48 +自然 48 +汤姆 48 +day 48 +一把 48 +一步 48 +逮捕 48 +超过 48 +赶紧 48 +much 48 +炸弹 48 +将会 48 +god 48 +主人 48 +可惜 48 +算是 48 +上校 48 +家族 48 +恶心 48 +陛下 48 +从未 48 +彼得 48 +通常 48 +撒谎 48 +兴趣 48 +旅行 48 +没想到 48 +比如 48 +人民 48 +穿着 48 +干净 48 +不少 48 +队长 48 +公平 48 +愉快 48 +不幸 48 +好不好 48 +new 48 +电脑 48 +火车 48 +有意思 48 +干得 48 +爱情 48 +还给 48 +一边 48 +宇宙 48 +请问 48 +一點 48 +死去 48 +心情 48 +明显 48 +不敢 48 +找个 48 +现场 48 +一部分 48 +失踪 48 +垃圾 48 +十分 48 +相当 48 +這兒 48 +before 48 +regular 48 +than 48 +接着 48 +不可 48 +健康 48 +提醒 48 +测试 48 +maybe 48 +随时 48 +事儿 48 +遗憾 48 +犯罪 48 +治疗 48 +狗屎 48 +這種 48 +太阳 48 +习惯 48 +night 48 +大卫 48 +last 48 +怪物 48 +靠近 48 +反应 48 +意见 48 +蛋糕 48 +时刻 48 +于是 48 +床上 48 +责任 48 +充满 48 +多年 48 +亨利 48 +心里 48 +说谎 48 +work 48 +祈祷 48 +疯子 48 +差点 48 +fndidactic 48 +活动 48 +stop 48 +对付 48 +命运 48 +醫生 48 +罗马 48 +發生 48 +同志 48 +必須 48 +服务 48 +call 48 +anything 48 +尊重 48 +关闭 48 +準備 48 +first 48 +期待 48 +号码 48 +again 48 +资料 48 +机器 48 +天使 48 +难过 48 +记忆 47 +過來 47 +线索 47 +nothing 47 +正好 47 +表示 47 +脑袋 47 +爷爷 47 +探员 47 +移动 47 +错过 47 +旁边 47 +彼此 47 +社会 47 +好事 47 +跟踪 47 +keep 47 +尼克 47 +不断 47 +away 47 +私人 47 +上床 47 +好玩 47 +神父 47 +周围 47 +put 47 +自从 47 +叫做 47 +主要 47 +创造 47 +无聊 47 +作品 47 +制作 47 +干杯 47 +保罗 47 +第二 47 +法庭 47 +设计 47 +干掉 47 +压力 47 +轻松 47 +those 47 +other 47 +上车 47 +说明 47 +谈论 47 +超级 47 +要么 47 +速度 47 +确保 47 +手上 47 +周末 47 +long 47 +袭击 47 +感覺 47 +won 47 +上来 47 +尤其 47 +船长 47 +浪费 47 +怀孕 47 +婚姻 47 +talk 47 +die 47 +信号 47 +天堂 47 +恐怖 47 +角色 47 +心脏 47 +恐惧 47 +去世 47 +性感 47 +记者 47 +離開 47 +客户 47 +由于 47 +thought 47 +庆祝 47 +总之 47 +惊喜 47 +土地 47 +更加 47 +出來 47 +会议 47 +女性 47 +技术 47 +送到 47 +地址 47 +困难 47 +在乎 47 +和平 47 +制造 47 +离婚 47 +認為 47 +似的 47 +sam 47 +同学 47 +请求 47 +看不到 47 +动作 47 +不停 47 +今年 47 +must 47 +演出 47 +参与 47 +逃跑 47 +香港 47 +汽车 47 +杀手 47 +ever 47 +after 47 +提到 47 +兴奋 47 +毒品 47 +home 47 +分手 47 +做爱 47 +马克 47 +一群 47 +好处 47 +宝宝 47 +森林 47 +赢得 47 +生存 47 +厕所 47 +things 47 +荣幸 47 +报警 47 +地上 47 +wait 47 +送给 47 +世上 47 +想念 47 +大约 47 +可恶 47 +合适 47 +普通 47 +天才 47 +生物 47 +之中 47 +妓女 47 +尽快 47 +报纸 47 +提出 47 +正式 47 +政治 47 +从不 47 +一辈子 47 +取消 47 +world 47 +监视 47 +产生 47 +玩意 47 +搞砸 47 +只不过 47 +显示 47 +价值 47 +带到 47 +恶魔 47 +一周 47 +told 47 +状况 47 +那位 47 +name 47 +guy 47 +建立 47 +山姆 47 +高中 47 +去年 47 +厨房 47 +特殊 47 +黑人 47 +邪恶 47 +勇敢 47 +不算 47 +石头 47 +等到 47 +does 47 +假装 47 +better 47 +运动 47 +介意 47 +手下 47 +印度 47 +耶稣 47 +手指 47 +思考 47 +沒錯 47 +机场 47 +地点 47 +恢复 47 +夜晚 47 +帽子 47 +执行 47 +时代 47 +状态 47 +years 47 +餐厅 47 +呵呵 47 +意大利 47 +背叛 47 +everything 47 +魔鬼 47 +情报 47 +great 47 +人员 47 +专家 47 +那樣 47 +了不起 47 +不怕 47 +骄傲 47 +哎呀 47 +唱歌 47 +站住 47 +之类 47 +满足 47 +停车 47 +现实 47 +牺牲 47 +胜利 47 +好消息 47 +见过 47 +引起 47 +醒来 47 +下车 47 +不许 47 +颜色 47 +有用 47 +暂时 47 +sans 47 +投票 47 +证人 47 +想像 47 +全世界 47 +around 47 +拍摄 47 +冒险 47 +place 47 +部队 47 +王子 47 +发展 47 +顺利 47 +甜心 47 +谈话 47 +不去 47 +吸血鬼 47 +吸引 47 +动手 47 +老实 47 +比利 47 +一段 46 +激动 46 +条件 46 +doesn 46 +特工 46 +always 46 +早餐 46 +追踪 46 +發現 46 +废话 46 +演员 46 +亲自 46 +所谓 46 +复杂 46 +学会 46 +距离 46 +好笑 46 +碰到 46 +受不了 46 +醒醒 46 +成员 46 +绑架 46 +即将 46 +发射 46 +魔法 46 +through 46 +every 46 +莎拉 46 +把手 46 +分享 46 +年前 46 +然而 46 +感激 46 +同事 46 +神秘 46 +made 46 +上尉 46 +feel 46 +fnfzhei 46 +哈利 46 +桌子 46 +someone 46 +禁止 46 +交流 46 +放手 46 +英里 46 +几年 46 +old 46 +安妮 46 +公园 46 +fuck 46 +别管 46 +done 46 +早安 46 +中间 46 +整天 46 +我想 46 +shit 46 +巧克力 46 +找出 46 +强大 46 +受害者 46 +某些 46 +后悔 46 +您好 46 +精彩 46 +有點 46 +卡尔 46 +睡着 46 +果然 46 +人人 46 +基地 46 +一片 46 +guys 46 +单独 46 +名单 46 +病毒 46 +may 46 +过程 46 +左右 46 +生病 46 +人物 46 +戒指 46 +自我 46 +立即 46 +女生 46 +星球 46 +变化 46 +卡车 46 +怎樣 46 +空间 46 +穿过 46 +旅馆 46 +打死 46 +观众 46 +依然 46 +kill 46 +做事 46 +三年 46 +乐队 46 +飞行 46 +接到 46 +两次 46 +导演 46 +满意 46 +認識 46 +导致 46 +which 46 +接触 46 +折磨 46 +骗子 46 +明星 46 +经验 46 +小伙子 46 +前进 46 +美女 46 +一部 46 +好运 46 +尽力 46 +ich 46 +密码 46 +刺激 46 +以来 46 +承诺 46 +艺术 46 +让开 46 +明明 46 +惊讶 46 +免费 46 +案件 46 +女孩子 46 +惩罚 46 +释放 46 +顺便 46 +无辜 46 +老鼠 46 +采取 46 +believe 46 +联络 46 +展示 46 +无关 46 +箱子 46 +挑战 46 +讲话 46 +练习 46 +点儿 46 +楼上 46 +擔心 46 +实现 46 +脖子 46 +理论 46 +关键 46 +電話 46 +聚会 46 +洛杉矶 46 +方便 46 +耳朵 46 +数字 46 +世纪 46 +奇迹 46 +各种 46 +窗户 46 +罪犯 46 +呼叫 46 +仅仅 46 +frank 46 +手术 46 +leave 46 +说服 46 +john 46 +死定了 46 +清理 46 +big 46 +宣布 46 +启动 46 +能量 46 +分开 46 +加上 46 +fnfangsong 46 +印象 46 +天空 46 +天气 46 +游泳 46 +拜拜 46 +滚开 46 +大部分 46 +stay 46 +教练 46 +阿姨 46 +估计 46 +科学 46 +而言 46 +乐意 46 +鼻子 46 +three 46 +不錯 46 +地区 46 +杀害 46 +孤独 46 +拿来 46 +夏天 46 +十年 46 +might 46 +如今 46 +某人 46 +不像 46 +欧洲 46 +船上 46 +肚子 46 +大声 46 +手里 46 +红色 46 +婴儿 46 +环境 46 +团队 46 +神奇 46 +年纪 46 +瞧瞧 46 +kind 46 +年代 46 +上学 46 +大量 46 +快去 46 +伤心 46 +何时 46 +紧急 46 +暴力 46 +付出 46 +尽量 46 +太多 46 +互相 46 +繼續 46 +空气 46 +邻居 46 +dead 46 +某种 46 +他人 46 +规则 46 +我妈 46 +being 46 +司机 46 +好奇 46 +打破 46 +coming 46 +信仰 46 +baby 46 +持续 46 +无所谓 46 +协议 46 +父親 46 +假如 46 +阁下 46 +打赌 46 +城堡 46 +说不定 46 +彻底 46 +地图 46 +牧师 46 +墨西哥 46 +重复 46 +犯人 46 +我爸 46 +运气 46 +不太 46 +华文 46 +早晨 46 +girl 46 +幹什麼 46 +house 46 +fntahoma 46 +一对 46 +hell 46 +小孩子 46 +money 46 +相同 46 +isn 46 +裤子 46 +规矩 46 +到来 46 +逃走 46 +程序 46 +发出 46 +趴下 46 +白人 46 +中尉 46 +不肯 46 +街上 46 +own 46 +打架 46 +lot 46 +女王 46 +救救 46 +三天 46 +午餐 46 +看法 46 +喝醉 46 +一支 46 +enough 46 +权力 46 +也好 46 +留言 46 +摧毁 46 +全是 46 +故意 46 +打败 46 +哪裡 46 +hello 46 +背后 46 +远离 46 +毕竟 46 +警探 46 +洗澡 46 +bad 46 +黑色 46 +便宜 46 +行李 46 +毫无 46 +一人 46 +蓝色 46 +浪漫 46 +起床 46 +很少 46 +大脑 46 +师父 46 +殿下 46 +指控 46 +道理 46 +尖叫 46 +爵士 46 +冠军 46 +按照 46 +hear 46 +重点 46 +教育 46 +一堆 46 +前往 46 +对待 46 +another 45 +进展 45 +有所 45 +大叔 45 +仔细 45 +护士 45 +一颗 45 +罗斯 45 +战士 45 +太空 45 +申请 45 +happened 45 +勇气 45 +万岁 45 +left 45 +付钱 45 +面包 45 +不仅 45 +关注 45 +不久 45 +看不见 45 +万一 45 +熟悉 45 +正如 45 +笑话 45 +经典 45 diff --git a/app/src/main/resources/neuro/keys/unstemmed.txt b/app/src/main/resources/neuro/keys/unstemmed.txt index 4b5bf74239..dbebfe5163 100644 --- a/app/src/main/resources/neuro/keys/unstemmed.txt +++ b/app/src/main/resources/neuro/keys/unstemmed.txt @@ -1003,3 +1003,11 @@ yarn yoga zelda zone +教育 +新闻 +旅游 +游戏 +电影 +科技 +美食 +音乐 diff --git a/app/src/main/resources/neuro/runbreak/zh.txt b/app/src/main/resources/neuro/runbreak/zh.txt new file mode 100644 index 0000000000..3e05756440 --- /dev/null +++ b/app/src/main/resources/neuro/runbreak/zh.txt @@ -0,0 +1,66 @@ +# zh merged-run break characters (NeuroChunker). Hans. +的 +了 +着 +过 +是 +在 +和 +与 +及 +或 +也 +都 +就 +才 +又 +还 +把 +被 +让 +给 +对 +向 +从 +到 +为 +地 +得 +之 +其 +我 +你 +他 +她 +它 +们 +这 +那 +哪 +吗 +呢 +吧 +啊 +呀 +么 +很 +不 +没 +有 +个 +些 +什 +怎 +如 +而 +但 +可 +并 +将 +要 +能 +去 +来 +第 +期 +话 diff --git a/app/src/main/resources/neuro/stop/zh.txt b/app/src/main/resources/neuro/stop/zh.txt index f99c8904cd..4c1063d911 100644 --- a/app/src/main/resources/neuro/stop/zh.txt +++ b/app/src/main/resources/neuro/stop/zh.txt @@ -1,4 +1,4 @@ -# stopwords-iso 6e8e9a2d1c01 stopwords-iso.json (MIT) | scripts/neuro/gen_stoplists.py v1 | +0 add, -0 keep +# stopwords-iso 6e8e9a2d1c01 stopwords-iso.json (MIT) | scripts/neuro/gen_stoplists.py v1 | +29 add, -0 keep 0 1 2 @@ -68,6 +68,7 @@ 个 个别 中 +中字 临 为 为了 @@ -109,6 +110,9 @@ 什么 什么样 今 +今天 +今年 +今日 介于 仍 仍旧 @@ -174,8 +178,11 @@ 先不先 光 光是 +入门 全体 +全程 全部 +全集 八 六 兮 @@ -214,6 +221,7 @@ 出来 分 分别 +分钟 则 则甚 别 @@ -239,6 +247,7 @@ 即若 却 去 +去年 又 又及 及 @@ -386,6 +395,7 @@ 如此 如若 始而 +字幕 孰料 孰知 宁 @@ -394,6 +404,9 @@ 宁肯 它 它们 +完整 +完整版 +官方 对 对于 对待 @@ -401,6 +414,8 @@ 对比 将 小 +小时 +小白 尔 尔后 尔尔 @@ -433,6 +448,7 @@ 当地 当然 当着 +影片 彼 彼时 彼此 @@ -486,6 +502,7 @@ 故 故此 故而 +新手 旁人 无 无宁 @@ -497,6 +514,8 @@ 日 时 时候 +明天 +昨天 是 是以 是的 @@ -505,6 +524,8 @@ 替 替代 最 +最新 +最新版 月 有 有些 @@ -517,7 +538,10 @@ 朝着 本 本人 +本周 本地 +本月 +本期 本着 本身 来 @@ -552,6 +576,7 @@ 比如 比方 没奈何 +没有 沿 沿着 漫说 @@ -601,6 +626,7 @@ 简言之 管 类如 +系列 紧接着 纵 纵令 @@ -661,6 +687,7 @@ 要不然 要么 要是 +视频 譬喻 譬如 让 @@ -762,4 +789,6 @@ 靠 顺 顺着 +频道 首先 +高清 diff --git a/app/src/main/resources/neuro/topical/zh.txt b/app/src/main/resources/neuro/topical/zh.txt index f3eaa3747e..4a7f0b3259 100644 --- a/app/src/main/resources/neuro/topical/zh.txt +++ b/app/src/main/resources/neuro/topical/zh.txt @@ -1,4 +1,4 @@ -# Channel-name topics for Chinese, Traditional and Simplified spellings. Each entry is one word of the platform word breaker. +# zh channel-name topics. Each entry folded to Hans is one breaker word. 新聞 新闻 英文 @@ -52,8 +52,6 @@ 哲學 哲学 心理 -心理學 -紀錄片 喜劇 喜剧 手機 @@ -92,9 +90,6 @@ 相機 相机 機車 -摩托車 -腳踏車 -自行車 裝修 装修 木工 @@ -105,6 +100,5 @@ 编织 創業 创业 -行銷 商業 商业 diff --git a/app/src/test/java/io/github/aedev/flow/data/recommendation/IcuWordBreaker.kt b/app/src/test/java/io/github/aedev/flow/data/recommendation/IcuWordBreaker.kt new file mode 100644 index 0000000000..d5c5d8b254 --- /dev/null +++ b/app/src/test/java/io/github/aedev/flow/data/recommendation/IcuWordBreaker.kt @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2025-2026 Flow | A-EDev + * + * This file is part of Flow (https://github.com/Flow-Tube/Flow). + */ + +package io.github.aedev.flow.data.recommendation + +import com.ibm.icu.text.BreakIterator +import java.util.Locale + +/** ICU4J dictionary breaker; the JDK breaker does not split Han runs. */ +internal object IcuWordBreaker : WordBreaker { + override fun split(chunk: String): List { + val breaks = BreakIterator.getWordInstance(Locale.ROOT) + breaks.setText(chunk) + val tokens = ArrayList() + var start = breaks.first() + var end = breaks.next() + while (end != BreakIterator.DONE) { + tokens += chunk.substring(start, end) + start = end + end = breaks.next() + } + return tokens + } +} diff --git a/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroChannelNameTest.kt b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroChannelNameTest.kt index b111a8645f..e5d99f975c 100644 --- a/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroChannelNameTest.kt +++ b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroChannelNameTest.kt @@ -13,6 +13,7 @@ import org.junit.Test /** Channel names give topics only for words the engine knows (#1341). */ class NeuroChannelNameTest { private val tokenizer = NeuroTokenizer() + private val icu = NeuroTokenizer(IcuWordBreaker) @Test fun `spaced names keep their topical words`() { @@ -28,15 +29,24 @@ class NeuroChannelNameTest { assertThat(tokenizer.tokenizeChannelName("央")).isEmpty() } + @Test + fun `a Chinese channel name gives its topic`() { + assertThat(tokenizer.tokenizeChannelName("TVBS新聞")).containsExactly("新闻") + assertThat(icu.tokenizeChannelName("央視新聞")).containsExactly("新闻") + assertThat(icu.tokenizeChannelName("阿滴英文")).containsExactly("英文") + assertThat(icu.tokenizeChannelName("英文教室 科技")).containsExactly("英文", "科技") + } + @Test fun `the Chinese topic list is folded, whole words`() { val topics = NeuroLanguages.channelTopics - assertThat(topics).contains("新聞") assertThat(topics).contains("新闻") + assertThat(topics).doesNotContain("新聞") topics.forEach { word -> assertWithMessage(word).that(word).isEqualTo(NeuroText.fold(word)) assertWithMessage(word).that(word.length).isAtLeast(2) assertWithMessage(word).that(word.any(Char::isWhitespace)).isFalse() + assertWithMessage(word).that(NeuroText.words(word, IcuWordBreaker)).containsExactly(word) } } diff --git a/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroHanFoldDecisionTest.kt b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroHanFoldDecisionTest.kt new file mode 100644 index 0000000000..b45cebcbe2 --- /dev/null +++ b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroHanFoldDecisionTest.kt @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2025-2026 Flow | A-EDev + * + * This file is part of Flow (https://github.com/Flow-Tube/Flow). + */ + +package io.github.aedev.flow.data.recommendation + +import com.google.common.truth.Truth.assertThat +import io.github.aedev.flow.data.model.Video +import org.junit.Test + +/** The Han fold is decided once per video, and blocking reads text fully folded (#1341). */ +class NeuroHanFoldDecisionTest { + private val tokenizer = NeuroTokenizer(IcuWordBreaker) + private val empty = IdfSnapshot(emptyMap(), 0) + + private fun video( + title: String, + channel: String = "", + tags: List = emptyList(), + ) = Video("v", title, channel, "", "", 600, 1L, uploadDate = "", tags = tags) + + private fun terms(video: Video) = tokenizer.documentTerms(video, tokenizer.extractFeatures(video, empty)) + + @Test + fun `a video with kana anywhere keeps Han unfolded in every field`() { + val title = "鬼滅 無限城編 予告" + + assertThat(tokenizer.hanFoldOf(video(title, channel = "アニメ公式"))).isEqualTo(HanFold.NONE) + assertThat(tokenizer.hanFoldOf(video(title, tags = listOf("アニメ")))).isEqualTo(HanFold.NONE) + assertThat(terms(video(title, channel = "アニメ公式"))).contains("無限") + assertThat(terms(video(title, channel = "アニメ公式"))).doesNotContain("无限") + } + + @Test + fun `a video without kana folds every field`() { + val title = "鬼滅 無限城編 予告" + + assertThat(tokenizer.hanFoldOf(video(title, channel = "央視新聞"))).isEqualTo(HanFold.ALWAYS) + assertThat(terms(video(title, channel = "央視新聞"))).contains("无限") + } + + @Test + fun `one video reads the same word one way whatever field it is in`() { + val withKanaTag = video("無限城編 予告", tags = listOf("無限", "アニメ")) + + assertThat(terms(withKanaTag)).doesNotContain("无限") + assertThat(tokenizer.hanFoldOf(withKanaTag)).isEqualTo(HanFold.NONE) + } + + @Test + fun `description keywords follow the video decision`() { + val description = "intro: 無限城編の予告です。機器學習 演唱會 周杰倫 的最新消息和內容介紹都在這裡。" + "詳細は概要欄をご覧ください。".repeat(2) + val plain = video("機器學習 演唱會", channel = "央視新聞").copy(description = description) + val kana = video("機器學習 演唱會", channel = "アニメ公式").copy(description = description) + + val folded = tokenizer.extractDescriptionKeywords(description, empty, tokenizer.hanFoldOf(plain)).keys + val kept = tokenizer.extractDescriptionKeywords(description, empty, tokenizer.hanFoldOf(kana)).keys + + assertThat(folded.any { '学' in it }).isTrue() + assertThat(kept.any { '学' in it }).isFalse() + assertThat(kept.any { '學' in it }).isTrue() + } + + @Test + fun `a Japanese viewer folds nothing at runtime`() { + val japanese = NeuroTokenizer(IcuWordBreaker).also { it.languageProfile = "ja" } + + assertThat(japanese.hanFolds).isFalse() + assertThat(japanese.fold("鬼滅")).isEqualTo("鬼滅") + assertThat(japanese.chunk("周杰倫 演唱會").merged).isEmpty() + assertThat(japanese.hanFoldOf(video("機器學習"))).isEqualTo(HanFold.NONE) + } + + @Test + fun `other viewers fold and merge`() { + assertThat(tokenizer.hanFolds).isTrue() + assertThat(tokenizer.fold("鬼滅")).isEqualTo("鬼灭") + } + + @Test + fun `blocking matches with both sides folded and kana ignored`() { + val matchers = NeuroScoring.buildBlockedMatchers(setOf("鬼灭"), emptyList(), tokenizer::normalizeLemma) + + assertThat(NeuroScoring.isBlockedByText("鬼滅 アニメ 最新話", "", matchers, tokenizer::normalizeLemma)).isTrue() + assertThat(NeuroScoring.isBlockedByText("鬼灭 アニメ", "", matchers, tokenizer::normalizeLemma)).isTrue() + assertThat(NeuroScoring.isBlockedByText("別的 アニメ", "", matchers, tokenizer::normalizeLemma)).isFalse() + } + + @Test + fun `a blocked keyword stored in either spelling matches`() { + val traditional = NeuroScoring.buildBlockedMatchers(setOf("鬼滅"), emptyList(), tokenizer::normalizeLemma) + + assertThat(NeuroScoring.isBlockedByText("鬼灭 最新话", "", traditional, tokenizer::normalizeLemma)).isTrue() + } + + @Test + fun `Japanese lists keep their kanji`() { + assertThat(NeuroLanguages.stopList("ja")).containsAtLeast("私達", "貴方", "貴方方") + } +} diff --git a/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroHanRunsTest.kt b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroHanRunsTest.kt new file mode 100644 index 0000000000..e4836abe79 --- /dev/null +++ b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroHanRunsTest.kt @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2025-2026 Flow | A-EDev + * + * This file is part of Flow (https://github.com/Flow-Tube/Flow). + */ + +package io.github.aedev.flow.data.recommendation + +import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage +import io.github.aedev.flow.data.model.Video +import org.junit.Test + +/** [NeuroKeySpec] 3: Hant fold and merged Han runs (#1341). */ +class NeuroHanRunsTest { + private val tokenizer = NeuroTokenizer(IcuWordBreaker) + + private fun merged(title: String) = tokenizer.chunk(title).merged + + private fun video( + id: String, + title: String, + ) = Video(id, title, "", "", "", 600, 1L, uploadDate = "") + + private fun idfOf(titles: List): IdfSnapshot { + val counts = HashMap() + titles.forEachIndexed { index, title -> + val video = video("v$index", title) + tokenizer + .documentTerms( + video, + tokenizer.extractFeatures(video, IdfSnapshot(emptyMap(), 0)), + ).forEach { counts.merge(it, 1, Int::plus) } + } + return IdfSnapshot(counts, titles.size) + } + + @Test + fun `Traditional characters read as Simplified ones`() { + assertThat(NeuroText.fold("機器學習")).isEqualTo("机器学习") + assertThat(tokenizer.chunk("英語學習 如何學好英文").chunks).isEqualTo(tokenizer.chunk("英语学习 如何学好英文").chunks) + assertThat(tokenizer.tokenizeChannelName("TVBS新聞")).containsExactly("新闻") + } + + @Test + fun `text with kana is Japanese and its kanji stay`() { + assertThat(NeuroText.fold("宮崎駿 ジブリ")).isEqualTo("宮崎駿 ジブリ") + assertThat(NeuroText.fold("宮崎駿")).isEqualTo("宫崎骏") + } + + @Test + fun `the table maps one character to one Simplified character that maps no further`() { + val lines = + NeuroText::class.java + .getResourceAsStream("/neuro/fold/hant-hans.txt")!! + .bufferedReader(Charsets.UTF_8) + .readLines() + .filter { !it.startsWith("#") } + val keys = lines.map { it[0] }.toSet() + assertThat(lines.size).isAtLeast(2000) + lines.forEach { line -> + assertWithMessage(line).that(line.length).isEqualTo(2) + assertWithMessage(line).that(line[1] in keys).isFalse() + assertWithMessage(line).that(NeuroText.fold(line.substring(0, 1))).isEqualTo(line.substring(1)) + } + } + + @Test + fun `a title keeps the spelling it was written in for display`() { + val spellings = NeuroDisplayForms.spellingsIn("英語學習 入門", listOf("英语 学习"), tokenizer) + + assertThat(spellings).containsExactly("英语 学习", "英語學習") + } + + @Test + fun `characters the breaker leaves single are one candidate, with the word before them`() { + assertThat(merged("【4K】周杰倫 演唱會 完整版")).containsExactly("周杰伦", "演唱会") + assertThat(merged("李永樂老師 講解 相對論")).containsExactly("李永乐", "相对论") + assertThat(tokenizer.chunk("周杰倫 演唱會").chunks).doesNotContain(listOf("周杰伦")) + } + + @Test + fun `a run ends at a break character and holds no stop word or format word`() { + assertThat(merged("老高與小茉")).containsExactly("老高", "小茉") + assertThat(merged("周杰倫的新歌")).containsExactly("周杰伦") + assertThat(merged("iPhone 17 Pro 開箱評測")).isEmpty() + assertThat(tokenizer.chunk("iPhone 17 Pro 開箱評測").formats).containsExactly("unboxing", "review") + } + + @Test + fun `a run is two to five characters`() { + assertThat(merged("周杰倫周杰倫")).isEmpty() + assertThat(merged("周杰")).containsExactly("周杰") + } + + @Test + fun `characters apart are not one run`() { + assertThat(merged("周 杰 倫")).isEmpty() + assertThat(merged("周，杰，倫")).isEmpty() + } + + @Test + fun `Japanese and earlier specs merge nothing`() { + assertThat(merged("大谷翔平 ホームラン 50号")).isEmpty() + assertThat(NeuroLanguages.runBreaks("ja")).isNull() + val spec2 = tokenizer.atKeySpec(2) + assertThat(spec2.chunk("周杰倫 演唱會").merged).isEmpty() + assertThat(spec2.chunk("英語學習").chunks).isEqualTo(listOf(listOf("英語", "學習"))) + } + + @Test + fun `a run is a key once it keeps appearing and not before`() { + val others = List(9) { "Tech review number $it" } + val twice = idfOf(listOf("周杰倫 新歌", "周杰倫 最新作品") + others) + val thrice = idfOf(listOf("周杰倫 新歌", "周杰倫 最新作品", "周杰倫 演唱會") + others) + val title = video("new", "周杰倫 新歌") + + assertThat(tokenizer.extractFeatures(title, IdfSnapshot(emptyMap(), 0)).topics).doesNotContainKey("周杰伦") + assertThat(tokenizer.extractFeatures(title, twice).topics).doesNotContainKey("周杰伦") + assertThat(tokenizer.extractFeatures(title, thrice).topics).containsKey("周杰伦") + } + + @Test + fun `a run whose characters mostly appear elsewhere is not a key`() { + val alone = List(12) { "大 小 号" } + val idf = idfOf(listOf("大小", "大小", "大小") + alone + List(5) { "other $it" }) + + assertThat(NeuroPhrases.isMergedRun("大小", idf)).isFalse() + } + + @Test + fun `the characters of a run count toward document frequency`() { + val video = video("v", "周杰倫 演唱會") + + val terms = tokenizer.documentTerms(video, tokenizer.extractFeatures(video, IdfSnapshot(emptyMap(), 0))) + + assertThat(terms).containsAtLeast("周杰伦", "周", "杰", "伦") + } + + @Test + fun `Traditional keys of a stored brain fold into Simplified ones and keep their spelling`() { + val brain = + UserBrain( + schemaVersion = 20, + globalVector = ContentVector(topics = mapOf("機器學習" to 0.4, "机器学习" to 0.2, "python" to 0.3)), + idfWordFrequency = mapOf("機器學習" to 3, "机器学习" to 4), + blockedTopics = setOf("新聞"), + ) + + val migrated = NeuroMaintenance.runIfNeeded(brain, tokenizer) + + assertThat(migrated.globalVector.topics.keys).containsAtLeast("机器学习", "python") + assertThat(migrated.globalVector.topics).doesNotContainKey("機器學習") + assertThat(migrated.globalVector.topics.getValue("机器学习")).isEqualTo(0.4) + assertThat(migrated.idfWordFrequency["机器学习"]).isEqualTo(7) + assertThat(NeuroDisplayForms.label(migrated.displayForms, "机器学习")).isEqualTo("機器學習") + assertThat(migrated.blockedTopics).containsExactly("新闻") + assertThat(migrated.schemaVersion).isEqualTo(NeuroMaintenance.KEY_SPEC_3_SCHEMA_VERSION) + } + + @Test + fun `a Japanese viewer keeps their kanji keys`() { + val japanese = NeuroTokenizer(IcuWordBreaker).also { it.languageProfile = "ja" } + val brain = UserBrain(schemaVersion = 20, globalVector = ContentVector(topics = mapOf("宮崎駿" to 0.4))) + + val migrated = NeuroMaintenance.runIfNeeded(brain, japanese) + + assertThat(migrated.globalVector.topics.keys).contains("宮崎駿") + } + + @Test + fun `the broad Chinese topics are never recorded as a dislike`() { + val news = ContentVector(topics = mapOf("新闻" to 0.9, "台积电" to 0.5)) + + val keys = NeuroScoring.extractRejectionKeys(news) + + assertThat(keys).contains("~台积电") + assertThat(keys).doesNotContain("~新闻") + } +} diff --git a/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroKeySpecTest.kt b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroKeySpecTest.kt index 9834796c5a..34e43becbc 100644 --- a/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroKeySpecTest.kt +++ b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroKeySpecTest.kt @@ -17,9 +17,9 @@ import kotlinx.serialization.json.jsonPrimitive import org.junit.Test import java.io.File -/** Key spec 1 ([NeuroKeySpec]): the shared golden cases, and the rules for runs. */ +/** Key spec 3 ([NeuroKeySpec]): the shared golden cases, and the rules for runs. */ class NeuroKeySpecTest { - private val tokenizer = NeuroTokenizer() + private val tokenizer = NeuroTokenizer(IcuWordBreaker) private fun key(phrase: String) = tokenizer.keysOf(phrase).single() @@ -40,6 +40,8 @@ class NeuroKeySpecTest { assertWithMessage(input).that(chunks.chunks).isEqualTo(case.getValue("chunks").jsonArray.map { it.jsonArray.strings() }) assertWithMessage(input).that(chunks.modelPairs).isEqualTo(case.getValue("modelPairs").jsonArray.strings()) assertWithMessage(input).that(chunks.formats.toList()).isEqualTo(case.getValue("formats").jsonArray.strings()) + case["merged"]?.let { assertWithMessage(input).that(chunks.merged).isEqualTo(it.jsonArray.strings()) } + case["singles"]?.let { assertWithMessage(input).that(chunks.singles.toList()).isEqualTo(it.jsonArray.strings()) } } } diff --git a/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroSpecificityTest.kt b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroSpecificityTest.kt index 32db1cfe5b..8f0bc53eda 100644 --- a/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroSpecificityTest.kt +++ b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroSpecificityTest.kt @@ -1,7 +1,9 @@ package io.github.aedev.flow.data.recommendation import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage import org.junit.Test +import java.io.File class NeuroSpecificityTest { private val tokenizer = NeuroTokenizer() @@ -44,6 +46,27 @@ class NeuroSpecificityTest { assertThat(weighted).isLessThan(plain * 0.3) } + @Test + fun `the Chinese table holds only words the word breaker keeps whole`() { + val file = File("src/main/resources/neuro/freq/zh.txt") + val lines = file.readLines().filter { it.isNotBlank() } + val whole = + lines.filter { line -> + val word = NeuroText.fold(line.substringBeforeLast(' ')) + word.length >= 2 && NeuroText.words(word, IcuWordBreaker) == listOf(word) + } + if (System.getenv("NEURO_WRITE_FREQ_ZH") == "1") file.writeText(whole.joinToString("\n") + "\n") + assertWithMessage("run the test with NEURO_WRITE_FREQ_ZH=1").that(whole).isEqualTo(lines) + } + + @Test + fun `Chinese words every subject uses are not specific`() { + val chinese = NeuroTokenizer(IcuWordBreaker).also { it.languageProfile = "zh" } + + listOf("新闻", "电影", "音乐", "游戏", "时候").forEach { assertWithWord(it).that(chinese.specificity(it)).isLessThan(0.5) } + assertThat(chinese.specificity("台积电")).isEqualTo(1.0) + } + private fun assertWithWord(word: String) = com.google.common.truth.Truth .assertWithMessage(word) diff --git a/app/src/test/resources/neuro/key_spec_golden.json b/app/src/test/resources/neuro/key_spec_golden.json index d3d4674d10..a4efac5046 100644 --- a/app/src/test/resources/neuro/key_spec_golden.json +++ b/app/src/test/resources/neuro/key_spec_golden.json @@ -1,6 +1,6 @@ { - "keySpec": 2, - "about": "NeuroKeySpec stages 1 to 6 and 8 (stems): each input's chunks, model pairs and formats, read with the viewer's language profile (en unless a case names one) and the stop lists in resources/neuro/stop. Flow Desktop checks the same cases. Hand-reviewed; Chinese, Japanese and Thai need the platform word breaker and are checked on a device.", + "keySpec": 3, + "about": "NeuroKeySpec stages 1 to 6 and 8 (stems): each input's chunks, model pairs and formats, read with the viewer's language profile (en unless a case names one) and the stop lists in resources/neuro/stop. Flow Desktop checks the same cases. Hand-reviewed. Chinese cases are read with a dictionary word breaker such as ICU4J (the JDK breaker does not split Chinese); they also hold merged (the candidate runs of Han characters) and singles (the single Han characters), and a Chinese key is Simplified. Japanese and Thai are checked on a device.", "cases": [ {"input": "Why Big Tech's Privacy Promises Are Lies", "chunks": [["big", "tech", "privaci", "promises"], ["lie"]], "modelPairs": [], "formats": []}, {"input": "Why Big Tech’s Privacy Promises Are Lies", "chunks": [["big", "tech", "privaci", "promises"], ["lie"]], "modelPairs": [], "formats": []}, @@ -76,6 +76,18 @@ {"input": "Les meilleures recettes de crêpes pour le goûter", "languageProfile": "fr", "chunks": [["meilleur", "recet"], ["crêp"], ["goût"]], "modelPairs": [], "formats": []}, {"input": "Как приготовить борщ дома", "languageProfile": "ru", "chunks": [["приготовить", "борщ", "дом"]], "modelPairs": [], "formats": []}, {"input": "Warum der Himmel blau ist", "languageProfile": "de", "chunks": [["himmel", "blau"]], "modelPairs": [], "formats": []}, - {"input": "How to make the best pizza at home", "languageProfile": "es", "chunks": [["pizz"], ["home"]], "modelPairs": [], "formats": []} + {"input": "How to make the best pizza at home", "languageProfile": "es", "chunks": [["pizz"], ["home"]], "modelPairs": [], "formats": []}, + {"input": "【4K】周杰倫 演唱會 完整版", "chunks": [["4k"], ["演唱"]], "modelPairs": [], "formats": [], "merged": ["周杰伦", "演唱会"], "singles": ["周", "杰", "伦", "会", "版"]}, + {"input": "【4K】周杰伦 演唱会 完整版", "chunks": [["4k"], ["演唱"]], "modelPairs": [], "formats": [], "merged": ["周杰伦", "演唱会"], "singles": ["周", "杰", "伦", "会", "版"]}, + {"input": "原神 5.0 新角色 瑪維卡 培養攻略", "chunks": [["角色"], ["培养"]], "modelPairs": [], "formats": ["walkthrough"], "merged": ["原神", "新角色", "玛维卡"], "singles": ["原", "神", "新", "玛", "维", "卡"]}, + {"input": "iPhone 17 Pro 開箱評測", "chunks": [["iphon"], ["pro"]], "modelPairs": ["iphon 17"], "formats": ["unboxing", "review"], "merged": [], "singles": ["开", "箱", "评", "测"]}, + {"input": "李永樂老師 講解 相對論", "chunks": [["永乐", "老师"], ["相对"]], "modelPairs": [], "formats": ["explainer"], "merged": ["李永乐", "相对论"], "singles": ["李", "论"]}, + {"input": "半導體台積電先進製程分析", "chunks": [["半导体"], ["先进", "制程", "分析"]], "modelPairs": [], "formats": [], "merged": ["台积电"], "singles": ["台", "积", "电"]}, + {"input": "老高與小茉 的故事", "chunks": [["故事"]], "modelPairs": [], "formats": [], "merged": ["老高", "小茉"], "singles": ["老", "高", "小", "茉"]}, + {"input": "TVBS新聞 直播 今天的重大新聞", "chunks": [["tvbs", "新闻"], ["重大", "新闻"]], "modelPairs": [], "formats": ["live"], "merged": [], "singles": []}, + {"input": "英語學習 英语学习 如何學好英文", "chunks": [["英语", "学习", "英语", "学习"], ["学好", "英文"]], "modelPairs": [], "formats": [], "merged": [], "singles": []}, + {"input": "我們沒有什麼好說的", "chunks": [["好说"]], "modelPairs": [], "formats": [], "merged": [], "singles": []}, + {"input": "宮崎駿 ジブリ 名作 解説", "chunks": [["宮崎"], ["ジブリ", "名作", "解説"]], "modelPairs": [], "formats": [], "merged": [], "singles": []}, + {"input": "米津玄師 新曲 Lemon", "chunks": [["米津"], ["新曲", "lemon"]], "modelPairs": [], "formats": [], "merged": ["玄师", "米津玄师"], "singles": ["玄", "师"]} ] } diff --git a/app/src/test/resources/neuro/key_spec_tables.json b/app/src/test/resources/neuro/key_spec_tables.json index a41821a24b..9597560ae4 100644 --- a/app/src/test/resources/neuro/key_spec_tables.json +++ b/app/src/test/resources/neuro/key_spec_tables.json @@ -1,5 +1,5 @@ { - "keySpec": 2, + "keySpec": 3, "about": "Tables NeuroKeySpec reads titles by, written by KeySpecTablesTest. Flow Desktop mirrors this file.", "maxRunWords": 3, "maxKeyLength": 40, @@ -1111,43 +1111,55 @@ "compilation", "compilación", "compilação", - "подборка" + "подборка", + "合集", + "混剪" ], "cover": [ "cover", "covers", - "кавер" + "кавер", + "翻唱" ], "documentary": [ "documentary", "documental", "documentário", "documentaire", - "belgesel" + "belgesel", + "纪录片" ], "explainer": [ "explained", - "explainer" + "explainer", + "解说", + "讲解" ], "gameplay": [ "gameplay", - "геймплей" + "геймплей", + "实况" ], "highlights": [ "highlights", "resumen", - "хайлайты" + "хайлайты", + "集锦" ], "interview": [ "interview", "interviews", "entrevista", "интервью", - "röportaj" + "röportaj", + "采访", + "访谈", + "专访" ], "live": [ "live", - "livestream" + "livestream", + "直播" ], "playalong": [ "playalong" @@ -1155,7 +1167,8 @@ "podcast": [ "podcast", "podcasts", - "подкаст" + "подкаст", + "播客" ], "reaction": [ "reaction", @@ -1170,7 +1183,8 @@ "remix": [ "remix", "remixes", - "ремикс" + "ремикс", + "混音" ], "review": [ "review", @@ -1179,7 +1193,10 @@ "análisis", "análise", "обзор", - "inceleme" + "inceleme", + "评测", + "测评", + "点评" ], "tier list": [ "tier list" @@ -1189,19 +1206,24 @@ "teaser", "tráiler", "трейлер", - "fragman" + "fragman", + "预告", + "预告片" ], "tutorial": [ "tutorial", "tutorials", "tuto", "tutoriel", - "урок" + "урок", + "教程", + "教学" ], "unboxing": [ "unboxing", "déballage", - "распаковка" + "распаковка", + "开箱" ], "versus": [ "versus", @@ -1214,7 +1236,9 @@ "walkthrough": [ "walkthrough", "playthrough", - "прохождение" + "прохождение", + "攻略", + "通关" ] }, "functionWords": { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c13f476167..d972f7608e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -37,6 +37,7 @@ re2j = "1.7" desugarJdkLibs = "2.1.5" mockk = "1.14.11" truth = "1.4.5" +icu4j = "77.1" turbine = "1.2.1" core-splashscreen = "1.2.0" profileinstaller = "1.4.1" @@ -119,6 +120,7 @@ kxml2 = { group = "net.sf.kxml", name = "kxml2", version.ref = "kxml2" } desugar-jdk-libs-nio = { group = "com.android.tools", name = "desugar_jdk_libs_nio", version.ref = "desugarJdkLibs" } mockk = { group = "io.mockk", name = "mockk", version.ref = "mockk" } truth = { group = "com.google.truth", name = "truth", version.ref = "truth" } +icu4j = { group = "com.ibm.icu", name = "icu4j", version.ref = "icu4j" } turbine = { group = "app.cash.turbine", name = "turbine", version.ref = "turbine" } kotlinx-coroutines-test = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-test", version.ref = "coroutines" } hilt-android-testing = { group = "com.google.dagger", name = "hilt-android-testing", version.ref = "hilt" } diff --git a/scripts/neuro/add/zh.txt b/scripts/neuro/add/zh.txt new file mode 100644 index 0000000000..6c3182f103 --- /dev/null +++ b/scripts/neuro/add/zh.txt @@ -0,0 +1,30 @@ +# zh video-structure words. Hans. +分钟 +小时 +完整 +完整版 +全集 +全程 +新手 +入门 +小白 +视频 +影片 +频道 +官方 +最新 +最新版 +高清 +字幕 +中字 +系列 +本期 +没有 +今天 +昨天 +明天 +今日 +本周 +本月 +今年 +去年 diff --git a/scripts/neuro/gen_hant_hans.py b/scripts/neuro/gen_hant_hans.py new file mode 100644 index 0000000000..dbd4884d45 --- /dev/null +++ b/scripts/neuro/gen_hant_hans.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Writes app/src/main/resources/neuro/fold/hant-hans.txt from OpenCC TSCharacters.txt (pinned, Apache-2.0). + +BMP 1:1 mappings only; first variant wins; chains resolved to a fixed point. + + python3 -I scripts/neuro/gen_hant_hans.py +""" +import os +import sys +import urllib.request + +COMMIT = "ba09ffb8353f680381200c72452a36c72d540155" +URL = f"https://raw.githubusercontent.com/BYVoid/OpenCC/{COMMIT}/data/dictionary/TSCharacters.txt" +OUT = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "app", "src", "main", "resources", "neuro", "fold", "hant-hans.txt", +) + + +def main(): + with urllib.request.urlopen(URL, timeout=60) as response: + text = response.read().decode("utf-8") + pairs = {} + for line in text.splitlines(): + if not line.strip() or line.startswith("#"): + continue + key, _, values = line.partition("\t") + first = values.split()[0] if values.split() else "" + if len(key) == 1 and len(first) == 1 and key != first and ord(key) < 0x10000 and ord(first) < 0x10000: + pairs[key] = first + # Resolve chains (薴 to 苧 to 苎) so the fold is idempotent. + for key in list(pairs): + value, seen = pairs[key], {key} + while value in pairs and value not in seen: + seen.add(value) + value = pairs[value] + pairs[key] = value + pairs = {key: value for key, value in pairs.items() if key != value} + os.makedirs(os.path.dirname(OUT), exist_ok=True) + with open(OUT, "w", encoding="utf-8", newline="\n") as f: + f.write("# OpenCC %s TSCharacters.txt (Apache-2.0) | scripts/neuro/gen_hant_hans.py | traditional, simplified\n" % COMMIT[:12]) + for key in sorted(pairs): + f.write("%s%s\n" % (key, pairs[key])) + print(len(pairs)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/neuro/gen_wordfreq.py b/scripts/neuro/gen_wordfreq.py index f98ce29e69..29e1dd7ea1 100644 --- a/scripts/neuro/gen_wordfreq.py +++ b/scripts/neuro/gen_wordfreq.py @@ -16,10 +16,10 @@ COMMIT = "525f9b560de45753a5ea01069454e72e9aa541c6" BASE = f"https://raw.githubusercontent.com/hermitdave/FrequencyWords/{COMMIT}/content/2018" -# Languages whose keys are words; CJK keys are character pairs and get no table. +# zh is Hans; NeuroSpecificityTest prunes it to single breaker words (NEURO_WRITE_FREQ_ZH=1). LANGUAGES = { "en": "en", "es": "es", "pt": "pt_br", "fr": "fr", "de": "de", "it": "it", "ru": "ru", "id": "id", - "tr": "tr", "pl": "pl", "cs": "cs", "uk": "uk", "ar": "ar", "fa": "fa", "vi": "vi", "et": "et", + "tr": "tr", "pl": "pl", "cs": "cs", "uk": "uk", "ar": "ar", "fa": "fa", "vi": "vi", "et": "et", "zh": "zh_cn", } TOP = {"en": 10000} # Past this rank a word sits near LOW_ZIPF and barely changes a key's specificity.