From 1be1fcb08426e5e69e4675c73339a30d93590b99 Mon Sep 17 00:00:00 2001 From: Chiehx0220 Date: Mon, 19 Oct 2026 10:36:36 +0800 Subject: [PATCH] feat(neuro): split channel names into words and add Chinese topic words (#1341) --- .../NeuroTextSegmentationTest.kt | 9 ++ .../data/recommendation/NeuroLanguages.kt | 17 +++ .../data/recommendation/NeuroTokenizer.kt | 30 ++--- app/src/main/resources/neuro/topical/zh.txt | 110 ++++++++++++++++++ .../recommendation/NeuroChannelNameTest.kt | 48 ++++++++ 5 files changed, 199 insertions(+), 15 deletions(-) create mode 100644 app/src/main/resources/neuro/topical/zh.txt create mode 100644 app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroChannelNameTest.kt diff --git a/app/src/androidTest/java/io/github/aedev/flow/data/recommendation/NeuroTextSegmentationTest.kt b/app/src/androidTest/java/io/github/aedev/flow/data/recommendation/NeuroTextSegmentationTest.kt index 5551f276bb..b3992b4de3 100644 --- a/app/src/androidTest/java/io/github/aedev/flow/data/recommendation/NeuroTextSegmentationTest.kt +++ b/app/src/androidTest/java/io/github/aedev/flow/data/recommendation/NeuroTextSegmentationTest.kt @@ -40,6 +40,15 @@ class NeuroTextSegmentationTest { @Test fun thaiTitlesSplitIntoWords() = assertTopics("สอนเล่นกีตาร์สำหรับมือใหม่", "กีตาร์", "สำหรับ") + @Test + fun chineseChannelNamesGiveTheirTopicalWords() { + listOf("央視新聞" to "新聞", "央视新闻" to "新闻", "英文教室 科技" to "科技").forEach { (name, topic) -> + val topics = tokenizer.tokenizeChannelName(name) + assertTrue("$topic not in $topics for \"$name\"", topic in topics) + } + assertTrue(tokenizer.tokenizeChannelName("小明").isEmpty()) + } + @Test fun spacedScriptsAreUntouched() = assertTopics("Pixel 10 Pro camera test", "pixel", "camera") } diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroLanguages.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroLanguages.kt index 203e1d7b3b..9ec348f4c3 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroLanguages.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroLanguages.kt @@ -112,6 +112,23 @@ internal object NeuroLanguages { } } + private val TOPICAL_LANGUAGES = listOf("zh") + + /** + * Words of channel names in scripts written without spaces that name a topic, folded + * (`resources/neuro/topical`). English has [NeuroTokenizer]'s own list. Read once per process. + */ + val channelTopics: Set by lazy { + buildSet { + TOPICAL_LANGUAGES.forEach { language -> + val stream = NeuroLanguages::class.java.getResourceAsStream("/neuro/topical/$language.txt") ?: return@forEach + stream.bufferedReader(Charsets.UTF_8).useLines { lines -> + lines.filter { it.isNotBlank() && !it.startsWith("#") }.mapTo(this) { NeuroText.fold(it.trim()) } + } + } + } + } + /** * The language of [pieces], the folded whitespace-separated pieces of a title, or null when * unsure. Scripts used by one language decide alone; Latin titles need two function words and a diff --git a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroTokenizer.kt b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroTokenizer.kt index 5a34b49301..a835563533 100644 --- a/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroTokenizer.kt +++ b/app/src/main/java/io/github/aedev/flow/data/recommendation/NeuroTokenizer.kt @@ -2935,21 +2935,17 @@ internal class NeuroTokenizer { * "Python Engineer" → ["python"] (engineer not in list, python is) * "Binging with Babish" → [] (no topic words) * "History Matters" → ["history"] + * "央視新聞" → ["新聞"] (央視 = a brand, 新聞 = news) */ fun tokenizeChannelName(channelName: String): List { - val rawOriginal = - NeuroText - .fold(channelName) - .split(WHITESPACE_REGEX) - .map { word -> word.trim { !it.isLetterOrDigit() } } - .filter { it.length > 1 } + val rawOriginal = nameWords(channelName) if (rawOriginal.isEmpty()) return emptyList() if (rawOriginal.size == 1) { val word = rawOriginal[0].lowercase() val lemma = keyOf(word) - return if (lemma in alwaysTopical || word in alwaysTopical) { + return if (isChannelTopic(lemma) || isChannelTopic(word)) { listOf(lemma) } else { emptyList() @@ -2962,13 +2958,22 @@ internal class NeuroTokenizer { val lemma = keyOf(lower) when { - lemma in alwaysTopical -> topical.add(lemma) - lower in alwaysTopical -> topical.add(lower) + isChannelTopic(lemma) -> topical.add(lemma) + isChannelTopic(lower) -> topical.add(lower) } } return topical.distinct() } + private fun isChannelTopic(word: String): Boolean = word in alwaysTopical || word in NeuroLanguages.channelTopics + + /** Words of a channel name; scripts written without spaces are split by the word breaker. */ + private fun nameWords(name: String): List = + NeuroText + .words(NeuroText.fold(name)) + .map { word -> word.trim { !NeuroText.isWordChar(it) } } + .filter { it.length > 1 } + /** * Builds a flat list of all meaningful words from the video's title, * channel name, and description. Used as context for domain disambiguation. @@ -2986,12 +2991,7 @@ internal class NeuroTokenizer { .filter { it.length > 1 } .forEach { words.add(it) } - NeuroText - .fold(video.channelName) - .split(WHITESPACE_REGEX) - .map { it.trim { c -> !c.isLetterOrDigit() } } - .filter { it.length > 1 } - .forEach { words.add(it) } + words.addAll(nameWords(video.channelName)) if (!video.description.isNullOrBlank()) { video.description diff --git a/app/src/main/resources/neuro/topical/zh.txt b/app/src/main/resources/neuro/topical/zh.txt new file mode 100644 index 0000000000..f3eaa3747e --- /dev/null +++ b/app/src/main/resources/neuro/topical/zh.txt @@ -0,0 +1,110 @@ +# Channel-name topics for Chinese, Traditional and Simplified spellings. Each entry is one word of the platform word breaker. +新聞 +新闻 +英文 +英語 +英语 +日语 +韓語 +韩语 +科技 +遊戲 +游戏 +音樂 +音乐 +電影 +电影 +美食 +料理 +旅遊 +旅游 +旅行 +健身 +財經 +财经 +投資 +投资 +股票 +汽車 +汽车 +攝影 +摄影 +動畫 +动画 +籃球 +篮球 +棒球 +足球 +網球 +网球 +數學 +数学 +物理 +化學 +化学 +生物 +歷史 +历史 +地理 +經濟 +经济 +政治 +哲學 +哲学 +心理 +心理學 +紀錄片 +喜劇 +喜剧 +手機 +手机 +烹飪 +烹饪 +烘焙 +寵物 +宠物 +科學 +科学 +教育 +語言 +语言 +健康 +瑜伽 +運動 +运动 +時尚 +时尚 +鋼琴 +钢琴 +吉他 +小提琴 +爵士 +搖滾 +摇滚 +嘻哈 +古典 +天文 +太空 +宇宙 +駭客 +電腦 +电脑 +相機 +相机 +機車 +摩托車 +腳踏車 +自行車 +裝修 +装修 +木工 +手工 +刺繡 +刺绣 +編織 +编织 +創業 +创业 +行銷 +商業 +商业 diff --git a/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroChannelNameTest.kt b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroChannelNameTest.kt new file mode 100644 index 0000000000..b111a8645f --- /dev/null +++ b/app/src/test/java/io/github/aedev/flow/data/recommendation/NeuroChannelNameTest.kt @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2025-2026 Flow | A-EDev + * + * This file is part of Flow (https://github.com/Flow-Tube/Flow). + */ + +package io.github.aedev.flow.data.recommendation + +import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage +import org.junit.Test + +/** Channel names give topics only for words the engine knows (#1341). */ +class NeuroChannelNameTest { + private val tokenizer = NeuroTokenizer() + + @Test + fun `spaced names keep their topical words`() { + assertThat(tokenizer.tokenizeChannelName("Linus Tech Tips")).containsExactly("tech") + assertThat(tokenizer.tokenizeChannelName("Python Engineer")).containsExactly("python") + assertThat(tokenizer.tokenizeChannelName("Fireship")).isEmpty() + assertThat(tokenizer.tokenizeChannelName("Binging with Babish")).isEmpty() + } + + @Test + fun `a name of a script without spaces has no topics unless it holds a topical word`() { + assertThat(tokenizer.tokenizeChannelName("")).isEmpty() + assertThat(tokenizer.tokenizeChannelName("央")).isEmpty() + } + + @Test + fun `the Chinese topic list is folded, whole words`() { + val topics = NeuroLanguages.channelTopics + assertThat(topics).contains("新聞") + assertThat(topics).contains("新闻") + topics.forEach { word -> + assertWithMessage(word).that(word).isEqualTo(NeuroText.fold(word)) + assertWithMessage(word).that(word.length).isAtLeast(2) + assertWithMessage(word).that(word.any(Char::isWhitespace)).isFalse() + } + } + + @Test + fun `no topic word is also a stop word`() { + val stops = NeuroLanguages.stopList("zh") + assertThat(NeuroLanguages.channelTopics.filter { it in stops }).isEmpty() + } +}