Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,15 @@ class NeuroTextSegmentationTest {
@Test
fun thaiTitlesSplitIntoWords() = assertTopics("สอนเล่นกีตาร์สำหรับมือใหม่", "กีตาร์", "สำหรับ")

@Test
fun chineseChannelNamesGiveTheirTopicalWords() {
listOf("央視新聞" to "新聞", "央视新闻" to "新闻", "英文教室 科技" to "科技").forEach { (name, topic) ->
val topics = tokenizer.tokenizeChannelName(name)
assertTrue("$topic not in $topics for \"$name\"", topic in topics)
}
assertTrue(tokenizer.tokenizeChannelName("小明").isEmpty())
}

@Test
fun spacedScriptsAreUntouched() = assertTopics("Pixel 10 Pro camera test", "pixel", "camera")
}
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,23 @@ internal object NeuroLanguages {
}
}

private val TOPICAL_LANGUAGES = listOf("zh")

/**
* Words of channel names in scripts written without spaces that name a topic, folded
* (`resources/neuro/topical`). English has [NeuroTokenizer]'s own list. Read once per process.
*/
val channelTopics: Set<String> by lazy {
buildSet {
TOPICAL_LANGUAGES.forEach { language ->
val stream = NeuroLanguages::class.java.getResourceAsStream("/neuro/topical/$language.txt") ?: return@forEach
stream.bufferedReader(Charsets.UTF_8).useLines { lines ->
lines.filter { it.isNotBlank() && !it.startsWith("#") }.mapTo(this) { NeuroText.fold(it.trim()) }
}
}
}
}

/**
* The language of [pieces], the folded whitespace-separated pieces of a title, or null when
* unsure. Scripts used by one language decide alone; Latin titles need two function words and a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2935,21 +2935,17 @@ internal class NeuroTokenizer {
* "Python Engineer" → ["python"] (engineer not in list, python is)
* "Binging with Babish" → [] (no topic words)
* "History Matters" → ["history"]
* "央視新聞" → ["新聞"] (央視 = a brand, 新聞 = news)
*/
fun tokenizeChannelName(channelName: String): List<String> {
val rawOriginal =
NeuroText
.fold(channelName)
.split(WHITESPACE_REGEX)
.map { word -> word.trim { !it.isLetterOrDigit() } }
.filter { it.length > 1 }
val rawOriginal = nameWords(channelName)

if (rawOriginal.isEmpty()) return emptyList()

if (rawOriginal.size == 1) {
val word = rawOriginal[0].lowercase()
val lemma = keyOf(word)
return if (lemma in alwaysTopical || word in alwaysTopical) {
return if (isChannelTopic(lemma) || isChannelTopic(word)) {
listOf(lemma)
} else {
emptyList()
Expand All @@ -2962,13 +2958,22 @@ internal class NeuroTokenizer {
val lemma = keyOf(lower)

when {
lemma in alwaysTopical -> topical.add(lemma)
lower in alwaysTopical -> topical.add(lower)
isChannelTopic(lemma) -> topical.add(lemma)
isChannelTopic(lower) -> topical.add(lower)
}
}
return topical.distinct()
}

private fun isChannelTopic(word: String): Boolean = word in alwaysTopical || word in NeuroLanguages.channelTopics

/** Words of a channel name; scripts written without spaces are split by the word breaker. */
private fun nameWords(name: String): List<String> =
NeuroText
.words(NeuroText.fold(name))
.map { word -> word.trim { !NeuroText.isWordChar(it) } }
.filter { it.length > 1 }

/**
* Builds a flat list of all meaningful words from the video's title,
* channel name, and description. Used as context for domain disambiguation.
Expand All @@ -2986,12 +2991,7 @@ internal class NeuroTokenizer {
.filter { it.length > 1 }
.forEach { words.add(it) }

NeuroText
.fold(video.channelName)
.split(WHITESPACE_REGEX)
.map { it.trim { c -> !c.isLetterOrDigit() } }
.filter { it.length > 1 }
.forEach { words.add(it) }
words.addAll(nameWords(video.channelName))

if (!video.description.isNullOrBlank()) {
video.description
Expand Down
110 changes: 110 additions & 0 deletions app/src/main/resources/neuro/topical/zh.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
# Channel-name topics for Chinese, Traditional and Simplified spellings. Each entry is one word of the platform word breaker.
新聞
新闻
英文
英語
英语
日语
韓語
韩语
科技
遊戲
游戏
音樂
音乐
電影
电影
美食
料理
旅遊
旅游
旅行
健身
財經
财经
投資
投资
股票
汽車
汽车
攝影
摄影
動畫
动画
籃球
篮球
棒球
足球
網球
网球
數學
数学
物理
化學
化学
生物
歷史
历史
地理
經濟
经济
政治
哲學
哲学
心理
心理學
紀錄片
喜劇
喜剧
手機
手机
烹飪
烹饪
烘焙
寵物
宠物
科學
科学
教育
語言
语言
健康
瑜伽
運動
运动
時尚
时尚
鋼琴
钢琴
吉他
小提琴
爵士
搖滾
摇滚
嘻哈
古典
天文
太空
宇宙
駭客
電腦
电脑
相機
相机
機車
摩托車
腳踏車
自行車
裝修
装修
木工
手工
刺繡
刺绣
編織
编织
創業
创业
行銷
商業
商业
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Copyright (C) 2025-2026 Flow | A-EDev
*
* This file is part of Flow (https://github.com/Flow-Tube/Flow).
*/

package io.github.aedev.flow.data.recommendation

import com.google.common.truth.Truth.assertThat
import com.google.common.truth.Truth.assertWithMessage
import org.junit.Test

/** Channel names give topics only for words the engine knows (#1341). */
class NeuroChannelNameTest {
private val tokenizer = NeuroTokenizer()

@Test
fun `spaced names keep their topical words`() {
assertThat(tokenizer.tokenizeChannelName("Linus Tech Tips")).containsExactly("tech")
assertThat(tokenizer.tokenizeChannelName("Python Engineer")).containsExactly("python")
assertThat(tokenizer.tokenizeChannelName("Fireship")).isEmpty()
assertThat(tokenizer.tokenizeChannelName("Binging with Babish")).isEmpty()
}

@Test
fun `a name of a script without spaces has no topics unless it holds a topical word`() {
assertThat(tokenizer.tokenizeChannelName("")).isEmpty()
assertThat(tokenizer.tokenizeChannelName("央")).isEmpty()
}

@Test
fun `the Chinese topic list is folded, whole words`() {
val topics = NeuroLanguages.channelTopics
assertThat(topics).contains("新聞")
assertThat(topics).contains("新闻")
topics.forEach { word ->
assertWithMessage(word).that(word).isEqualTo(NeuroText.fold(word))
assertWithMessage(word).that(word.length).isAtLeast(2)
assertWithMessage(word).that(word.any(Char::isWhitespace)).isFalse()
}
}

@Test
fun `no topic word is also a stop word`() {
val stops = NeuroLanguages.stopList("zh")
assertThat(NeuroLanguages.channelTopics.filter { it in stops }).isEmpty()
}
}
Loading