Skip to content

feat: implement search utility functions and add tests for Anime prov… #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
heiznerd merged 1 commit into from
Jul 23, 2026
Merged

feat: implement search utility functions and add tests for Anime prov… #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
228 changes: 228 additions & 0 deletions checks/search-check.mts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
import assert from 'node:assert/strict'
import { Anime47Provider } from '../scrapers/providers/anime47'
import { AnimeVietsubProvider } from '../scrapers/providers/animevietsub'
import { SiteDownError } from '../scrapers/fingerprint'
import { cleanSearchQuery, rankSearchResults } from '../scrapers/search-utils'
import type { AnimeSearchResult } from '../scrapers/base'

const result = (id: string, title: string, titleAlt?: string): AnimeSearchResult => ({
id,
source: 'check',
title,
titleAlt,
})

function cardHtml(items: AnimeSearchResult[]): string {
const cards = items.map((item) => `
<article class="TPostMv">
<a href="/phim/${item.id}/" title="${item.title}">
<img alt="${item.title}" src="/${item.id}.jpg">
<h2>${item.title}</h2>
${item.titleAlt ? `<small>${item.titleAlt}</small>` : ''}
</a>
</article>`).join('')
return `<!doctype html><html><body>${cards}</body></html>`.padEnd(120, ' ')
}

async function checkSearchUtils(): Promise<void> {
assert.equal(cleanSearchQuery(' One　 Piece '), 'One Piece')
assert.deepEqual(rankSearchResults([], 'one piece', 10), [])
assert.deepEqual(rankSearchResults([result('x', 'One Piece')], ' ', 10), [])
assert.deepEqual(rankSearchResults([result('x', 'One Piece')], 'one piece', 0), [])
assert.deepEqual(rankSearchResults([result('x', 'One Piece')], 'one piece', -1), [])
assert.equal(rankSearchResults([result('short', 'A')], 'a', 10)[0]?.id, 'short')
assert.deepEqual(rankSearchResults([result('short', 'Tales of Demons')], 'a', 10), [])

const accentRank = rankSearchResults([
result('plain', 'Co Dau'),
result('accent', 'Có Dấu'),
], 'có dấu', 10)
assert.deepEqual(accentRank.map((item) => item.id), ['accent', 'plain'])
assert.equal(rankSearchResults([result('accent', 'Có Dấu')], 'co dau', 10)[0]?.id, 'accent')
assert.equal(rankSearchResults([result('d', 'Đấu La Đại Lục')], 'dau la', 10)[0]?.id, 'd')

assert.equal(rankSearchResults([
result('alt', 'Hải Tặc Mũ Rơm', 'One Piece'),
], 'one piece', 10)[0]?.id, 'alt')
assert.equal(rankSearchResults([
result('label', 'Frieren - Vietsub'),
], 'frieren', 10)[0]?.id, 'label')
assert.equal(rankSearchResults([
result('episode', 'Solo Leveling - Tập 12'),
], 'solo leveling', 10)[0]?.id, 'episode')
assert.equal(rankSearchResults([
result('bracket', 'Sousou no Frieren [Vietsub]'),
], 'sousou no frieren', 10)[0]?.id, 'bracket')
assert.equal(rankSearchResults([
result('quality', 'Bleach - HD'),
], 'bleach', 10)[0]?.id, 'quality')
assert.equal(rankSearchResults([
result('year', 'Bleach 2024'),
], '2024', 10)[0]?.id, 'year')

assert.deepEqual(rankSearchResults([
result('noise', 'Tin anime hôm nay'),
result('match', 'One Piece Film Red'),
], 'one piece', 10).map((item) => item.id), ['match'])
assert.deepEqual(rankSearchResults([
result('weak', 'One Random Show'),
], 'one piece red', 10), [])
assert.deepEqual(rankSearchResults([
result('short-noise', 'Tales of Demons'),
], 'of', 10), [])

const duplicateRank = rankSearchResults([
result(' SAME ', 'One Piece Special'),
result('same', 'One Piece'),
], 'one piece', 10)
assert.deepEqual(duplicateRank.map((item) => item.title), ['One Piece'])

const stableRank = rankSearchResults([
result('first', 'Naruto Movie'),
result('second', 'Naruto Shippuden'),
], 'naruto', 10)
assert.deepEqual(stableRank.map((item) => item.id), ['first', 'second'])
}

async function checkAnimeVietsub(): Promise<void> {
const provider = new AnimeVietsubProvider() as any
const calls: Array<{ url: string; options: Record<string, unknown> }> = []
const enough = Array.from({ length: 12 }, (_, index) => result(`one-piece-a${index}`, `One Piece ${index}`))
provider.fetchHtml = async (url: string, options: Record<string, unknown>) => {
calls.push({ url, options })
return cardHtml(enough)
}

const enoughResults = await provider.search(' One Piece ')
assert.equal(enoughResults.length, 12)
assert.equal(calls.length, 1)
assert.match(calls[0].url, /\/tim-kiem\/One%20Piece\/$/)
assert.deepEqual(calls[0].options, {
timeoutMs: 8000,
retries: 0,
useBrowserFallback: false,
allowDomainFallback: false,
})

const mergingProvider = new AnimeVietsubProvider() as any
const mergeCalls: Array<Record<string, unknown>> = []
mergingProvider.fetchHtml = async (_url: string, options: Record<string, unknown>) => {
mergeCalls.push(options)
if (mergeCalls.length === 1) {
return cardHtml([
result('one-piece-a1', 'One Piece'),
result('sidebar-a2', 'Tin anime hôm nay'),
])
}
return cardHtml([
result('one-piece-a1', 'One Piece Duplicate'),
result('one-piece-a3', 'One Piece Film Red'),
])
}

const merged = await mergingProvider.search('one piece')
assert.deepEqual(merged.map((item: AnimeSearchResult) => item.id), ['one-piece-a1', 'one-piece-a3'])
assert.equal(mergeCalls.length, 2)

const browserProvider = new AnimeVietsubProvider() as any
const rawCalls: Array<Record<string, unknown>> = []
const browserCalls: Array<{ url: string; timeoutMs: number }> = []
browserProvider.fetchHtml = async (_url: string, options: Record<string, unknown>) => {
rawCalls.push(options)
return cardHtml([result('noise-a1', 'Tin anime')])
}
browserProvider.fetchHtmlWithPlaywright = async (url: string, timeoutMs: number) => {
browserCalls.push({ url, timeoutMs })
return cardHtml([result('bleach-a2', 'Bleach')])
}
assert.deepEqual((await browserProvider.search('bleach')).map((item: AnimeSearchResult) => item.id), ['bleach-a2'])
assert.equal(rawCalls.length, 2)
assert.equal(browserCalls.length, 1)
assert.equal(browserCalls[0].timeoutMs, 20000)

const downProvider = new AnimeVietsubProvider() as any
downProvider.fetchHtml = async () => { throw new SiteDownError('AnimeVietsub') }
await assert.rejects(() => downProvider.search('one piece'), SiteDownError)
}

async function checkAnime47(): Promise<void> {
const mappingProvider = new Anime47Provider() as any
assert.deepEqual(mappingProvider.mapStateAnime({
url: '/phim/one-piece/m12.html',
name: 'One Piece',
}), {
id: 'one-piece-m12',
source: 'anime47',
title: 'One Piece',
titleAlt: undefined,
thumbnail: '',
cover: '',
status: undefined,
rating: undefined,
year: undefined,
totalEpisodes: undefined,
})

const provider = new Anime47Provider() as any
const pages: number[] = []
provider.fetchJson = async (url: string, options: Record<string, unknown>) => {
const page = Number(new URL(url).searchParams.get('page'))
pages.push(page)
assert.deepEqual(options, { timeoutMs: 8000, retries: 0 })
return page === 1
? { data: [
{ url: '/phim/one-piece/m1.html', name: 'One Piece' },
{ url: '/phim/sidebar/m2.html', name: 'Tin anime' },
] }
: { items: Array.from({ length: 11 }, (_, index) => ({
link: `/phim/one-piece-${index}/m${index + 10}.html`,
title: `One Piece ${index}`,
})) }
}
provider.fetchHtml = async () => { throw new Error('HTML fallback must not run') }

const results = await provider.search(' One Piece ')
assert.equal(results.length, 12)
assert.deepEqual(pages, [1, 2])

const emptyPageProvider = new Anime47Provider() as any
const emptyPages: number[] = []
const htmlCalls: Array<{ url: string; options: Record<string, unknown> }> = []
emptyPageProvider.fetchJson = async (url: string) => {
emptyPages.push(Number(new URL(url).searchParams.get('page')))
return { results: [] }
}
emptyPageProvider.fetchHtml = async (url: string, options: Record<string, unknown>) => {
htmlCalls.push({ url, options })
return `<!doctype html><html><body>
<article><a href="/phim/bleach/m99.html" title="Bleach"><img alt="Bleach"></a></article>
</body></html>`
}

assert.deepEqual((await emptyPageProvider.search('bleach')).map((item: AnimeSearchResult) => item.id), ['bleach-m99'])
assert.deepEqual(emptyPages, [1])
assert.equal(htmlCalls.length, 1)
assert.deepEqual(htmlCalls[0].options, {
timeoutMs: 8000,
retries: 0,
useBrowserFallback: false,
})

const noiseProvider = new Anime47Provider() as any
let fallbackCount = 0
noiseProvider.fetchJson = async () => ({ results: [{
link: '/phim/sidebar/m1.html',
title: 'Tin anime',
}] })
noiseProvider.fetchHtml = async () => {
fallbackCount++
return '<html><body>No cards</body></html>'
}
assert.deepEqual(await noiseProvider.search('bleach'), [])
assert.equal(fallbackCount, 2)
}

await checkSearchUtils()
await checkAnimeVietsub()
await checkAnime47()
console.log('search-check: ok')
72 changes: 41 additions & 31 deletions scrapers/providers/anime47.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ import { enrichWithAniList } from '../anilist'
import { externalApi } from '../external-api'
import { getProviderCookieHeader, getProviderToken, loadAuthSession } from '../../storage'
import { fetchA47Page, interceptA47StreamUrl } from '../auth-service'
import { cleanSearchQuery, rankSearchResults } from '../search-utils'

type Anime47HtmlOptions = {
timeoutMs?: number
retries?: number
useBrowserFallback?: boolean
}

type AnimeCachePayload = {
id: string
Expand Down Expand Up @@ -145,9 +152,9 @@ export class Anime47Provider extends BaseScraper {
* authenticate anime47 SPA HTML pages.
* Falls back to standard fetch (Bearer header) for unauthenticated or API routes.
*/
private async fetchHtml(url: string, options: { timeoutMs?: number; retries?: number } = {}): Promise<string> {
private async fetchHtml(url: string, options: Anime47HtmlOptions = {}): Promise<string> {
const token = getProviderToken('anime47')
if (token) {
if (token && options.useBrowserFallback !== false) {
const html = await fetchA47Page(url)
if (html) return html
// If Playwright failed (e.g. redirected to login), try standard fetch as last resort
Expand Down Expand Up @@ -322,9 +329,9 @@ export class Anime47Provider extends BaseScraper {

private mapStateAnime(item: any): AnimeSearchResult | null {
if (!item || typeof item !== 'object') return null
const link = String(item.link || item.canonical_url || '')
const link = String(item.link || item.canonical_url || item.url || '')
const id = this.normalizeAnimeIdFromLink(link)
const title = String(item.title || '').trim()
const title = String(item.title || item.name || '').trim()
if (!id || !title) return null

const rawPoster = String(item.poster || item.thumbnail || item.image || item.images?.poster || '')
Expand Down Expand Up @@ -495,64 +502,67 @@ export class Anime47Provider extends BaseScraper {
}

async search(query: string): Promise<AnimeSearchResult[]> {
const q = query.trim()
const q = cleanSearchQuery(query)
if (!q) return []

const merged: AnimeSearchResult[] = []
const seen = new Set<string>()
const addResult = (item: AnimeSearchResult | null) => {
if (!item || !item.id || seen.has(item.id)) return
seen.add(item.id)
merged.push(item)
}
const addResults = (items: AnimeSearchResult[]) => merged.push(...items)
const hasRelevant = () => rankSearchResults(merged, q, 48).length >= 12

// Primary path: real Anime47 API used by site frontend
// Primary path: real Anime47 API used by site frontend. Keep pages serial
// to avoid multiplying load on provider while stopping as soon as useful.
try {
for (let page = 1; page <= 3; page++) {
const apiUrl = `${this.apiBase}/search/full/?lang=vi&keyword=${encodeURIComponent(q)}&page=${page}`
const response = await this.fetchJson<{ results?: any[] }>(apiUrl, { retries: 1 })
const rows = Array.isArray(response?.results) ? response.results : []
const response = await this.fetchJson<{ results?: any[]; data?: any[]; items?: any[] }>(apiUrl, {
timeoutMs: 8000,
retries: 0
})
const rows = Array.isArray(response)
? response
: Array.isArray(response?.results)
? response.results
: Array.isArray(response?.data)
? response.data
: Array.isArray(response?.items)
? response.items
: []
if (rows.length === 0) break
for (const row of rows) addResult(this.mapStateAnime(row))
addResults(rows.map((row) => this.mapStateAnime(row)).filter((item): item is AnimeSearchResult => Boolean(item)))
if (hasRelevant()) break
}
} catch (error) {
console.warn('[Anime47] search api failed:', error)
}

// Secondary fallback: parse page HTML (only when API returned no results)
if (merged.length === 0) {
// Secondary fallback: parse page HTML only when API has no relevant hit.
if (rankSearchResults(merged, q, 48).length === 0) {
const urls = [
`${this.baseUrl}/tim-kiem/${encodeURIComponent(q)}`,
`${this.baseUrl}/filter?keyword=${encodeURIComponent(q)}`
]

for (const url of urls) {
try {
const html = await this.fetchHtml(url, { retries: 1 })
const html = await this.fetchHtml(url, {
timeoutMs: 8000,
retries: 0,
useBrowserFallback: false
})
const state = this.extractInitialState(html)
const stateResults = this.extractListsFromState(state)
.flat()
.map((item) => this.mapStateAnime(item))
.filter((v): v is AnimeSearchResult => Boolean(v))
const htmlResults = this.parseAnimeCardsFromHtml(html)

for (const item of [...stateResults, ...htmlResults]) addResult(item)
if (merged.length > 0) break // stop after first URL that yields results
addResults([...stateResults, ...this.parseAnimeCardsFromHtml(html)])
if (rankSearchResults(merged, q, 48).length > 0) break
} catch (error) {
console.warn('[Anime47] search url failed:', url, error)
}
}
}

const lowered = q.toLowerCase()
const ranked = merged
.sort((a, b) => {
const aHit = a.title.toLowerCase().includes(lowered) ? 1 : 0
const bHit = b.title.toLowerCase().includes(lowered) ? 1 : 0
return bHit - aHit
})
.slice(0, 48)
return ranked
return rankSearchResults(merged, q, 48)
}

async getAnimeDetail(id: string): Promise<AnimeDetail | null> {
Expand Down
Loading