From 799cfb1520db2edb6cd9ad016f343807e2e4081f Mon Sep 17 00:00:00 2001 From: Heiznerd Date: Thu, 23 Jul 2026 08:30:45 +0700 Subject: [PATCH] feat: implement search utility functions and add tests for Anime providers --- checks/search-check.mts | 228 +++++++++++++++++++++++++++++ scrapers/providers/anime47.ts | 72 +++++---- scrapers/providers/animevietsub.ts | 124 ++++++++++------ scrapers/search-utils.ts | 122 +++++++++++++++ 4 files changed, 469 insertions(+), 77 deletions(-) create mode 100644 checks/search-check.mts create mode 100644 scrapers/search-utils.ts diff --git a/checks/search-check.mts b/checks/search-check.mts new file mode 100644 index 0000000..6f47923 --- /dev/null +++ b/checks/search-check.mts @@ -0,0 +1,228 @@ +import assert from 'node:assert/strict' +import { Anime47Provider } from '../scrapers/providers/anime47' +import { AnimeVietsubProvider } from '../scrapers/providers/animevietsub' +import { SiteDownError } from '../scrapers/fingerprint' +import { cleanSearchQuery, rankSearchResults } from '../scrapers/search-utils' +import type { AnimeSearchResult } from '../scrapers/base' + +const result = (id: string, title: string, titleAlt?: string): AnimeSearchResult => ({ + id, + source: 'check', + title, + titleAlt, +}) + +function cardHtml(items: AnimeSearchResult[]): string { + const cards = items.map((item) => ` +
+ + ${item.title} +

${item.title}

+ ${item.titleAlt ? `${item.titleAlt}` : ''} + +
`).join('') + return `${cards}`.padEnd(120, ' ') +} + +async function checkSearchUtils(): Promise { + assert.equal(cleanSearchQuery(' One　 Piece '), 'One Piece') + assert.deepEqual(rankSearchResults([], 'one piece', 10), []) + assert.deepEqual(rankSearchResults([result('x', 'One Piece')], ' ', 10), []) + assert.deepEqual(rankSearchResults([result('x', 'One Piece')], 'one piece', 0), []) + assert.deepEqual(rankSearchResults([result('x', 'One Piece')], 'one piece', -1), []) + assert.equal(rankSearchResults([result('short', 'A')], 'a', 10)[0]?.id, 'short') + assert.deepEqual(rankSearchResults([result('short', 'Tales of Demons')], 'a', 10), []) + + const accentRank = rankSearchResults([ + result('plain', 'Co Dau'), + result('accent', 'Có Dấu'), + ], 'có dấu', 10) + assert.deepEqual(accentRank.map((item) => item.id), ['accent', 'plain']) + assert.equal(rankSearchResults([result('accent', 'Có Dấu')], 'co dau', 10)[0]?.id, 'accent') + assert.equal(rankSearchResults([result('d', 'Đấu La Đại Lục')], 'dau la', 10)[0]?.id, 'd') + + assert.equal(rankSearchResults([ + result('alt', 'Hải Tặc Mũ Rơm', 'One Piece'), + ], 'one piece', 10)[0]?.id, 'alt') + assert.equal(rankSearchResults([ + result('label', 'Frieren - Vietsub'), + ], 'frieren', 10)[0]?.id, 'label') + assert.equal(rankSearchResults([ + result('episode', 'Solo Leveling - Tập 12'), + ], 'solo leveling', 10)[0]?.id, 'episode') + assert.equal(rankSearchResults([ + result('bracket', 'Sousou no Frieren [Vietsub]'), + ], 'sousou no frieren', 10)[0]?.id, 'bracket') + assert.equal(rankSearchResults([ + result('quality', 'Bleach - HD'), + ], 'bleach', 10)[0]?.id, 'quality') + assert.equal(rankSearchResults([ + result('year', 'Bleach 2024'), + ], '2024', 10)[0]?.id, 'year') + + assert.deepEqual(rankSearchResults([ + result('noise', 'Tin anime hôm nay'), + result('match', 'One Piece Film Red'), + ], 'one piece', 10).map((item) => item.id), ['match']) + assert.deepEqual(rankSearchResults([ + result('weak', 'One Random Show'), + ], 'one piece red', 10), []) + assert.deepEqual(rankSearchResults([ + result('short-noise', 'Tales of Demons'), + ], 'of', 10), []) + + const duplicateRank = rankSearchResults([ + result(' SAME ', 'One Piece Special'), + result('same', 'One Piece'), + ], 'one piece', 10) + assert.deepEqual(duplicateRank.map((item) => item.title), ['One Piece']) + + const stableRank = rankSearchResults([ + result('first', 'Naruto Movie'), + result('second', 'Naruto Shippuden'), + ], 'naruto', 10) + assert.deepEqual(stableRank.map((item) => item.id), ['first', 'second']) +} + +async function checkAnimeVietsub(): Promise { + const provider = new AnimeVietsubProvider() as any + const calls: Array<{ url: string; options: Record }> = [] + const enough = Array.from({ length: 12 }, (_, index) => result(`one-piece-a${index}`, `One Piece ${index}`)) + provider.fetchHtml = async (url: string, options: Record) => { + calls.push({ url, options }) + return cardHtml(enough) + } + + const enoughResults = await provider.search(' One Piece ') + assert.equal(enoughResults.length, 12) + assert.equal(calls.length, 1) + assert.match(calls[0].url, /\/tim-kiem\/One%20Piece\/$/) + assert.deepEqual(calls[0].options, { + timeoutMs: 8000, + retries: 0, + useBrowserFallback: false, + allowDomainFallback: false, + }) + + const mergingProvider = new AnimeVietsubProvider() as any + const mergeCalls: Array> = [] + mergingProvider.fetchHtml = async (_url: string, options: Record) => { + mergeCalls.push(options) + if (mergeCalls.length === 1) { + return cardHtml([ + result('one-piece-a1', 'One Piece'), + result('sidebar-a2', 'Tin anime hôm nay'), + ]) + } + return cardHtml([ + result('one-piece-a1', 'One Piece Duplicate'), + result('one-piece-a3', 'One Piece Film Red'), + ]) + } + + const merged = await mergingProvider.search('one piece') + assert.deepEqual(merged.map((item: AnimeSearchResult) => item.id), ['one-piece-a1', 'one-piece-a3']) + assert.equal(mergeCalls.length, 2) + + const browserProvider = new AnimeVietsubProvider() as any + const rawCalls: Array> = [] + const browserCalls: Array<{ url: string; timeoutMs: number }> = [] + browserProvider.fetchHtml = async (_url: string, options: Record) => { + rawCalls.push(options) + return cardHtml([result('noise-a1', 'Tin anime')]) + } + browserProvider.fetchHtmlWithPlaywright = async (url: string, timeoutMs: number) => { + browserCalls.push({ url, timeoutMs }) + return cardHtml([result('bleach-a2', 'Bleach')]) + } + assert.deepEqual((await browserProvider.search('bleach')).map((item: AnimeSearchResult) => item.id), ['bleach-a2']) + assert.equal(rawCalls.length, 2) + assert.equal(browserCalls.length, 1) + assert.equal(browserCalls[0].timeoutMs, 20000) + + const downProvider = new AnimeVietsubProvider() as any + downProvider.fetchHtml = async () => { throw new SiteDownError('AnimeVietsub') } + await assert.rejects(() => downProvider.search('one piece'), SiteDownError) +} + +async function checkAnime47(): Promise { + const mappingProvider = new Anime47Provider() as any + assert.deepEqual(mappingProvider.mapStateAnime({ + url: '/phim/one-piece/m12.html', + name: 'One Piece', + }), { + id: 'one-piece-m12', + source: 'anime47', + title: 'One Piece', + titleAlt: undefined, + thumbnail: '', + cover: '', + status: undefined, + rating: undefined, + year: undefined, + totalEpisodes: undefined, + }) + + const provider = new Anime47Provider() as any + const pages: number[] = [] + provider.fetchJson = async (url: string, options: Record) => { + const page = Number(new URL(url).searchParams.get('page')) + pages.push(page) + assert.deepEqual(options, { timeoutMs: 8000, retries: 0 }) + return page === 1 + ? { data: [ + { url: '/phim/one-piece/m1.html', name: 'One Piece' }, + { url: '/phim/sidebar/m2.html', name: 'Tin anime' }, + ] } + : { items: Array.from({ length: 11 }, (_, index) => ({ + link: `/phim/one-piece-${index}/m${index + 10}.html`, + title: `One Piece ${index}`, + })) } + } + provider.fetchHtml = async () => { throw new Error('HTML fallback must not run') } + + const results = await provider.search(' One Piece ') + assert.equal(results.length, 12) + assert.deepEqual(pages, [1, 2]) + + const emptyPageProvider = new Anime47Provider() as any + const emptyPages: number[] = [] + const htmlCalls: Array<{ url: string; options: Record }> = [] + emptyPageProvider.fetchJson = async (url: string) => { + emptyPages.push(Number(new URL(url).searchParams.get('page'))) + return { results: [] } + } + emptyPageProvider.fetchHtml = async (url: string, options: Record) => { + htmlCalls.push({ url, options }) + return ` +
Bleach
+ ` + } + + assert.deepEqual((await emptyPageProvider.search('bleach')).map((item: AnimeSearchResult) => item.id), ['bleach-m99']) + assert.deepEqual(emptyPages, [1]) + assert.equal(htmlCalls.length, 1) + assert.deepEqual(htmlCalls[0].options, { + timeoutMs: 8000, + retries: 0, + useBrowserFallback: false, + }) + + const noiseProvider = new Anime47Provider() as any + let fallbackCount = 0 + noiseProvider.fetchJson = async () => ({ results: [{ + link: '/phim/sidebar/m1.html', + title: 'Tin anime', + }] }) + noiseProvider.fetchHtml = async () => { + fallbackCount++ + return 'No cards' + } + assert.deepEqual(await noiseProvider.search('bleach'), []) + assert.equal(fallbackCount, 2) +} + +await checkSearchUtils() +await checkAnimeVietsub() +await checkAnime47() +console.log('search-check: ok') diff --git a/scrapers/providers/anime47.ts b/scrapers/providers/anime47.ts index a97fc79..1a50fbb 100644 --- a/scrapers/providers/anime47.ts +++ b/scrapers/providers/anime47.ts @@ -5,6 +5,13 @@ import { enrichWithAniList } from '../anilist' import { externalApi } from '../external-api' import { getProviderCookieHeader, getProviderToken, loadAuthSession } from '../../storage' import { fetchA47Page, interceptA47StreamUrl } from '../auth-service' +import { cleanSearchQuery, rankSearchResults } from '../search-utils' + +type Anime47HtmlOptions = { + timeoutMs?: number + retries?: number + useBrowserFallback?: boolean +} type AnimeCachePayload = { id: string @@ -145,9 +152,9 @@ export class Anime47Provider extends BaseScraper { * authenticate anime47 SPA HTML pages. * Falls back to standard fetch (Bearer header) for unauthenticated or API routes. */ - private async fetchHtml(url: string, options: { timeoutMs?: number; retries?: number } = {}): Promise { + private async fetchHtml(url: string, options: Anime47HtmlOptions = {}): Promise { const token = getProviderToken('anime47') - if (token) { + if (token && options.useBrowserFallback !== false) { const html = await fetchA47Page(url) if (html) return html // If Playwright failed (e.g. redirected to login), try standard fetch as last resort @@ -322,9 +329,9 @@ export class Anime47Provider extends BaseScraper { private mapStateAnime(item: any): AnimeSearchResult | null { if (!item || typeof item !== 'object') return null - const link = String(item.link || item.canonical_url || '') + const link = String(item.link || item.canonical_url || item.url || '') const id = this.normalizeAnimeIdFromLink(link) - const title = String(item.title || '').trim() + const title = String(item.title || item.name || '').trim() if (!id || !title) return null const rawPoster = String(item.poster || item.thumbnail || item.image || item.images?.poster || '') @@ -495,32 +502,41 @@ export class Anime47Provider extends BaseScraper { } async search(query: string): Promise { - const q = query.trim() + const q = cleanSearchQuery(query) if (!q) return [] const merged: AnimeSearchResult[] = [] - const seen = new Set() - const addResult = (item: AnimeSearchResult | null) => { - if (!item || !item.id || seen.has(item.id)) return - seen.add(item.id) - merged.push(item) - } + const addResults = (items: AnimeSearchResult[]) => merged.push(...items) + const hasRelevant = () => rankSearchResults(merged, q, 48).length >= 12 - // Primary path: real Anime47 API used by site frontend + // Primary path: real Anime47 API used by site frontend. Keep pages serial + // to avoid multiplying load on provider while stopping as soon as useful. try { for (let page = 1; page <= 3; page++) { const apiUrl = `${this.apiBase}/search/full/?lang=vi&keyword=${encodeURIComponent(q)}&page=${page}` - const response = await this.fetchJson<{ results?: any[] }>(apiUrl, { retries: 1 }) - const rows = Array.isArray(response?.results) ? response.results : [] + const response = await this.fetchJson<{ results?: any[]; data?: any[]; items?: any[] }>(apiUrl, { + timeoutMs: 8000, + retries: 0 + }) + const rows = Array.isArray(response) + ? response + : Array.isArray(response?.results) + ? response.results + : Array.isArray(response?.data) + ? response.data + : Array.isArray(response?.items) + ? response.items + : [] if (rows.length === 0) break - for (const row of rows) addResult(this.mapStateAnime(row)) + addResults(rows.map((row) => this.mapStateAnime(row)).filter((item): item is AnimeSearchResult => Boolean(item))) + if (hasRelevant()) break } } catch (error) { console.warn('[Anime47] search api failed:', error) } - // Secondary fallback: parse page HTML (only when API returned no results) - if (merged.length === 0) { + // Secondary fallback: parse page HTML only when API has no relevant hit. + if (rankSearchResults(merged, q, 48).length === 0) { const urls = [ `${this.baseUrl}/tim-kiem/${encodeURIComponent(q)}`, `${this.baseUrl}/filter?keyword=${encodeURIComponent(q)}` @@ -528,31 +544,25 @@ export class Anime47Provider extends BaseScraper { for (const url of urls) { try { - const html = await this.fetchHtml(url, { retries: 1 }) + const html = await this.fetchHtml(url, { + timeoutMs: 8000, + retries: 0, + useBrowserFallback: false + }) const state = this.extractInitialState(html) const stateResults = this.extractListsFromState(state) .flat() .map((item) => this.mapStateAnime(item)) .filter((v): v is AnimeSearchResult => Boolean(v)) - const htmlResults = this.parseAnimeCardsFromHtml(html) - - for (const item of [...stateResults, ...htmlResults]) addResult(item) - if (merged.length > 0) break // stop after first URL that yields results + addResults([...stateResults, ...this.parseAnimeCardsFromHtml(html)]) + if (rankSearchResults(merged, q, 48).length > 0) break } catch (error) { console.warn('[Anime47] search url failed:', url, error) } } } - const lowered = q.toLowerCase() - const ranked = merged - .sort((a, b) => { - const aHit = a.title.toLowerCase().includes(lowered) ? 1 : 0 - const bHit = b.title.toLowerCase().includes(lowered) ? 1 : 0 - return bHit - aHit - }) - .slice(0, 48) - return ranked + return rankSearchResults(merged, q, 48) } async getAnimeDetail(id: string): Promise { diff --git a/scrapers/providers/animevietsub.ts b/scrapers/providers/animevietsub.ts index 3ea74f3..6c5b9f6 100644 --- a/scrapers/providers/animevietsub.ts +++ b/scrapers/providers/animevietsub.ts @@ -30,6 +30,16 @@ import { enrichWithAniList } from '../anilist' import { getProviderCookieHeader } from '../auth-service' import { loadAuthSession, loadSettings, saveSettings } from '../../storage' import { debugLog, debugWarn } from '../../logger' +import { cleanSearchQuery, rankSearchResults } from '../search-utils' + +type HtmlFetchOptions = { + timeoutMs?: number + retries?: number + useBrowserFallback?: boolean + allowDomainFallback?: boolean + browserTimeoutMs?: number +} + // Extended AnimeDetail with additional metadata export interface AnimeDetailExtended extends AnimeDetail { descriptionVi?: string @@ -144,7 +154,7 @@ export class AnimeVietsubProvider extends BaseScraper { private async fetchHtmlFromAlternateDomains( originalUrl: string, - options: { timeoutMs?: number; retries?: number; useBrowserFallback?: boolean; allowDomainFallback?: boolean } + options: HtmlFetchOptions ): Promise { if (!this.isAnimeVietsubUrl(originalUrl)) return null @@ -244,7 +254,7 @@ export class AnimeVietsubProvider extends BaseScraper { */ private async fetchHtml( url: string, - options: { timeoutMs?: number; retries?: number; useBrowserFallback?: boolean; allowDomainFallback?: boolean } = {} + options: HtmlFetchOptions = {} ): Promise { const timeoutMs = options.timeoutMs ?? 20000 const retries = options.retries ?? 2 @@ -258,7 +268,7 @@ export class AnimeVietsubProvider extends BaseScraper { try { hostname = new URL(url).hostname } catch { /* ignore */ } if (hostname && AnimeVietsubProvider.blockedHostCache.has(hostname)) { if (!useBrowserFallback) throw new Error(`Lỗi kết nối: host bị chặn (${hostname})`) - return await this.fetchHtmlWithPlaywright(url) + return await this.fetchHtmlWithPlaywright(url, options.browserTimeoutMs) } let lastError: unknown @@ -293,7 +303,7 @@ export class AnimeVietsubProvider extends BaseScraper { } if (useBrowserFallback) { debugWarn(`[Scraper] Blocked (${response.status}) after ${consecutiveBlocks} attempt(s), escalating to Playwright for ${url}`) - return await this.fetchHtmlWithPlaywright(url) + return await this.fetchHtmlWithPlaywright(url, options.browserTimeoutMs) } throw new Error(`Lỗi kết nối: HTTP ${response.status}`) } @@ -315,7 +325,7 @@ export class AnimeVietsubProvider extends BaseScraper { } if (useBrowserFallback) { debugWarn(`[Scraper] CF challenge persists, escalating to Playwright for ${url}`) - return await this.fetchHtmlWithPlaywright(url) + return await this.fetchHtmlWithPlaywright(url, options.browserTimeoutMs) } throw new Error('Lỗi kết nối: Cloudflare challenge không thể vượt qua') } @@ -336,7 +346,7 @@ export class AnimeVietsubProvider extends BaseScraper { } if (useBrowserFallback) { debugWarn(`[Scraper] Block error exhausted retries, escalating to Playwright for ${url}`) - return await this.fetchHtmlWithPlaywright(url) + return await this.fetchHtmlWithPlaywright(url, options.browserTimeoutMs) } throw new Error(`Lỗi kết nối: ${(error as Error).message}`) } @@ -392,8 +402,10 @@ export class AnimeVietsubProvider extends BaseScraper { * With it (set by the user's normal Electron browser visit), CF treats the * Playwright session as a recognized client. */ - private async fetchHtmlWithPlaywright(url: string): Promise { + private async fetchHtmlWithPlaywright(url: string, timeoutMs = 35000): Promise { const { chromium } = await import('playwright') + const deadline = Date.now() + timeoutMs + const remainingTime = (ceiling: number) => Math.max(1, Math.min(ceiling, deadline - Date.now())) // --- Step 1: Pull CF + session cookies from stored session --- let playwrightCookies: Array<{ @@ -470,11 +482,11 @@ export class AnimeVietsubProvider extends BaseScraper { const page = await context.newPage() try { - await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 35000 }) + await page.goto(url, { waitUntil: 'domcontentloaded', timeout: remainingTime(35000) }) // Wait for CF challenge to pass (if cookies didn't help, it may still loop) let cfWaitMs = 0 - while (cfWaitMs < 20000) { + while (cfWaitMs < 20000 && Date.now() < deadline) { const title = await page.title().catch(() => '') const snippet = await page.content().catch(() => '').then(h => h.slice(0, 2000).toLowerCase()) if ( @@ -482,8 +494,9 @@ export class AnimeVietsubProvider extends BaseScraper { snippet.includes('_cf_chl_opt') || snippet.includes('cf-browser-verification') ) { - await page.waitForTimeout(1500) - cfWaitMs += 1500 + const waitMs = remainingTime(1500) + await page.waitForTimeout(waitMs) + cfWaitMs += waitMs continue } break @@ -491,15 +504,15 @@ export class AnimeVietsubProvider extends BaseScraper { // Wait for SPA card content to render via AJAX try { - await page.waitForSelector('a[href*="/phim/"]', { timeout: 12000 }) + await page.waitForSelector('a[href*="/phim/"]', { timeout: remainingTime(12000) }) } catch { // Scroll to trigger lazy content await page.evaluate(() => (globalThis as any).scrollTo(0, (globalThis as any).document.body.scrollHeight / 2)) - await page.waitForTimeout(2000) + if (Date.now() < deadline) await page.waitForTimeout(remainingTime(2000)) } // Extra settle time - await page.waitForTimeout(1000) + if (Date.now() < deadline) await page.waitForTimeout(remainingTime(1000)) this.rememberWorkingDomain(page.url(), url) const html = await page.content() @@ -798,7 +811,7 @@ export class AnimeVietsubProvider extends BaseScraper { // Only fallback to search when HTML parsed OK but yielded 0 cards (not a 403/down scenario) debugWarn('[Scraper] getHomeCards: HTML loaded but 0 cards parsed, trying search fallback') - return this.search('anime') + return this.searchInternal('anime', true) } catch (error) { // Site is temporarily down — propagate as a typed error so UI can show proper message if (error instanceof SiteDownError) { @@ -817,49 +830,68 @@ export class AnimeVietsubProvider extends BaseScraper { } } - async search(query: string): Promise { - if (!query || query.trim().length === 0) { - return [] - } + private parseSearchCards(html: string): AnimeSearchResult[] { + if (html.length < 100) return [] + const $ = this.parseHtml(html) + const results: AnimeSearchResult[] = [] + const seenIds = new Set() + + this.selectCardElements($).each((_, el) => { + const card = this.extractCardData($(el), $) + if (!card || seenIds.has(card.id)) return + seenIds.add(card.id) + results.push(card) + }) + + return results + } + + private async searchInternal(query: string, allowUnmatched = false): Promise { + const normalizedQuery = cleanSearchQuery(query) + if (!normalizedQuery) return [] - // Search URL candidates — try each in order until one returns cards const searchUrls = [ - `${this.baseUrl}/tim-kiem/${encodeURIComponent(query)}/`, - `${this.baseUrl}/?s=${encodeURIComponent(query)}`, + `${this.baseUrl}/tim-kiem/${encodeURIComponent(normalizedQuery)}/`, + `${this.baseUrl}/?s=${encodeURIComponent(normalizedQuery)}`, ] + const candidates: AnimeSearchResult[] = [] + const rawOptions: HtmlFetchOptions = { + timeoutMs: 8000, + retries: 0, + useBrowserFallback: false, + allowDomainFallback: false, + } for (const searchUrl of searchUrls) { try { - // useBrowserFallback: true → 403 after retries auto-escalates to Playwright - const html = await this.fetchHtml(searchUrl, { retries: 2, useBrowserFallback: true }) - - if (html.length < 100) { - continue - } - - const $ = this.parseHtml(html) - - const results: AnimeSearchResult[] = [] - const seenIds = new Set() - this.selectCardElements($).each((_, el) => { - const card = this.extractCardData($(el), $) - if (!card || seenIds.has(card.id)) return - seenIds.add(card.id) - results.push(card) - }) - - if (results.length > 0) { - return results.slice(0, 24) - } - + const html = await this.fetchHtml(searchUrl, rawOptions) + candidates.push(...this.parseSearchCards(html)) + const ranked = rankSearchResults(candidates, normalizedQuery, 24) + if (ranked.length >= 12) return ranked } catch (error) { - // Site down — re-throw immediately, don't try other URLs if (error instanceof SiteDownError) throw error debugWarn('[Scraper] Search attempt failed for', searchUrl, error) } } - return [] + // One direct browser attempt handles pages that require rendering, without + // another native request or alternate-domain fan-out. + if (rankSearchResults(candidates, normalizedQuery, 24).length === 0) { + try { + const html = await this.fetchHtmlWithPlaywright(searchUrls[0], 20000) + candidates.push(...this.parseSearchCards(html)) + } catch (error) { + if (error instanceof SiteDownError) throw error + debugWarn('[Scraper] Search browser fallback failed for', searchUrls[0], error) + } + } + + const ranked = rankSearchResults(candidates, normalizedQuery, 24) + return ranked.length > 0 || !allowUnmatched ? ranked : candidates.slice(0, 24) + } + + async search(query: string): Promise { + return this.searchInternal(query) } async getAnimeDetail(id: string): Promise { diff --git a/scrapers/search-utils.ts b/scrapers/search-utils.ts new file mode 100644 index 0000000..b8b02e0 --- /dev/null +++ b/scrapers/search-utils.ts @@ -0,0 +1,122 @@ +export type SearchableResult = { + id: string + title: string + titleAlt?: string +} + +type NormalizedText = { + accented: string + folded: string + tokens: string[] +} + +type ScoredResult = { + item: T + score: number + index: number +} + +const DISPLAY_LABEL = '(?:vietsub|thuyết minh|lồng tiếng|fhd|hd|full|tập\\s+\\d+)' +const BRACKETED_LABEL = new RegExp(`\\s*[\\[(]\\s*${DISPLAY_LABEL}\\s*[\\])]\\s*$`, 'iu') +const TRAILING_LABEL = new RegExp(`(?:\\s*[-–—|:]\\s*|\\s+)${DISPLAY_LABEL}\\s*$`, 'iu') + +export function cleanSearchQuery(input: string): string { + return String(input || '') + .normalize('NFKC') + .replace(/\s+/g, ' ') + .trim() +} + +function stripDisplayLabels(input: string): string { + let value = input + let previous = '' + while (value !== previous) { + previous = value + value = value.replace(BRACKETED_LABEL, '').replace(TRAILING_LABEL, '').trim() + } + return value +} + +function normalizeText(input: string): NormalizedText { + const accented = stripDisplayLabels(cleanSearchQuery(input)) + .toLocaleLowerCase('vi-VN') + .replace(/[^\p{L}\p{N}]+/gu, ' ') + .replace(/\s+/g, ' ') + .trim() + const folded = accented + .normalize('NFD') + .replace(/[̀-ͯ]/g, '') + .replace(/đ/g, 'd') + const tokens = folded + .split(' ') + .filter((token) => token.length >= 3 || /^\d+$/.test(token)) + + return { accented, folded, tokens } +} + +function hasVietnameseMarks(text: NormalizedText): boolean { + return text.accented !== text.folded +} + +function containsWholePhrase(text: string, query: string): boolean { + return Boolean(query) && ` ${text} `.includes(` ${query} `) +} + +function startsWithWholePhrase(text: string, query: string): boolean { + return Boolean(query) && (text === query || text.startsWith(`${query} `)) +} + +function scoreTitle(title: string | undefined, query: NormalizedText, isAlt: boolean): number { + if (!title) return 0 + const candidate = normalizeText(title) + if (!candidate.folded || !query.folded) return 0 + + const offset = isAlt ? 30 : 0 + const queryHasMarks = hasVietnameseMarks(query) + const canUseBroadMatch = query.folded.length >= 3 || /^\d+$/.test(query.folded) + + if (queryHasMarks && candidate.accented === query.accented) return 1000 - offset + if (candidate.folded === query.folded) return 940 - offset + + if (canUseBroadMatch) { + if (queryHasMarks && startsWithWholePhrase(candidate.accented, query.accented)) return 850 - offset + if (startsWithWholePhrase(candidate.folded, query.folded)) return 800 - offset + if (queryHasMarks && containsWholePhrase(candidate.accented, query.accented)) return 700 - offset + if (containsWholePhrase(candidate.folded, query.folded)) return 650 - offset + } + + if (query.tokens.length === 0) return 0 + const candidateTokens = new Set(candidate.tokens) + const matchedTokens = query.tokens.filter((token) => candidateTokens.has(token)).length + const coverage = matchedTokens / query.tokens.length + if (coverage < 0.6) return 0 + + return Math.round(400 + coverage * 100) - (isAlt ? 20 : 0) +} + +export function rankSearchResults(items: T[], query: string, limit: number): T[] { + const normalizedQuery = normalizeText(query) + if (!normalizedQuery.folded || limit <= 0) return [] + + const byId = new Map>() + + items.forEach((item, index) => { + const score = Math.max( + scoreTitle(item.title, normalizedQuery, false), + scoreTitle(item.titleAlt, normalizedQuery, true) + ) + if (score === 0) return + + const id = cleanSearchQuery(item.id).toLocaleLowerCase('vi-VN') + if (!id) return + const current = byId.get(id) + if (!current || score > current.score) { + byId.set(id, { item, score, index }) + } + }) + + return [...byId.values()] + .sort((a, b) => b.score - a.score || a.index - b.index) + .slice(0, limit) + .map(({ item }) => item) +}