From 22508938b63dcc8ba266f9a2bc6108cd3e7bbe63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BA=91=E9=BE=99?= <76432572+nankingjing@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:47:59 +0800 Subject: [PATCH 1/3] fix(trending): decode HTML entities in scraped repo descriptions github.com/trending descriptions are HTML-escaped; the parser strips tags but left entities like &, ', < intact, so they showed up raw in the generated briefings. Decode named/decimal/hex entities after tag stripping. Search API results are already plain text and unaffected. --- src/trending.ts | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/src/trending.ts b/src/trending.ts index c0241ce..6b33162 100644 --- a/src/trending.ts +++ b/src/trending.ts @@ -45,6 +45,38 @@ const SEARCH_QUERIES = [ { q: "topic:machine-learning", label: "ml" }, ]; +// --------------------------------------------------------------------------- +// HTML entity decoding +// --------------------------------------------------------------------------- + +const NAMED_ENTITIES: Record = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: " ", +}; + +/** + * Decode the HTML entities that survive tag stripping on scraped github.com + * trending descriptions (e.g. `&`, `'`, `<`). Handles named, + * decimal (`&#NN;`) and hex (`&#xNN;`) references in a single pass so + * already-decoded text and unknown entities are left untouched. + */ +function decodeHtmlEntities(text: string): string { + return text.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, entity: string) => { + if (entity[0] === "#") { + const codePoint = + entity[1] === "x" || entity[1] === "X" + ? parseInt(entity.slice(2), 16) + : parseInt(entity.slice(1), 10); + return Number.isNaN(codePoint) ? match : String.fromCodePoint(codePoint); + } + return NAMED_ENTITIES[entity.toLowerCase()] ?? match; + }); +} + // --------------------------------------------------------------------------- // GitHub Trending HTML fetch // --------------------------------------------------------------------------- @@ -79,11 +111,15 @@ async function fetchGitHubTrending(): Promise<{ repos: TrendingRepo[]; success: // description from col-9 paragraph const descMatch = block.match(/]*class="[^"]*col-9[^"]*"[^>]*>([\s\S]*?)<\/p>/); - const description = descMatch?.[1] ? descMatch[1].replace(/<[^>]+>/g, "").trim() : ""; + const description = descMatch?.[1] + ? decodeHtmlEntities(descMatch[1].replace(/<[^>]+>/g, "").trim()) + : ""; // language const langMatch = block.match(/]+itemprop="programmingLanguage"[^>]*>([\s\S]*?)<\/span>/); - const language = langMatch?.[1] ? langMatch[1].replace(/<[^>]+>/g, "").trim() : ""; + const language = langMatch?.[1] + ? decodeHtmlEntities(langMatch[1].replace(/<[^>]+>/g, "").trim()) + : ""; // today stars const todayMatch = block.match(/([\d,]+)\s+stars?\s+today/i); From 7e02c39353d40d8c189f73aec828bbc211d8463b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BA=91=E9=BE=99?= <76432572+nankingjing@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:27:24 +0800 Subject: [PATCH 2/3] fix(trending): preserve invalid numeric entities --- src/trending.ts | 484 ++++++++++++++++++++++++------------------------ 1 file changed, 243 insertions(+), 241 deletions(-) diff --git a/src/trending.ts b/src/trending.ts index 6b33162..e3e8137 100644 --- a/src/trending.ts +++ b/src/trending.ts @@ -1,241 +1,243 @@ -/** - * GitHub trending and AI topic search data fetching. - */ - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface TrendingRepo { - fullName: string; - description: string; - language: string; - todayStars: number; - totalStars: number; - forks: number; - url: string; -} - -export interface SearchRepo { - fullName: string; - description: string | null; - language: string | null; - stargazersCount: number; - pushedAt: string; - url: string; - searchQuery: string; -} - -export interface TrendingData { - trendingRepos: TrendingRepo[]; - searchRepos: SearchRepo[]; - trendingFetchSuccess: boolean; -} - -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -const SEARCH_QUERIES = [ - { q: "topic:llm", label: "llm" }, - { q: "topic:ai-agent", label: "ai-agent" }, - { q: "topic:rag", label: "rag" }, - { q: "topic:vector-database", label: "vector-db" }, - { q: "topic:large-language-model", label: "llm-model" }, - { q: "topic:machine-learning", label: "ml" }, -]; - -// --------------------------------------------------------------------------- -// HTML entity decoding -// --------------------------------------------------------------------------- - -const NAMED_ENTITIES: Record = { - amp: "&", - lt: "<", - gt: ">", - quot: '"', - apos: "'", - nbsp: " ", -}; - -/** - * Decode the HTML entities that survive tag stripping on scraped github.com - * trending descriptions (e.g. `&`, `'`, `<`). Handles named, - * decimal (`&#NN;`) and hex (`&#xNN;`) references in a single pass so - * already-decoded text and unknown entities are left untouched. - */ -function decodeHtmlEntities(text: string): string { - return text.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, entity: string) => { - if (entity[0] === "#") { - const codePoint = - entity[1] === "x" || entity[1] === "X" - ? parseInt(entity.slice(2), 16) - : parseInt(entity.slice(1), 10); - return Number.isNaN(codePoint) ? match : String.fromCodePoint(codePoint); - } - return NAMED_ENTITIES[entity.toLowerCase()] ?? match; - }); -} - -// --------------------------------------------------------------------------- -// GitHub Trending HTML fetch -// --------------------------------------------------------------------------- - -async function fetchGitHubTrending(): Promise<{ repos: TrendingRepo[]; success: boolean }> { - try { - const resp = await fetch("https://github.com/trending?since=daily&spoken_language_code=", { - headers: { - "User-Agent": "Mozilla/5.0 (compatible; big-model-radar/1.0)", - Accept: "text/html", - }, - }); - if (!resp.ok) { - console.error(` [trending] HTTP ${resp.status} fetching github.com/trending`); - return { repos: [], success: false }; - } - - const html = await resp.text(); - const repos: TrendingRepo[] = []; - - // Split by article blocks - const articlePattern = - /]*class="[^"]*Box-row[^"]*"[\s\S]*?(?=]*class="[^"]*Box-row[^"]*"|$)/g; - const blocks = html.match(articlePattern) ?? []; - - for (const block of blocks) { - try { - // fullName from

> - const nameMatch = block.match(/]*>[\s\S]*?]+href="\/([^/"]+\/[^/"]+)"/); - if (!nameMatch?.[1]) continue; - const fullName = nameMatch[1].trim(); - - // description from col-9 paragraph - const descMatch = block.match(/]*class="[^"]*col-9[^"]*"[^>]*>([\s\S]*?)<\/p>/); - const description = descMatch?.[1] - ? decodeHtmlEntities(descMatch[1].replace(/<[^>]+>/g, "").trim()) - : ""; - - // language - const langMatch = block.match(/]+itemprop="programmingLanguage"[^>]*>([\s\S]*?)<\/span>/); - const language = langMatch?.[1] - ? decodeHtmlEntities(langMatch[1].replace(/<[^>]+>/g, "").trim()) - : ""; - - // today stars - const todayMatch = block.match(/([\d,]+)\s+stars?\s+today/i); - const todayStars = todayMatch?.[1] ? parseInt(todayMatch[1].replace(/,/g, ""), 10) : 0; - - // total stars — look for link with /stargazers - const totalMatch = block.match(/href="\/[^"]+\/stargazers"[^>]*>\s*<[^>]+>\s*([\d,]+)/); - const totalStars = totalMatch?.[1] ? parseInt(totalMatch[1].replace(/,/g, ""), 10) : 0; - - // forks - const forkMatch = block.match(/href="\/[^"]+\/forks"[^>]*>\s*<[^>]+>\s*([\d,]+)/); - const forks = forkMatch?.[1] ? parseInt(forkMatch[1].replace(/,/g, ""), 10) : 0; - - repos.push({ - fullName, - description, - language, - todayStars, - totalStars, - forks, - url: `https://github.com/${fullName}`, - }); - } catch { - // single block parse failure is non-fatal - } - } - - if (repos.length === 0) { - console.error(" [trending] Parsed 0 repos — HTML structure may have changed"); - return { repos: [], success: false }; - } - - console.log(` [trending] Parsed ${repos.length} trending repos from HTML`); - return { repos, success: true }; - } catch (err) { - console.error(` [trending] Fetch failed: ${err}`); - return { repos: [], success: false }; - } -} - -// --------------------------------------------------------------------------- -// GitHub Search API -// --------------------------------------------------------------------------- - -interface SearchApiItem { - full_name: string; - description: string | null; - language: string | null; - stargazers_count: number; - pushed_at: string; - html_url: string; -} - -interface SearchApiResponse { - items: SearchApiItem[]; -} - -async function searchAiRepos(sevenDaysAgo: string): Promise { - const token = process.env["GITHUB_TOKEN"] ?? ""; - const headers: Record = { - Accept: "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - }; - if (token) headers["Authorization"] = `Bearer ${token}`; - - const seen = new Set(); - const all: SearchRepo[] = []; - - await Promise.all( - SEARCH_QUERIES.map(async ({ q, label }) => { - try { - const query = `${q}+pushed:>${sevenDaysAgo}&sort=stars&order=desc`; - const url = `https://api.github.com/search/repositories?q=${query}&per_page=15`; - const resp = await fetch(url, { headers }); - if (!resp.ok) { - console.error(` [trending/search] "${label}": HTTP ${resp.status}`); - return; - } - const data = (await resp.json()) as SearchApiResponse; - let added = 0; - for (const item of data.items ?? []) { - if (!seen.has(item.full_name)) { - seen.add(item.full_name); - all.push({ - fullName: item.full_name, - description: item.description, - language: item.language, - stargazersCount: item.stargazers_count, - pushedAt: item.pushed_at, - url: item.html_url, - searchQuery: label, - }); - added++; - } - } - console.log(` [trending/search] "${label}": ${added} new repos`); - } catch (err) { - console.error(` [trending/search] "${label}": ${err}`); - } - }), - ); - - return all; -} - -// --------------------------------------------------------------------------- -// Export -// --------------------------------------------------------------------------- - -export async function fetchTrendingData(): Promise { - const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString().slice(0, 10); - - const [{ repos: trendingRepos, success }, searchRepos] = await Promise.all([ - fetchGitHubTrending(), - searchAiRepos(sevenDaysAgo), - ]); - - return { trendingRepos, searchRepos, trendingFetchSuccess: success }; -} +/** + * GitHub trending and AI topic search data fetching. + */ + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface TrendingRepo { + fullName: string; + description: string; + language: string; + todayStars: number; + totalStars: number; + forks: number; + url: string; +} + +export interface SearchRepo { + fullName: string; + description: string | null; + language: string | null; + stargazersCount: number; + pushedAt: string; + url: string; + searchQuery: string; +} + +export interface TrendingData { + trendingRepos: TrendingRepo[]; + searchRepos: SearchRepo[]; + trendingFetchSuccess: boolean; +} + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const SEARCH_QUERIES = [ + { q: "topic:llm", label: "llm" }, + { q: "topic:ai-agent", label: "ai-agent" }, + { q: "topic:rag", label: "rag" }, + { q: "topic:vector-database", label: "vector-db" }, + { q: "topic:large-language-model", label: "llm-model" }, + { q: "topic:machine-learning", label: "ml" }, +]; + +// --------------------------------------------------------------------------- +// HTML entity decoding +// --------------------------------------------------------------------------- + +const NAMED_ENTITIES: Record = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: " ", +}; + +/** + * Decode the HTML entities that survive tag stripping on scraped github.com + * trending descriptions (e.g. `&`, `'`, `<`). Handles named, + * decimal (`&#NN;`) and hex (`&#xNN;`) references in a single pass so + * already-decoded text and unknown entities are left untouched. + */ +function decodeHtmlEntities(text: string): string { + return text.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, entity: string) => { + if (entity[0] === "#") { + const codePoint = + entity[1] === "x" || entity[1] === "X" + ? parseInt(entity.slice(2), 16) + : parseInt(entity.slice(1), 10); + return Number.isNaN(codePoint) || codePoint > 0x10ffff + ? match + : String.fromCodePoint(codePoint); + } + return NAMED_ENTITIES[entity.toLowerCase()] ?? match; + }); +} + +// --------------------------------------------------------------------------- +// GitHub Trending HTML fetch +// --------------------------------------------------------------------------- + +async function fetchGitHubTrending(): Promise<{ repos: TrendingRepo[]; success: boolean }> { + try { + const resp = await fetch("https://github.com/trending?since=daily&spoken_language_code=", { + headers: { + "User-Agent": "Mozilla/5.0 (compatible; big-model-radar/1.0)", + Accept: "text/html", + }, + }); + if (!resp.ok) { + console.error(` [trending] HTTP ${resp.status} fetching github.com/trending`); + return { repos: [], success: false }; + } + + const html = await resp.text(); + const repos: TrendingRepo[] = []; + + // Split by article blocks + const articlePattern = + /]*class="[^"]*Box-row[^"]*"[\s\S]*?(?=]*class="[^"]*Box-row[^"]*"|$)/g; + const blocks = html.match(articlePattern) ?? []; + + for (const block of blocks) { + try { + // fullName from

> + const nameMatch = block.match(/]*>[\s\S]*?]+href="\/([^/"]+\/[^/"]+)"/); + if (!nameMatch?.[1]) continue; + const fullName = nameMatch[1].trim(); + + // description from col-9 paragraph + const descMatch = block.match(/]*class="[^"]*col-9[^"]*"[^>]*>([\s\S]*?)<\/p>/); + const description = descMatch?.[1] + ? decodeHtmlEntities(descMatch[1].replace(/<[^>]+>/g, "").trim()) + : ""; + + // language + const langMatch = block.match(/]+itemprop="programmingLanguage"[^>]*>([\s\S]*?)<\/span>/); + const language = langMatch?.[1] + ? decodeHtmlEntities(langMatch[1].replace(/<[^>]+>/g, "").trim()) + : ""; + + // today stars + const todayMatch = block.match(/([\d,]+)\s+stars?\s+today/i); + const todayStars = todayMatch?.[1] ? parseInt(todayMatch[1].replace(/,/g, ""), 10) : 0; + + // total stars — look for link with /stargazers + const totalMatch = block.match(/href="\/[^"]+\/stargazers"[^>]*>\s*<[^>]+>\s*([\d,]+)/); + const totalStars = totalMatch?.[1] ? parseInt(totalMatch[1].replace(/,/g, ""), 10) : 0; + + // forks + const forkMatch = block.match(/href="\/[^"]+\/forks"[^>]*>\s*<[^>]+>\s*([\d,]+)/); + const forks = forkMatch?.[1] ? parseInt(forkMatch[1].replace(/,/g, ""), 10) : 0; + + repos.push({ + fullName, + description, + language, + todayStars, + totalStars, + forks, + url: `https://github.com/${fullName}`, + }); + } catch { + // single block parse failure is non-fatal + } + } + + if (repos.length === 0) { + console.error(" [trending] Parsed 0 repos — HTML structure may have changed"); + return { repos: [], success: false }; + } + + console.log(` [trending] Parsed ${repos.length} trending repos from HTML`); + return { repos, success: true }; + } catch (err) { + console.error(` [trending] Fetch failed: ${err}`); + return { repos: [], success: false }; + } +} + +// --------------------------------------------------------------------------- +// GitHub Search API +// --------------------------------------------------------------------------- + +interface SearchApiItem { + full_name: string; + description: string | null; + language: string | null; + stargazers_count: number; + pushed_at: string; + html_url: string; +} + +interface SearchApiResponse { + items: SearchApiItem[]; +} + +async function searchAiRepos(sevenDaysAgo: string): Promise { + const token = process.env["GITHUB_TOKEN"] ?? ""; + const headers: Record = { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }; + if (token) headers["Authorization"] = `Bearer ${token}`; + + const seen = new Set(); + const all: SearchRepo[] = []; + + await Promise.all( + SEARCH_QUERIES.map(async ({ q, label }) => { + try { + const query = `${q}+pushed:>${sevenDaysAgo}&sort=stars&order=desc`; + const url = `https://api.github.com/search/repositories?q=${query}&per_page=15`; + const resp = await fetch(url, { headers }); + if (!resp.ok) { + console.error(` [trending/search] "${label}": HTTP ${resp.status}`); + return; + } + const data = (await resp.json()) as SearchApiResponse; + let added = 0; + for (const item of data.items ?? []) { + if (!seen.has(item.full_name)) { + seen.add(item.full_name); + all.push({ + fullName: item.full_name, + description: item.description, + language: item.language, + stargazersCount: item.stargazers_count, + pushedAt: item.pushed_at, + url: item.html_url, + searchQuery: label, + }); + added++; + } + } + console.log(` [trending/search] "${label}": ${added} new repos`); + } catch (err) { + console.error(` [trending/search] "${label}": ${err}`); + } + }), + ); + + return all; +} + +// --------------------------------------------------------------------------- +// Export +// --------------------------------------------------------------------------- + +export async function fetchTrendingData(): Promise { + const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString().slice(0, 10); + + const [{ repos: trendingRepos, success }, searchRepos] = await Promise.all([ + fetchGitHubTrending(), + searchAiRepos(sevenDaysAgo), + ]); + + return { trendingRepos, searchRepos, trendingFetchSuccess: success }; +} From afc47e75f9646082fe4f3b1eb1f5d116bbe9cc9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BA=91=E9=BE=99?= <76432572+nankingjing@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:29:43 +0800 Subject: [PATCH 3/3] fix(trending): preserve invalid numeric entities --- src/trending.ts | 486 ++++++++++++++++++++++++------------------------ 1 file changed, 243 insertions(+), 243 deletions(-) diff --git a/src/trending.ts b/src/trending.ts index e3e8137..caef815 100644 --- a/src/trending.ts +++ b/src/trending.ts @@ -1,243 +1,243 @@ -/** - * GitHub trending and AI topic search data fetching. - */ - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface TrendingRepo { - fullName: string; - description: string; - language: string; - todayStars: number; - totalStars: number; - forks: number; - url: string; -} - -export interface SearchRepo { - fullName: string; - description: string | null; - language: string | null; - stargazersCount: number; - pushedAt: string; - url: string; - searchQuery: string; -} - -export interface TrendingData { - trendingRepos: TrendingRepo[]; - searchRepos: SearchRepo[]; - trendingFetchSuccess: boolean; -} - -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -const SEARCH_QUERIES = [ - { q: "topic:llm", label: "llm" }, - { q: "topic:ai-agent", label: "ai-agent" }, - { q: "topic:rag", label: "rag" }, - { q: "topic:vector-database", label: "vector-db" }, - { q: "topic:large-language-model", label: "llm-model" }, - { q: "topic:machine-learning", label: "ml" }, -]; - -// --------------------------------------------------------------------------- -// HTML entity decoding -// --------------------------------------------------------------------------- - -const NAMED_ENTITIES: Record = { - amp: "&", - lt: "<", - gt: ">", - quot: '"', - apos: "'", - nbsp: " ", -}; - -/** - * Decode the HTML entities that survive tag stripping on scraped github.com - * trending descriptions (e.g. `&`, `'`, `<`). Handles named, - * decimal (`&#NN;`) and hex (`&#xNN;`) references in a single pass so - * already-decoded text and unknown entities are left untouched. - */ -function decodeHtmlEntities(text: string): string { - return text.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, entity: string) => { - if (entity[0] === "#") { - const codePoint = - entity[1] === "x" || entity[1] === "X" - ? parseInt(entity.slice(2), 16) - : parseInt(entity.slice(1), 10); - return Number.isNaN(codePoint) || codePoint > 0x10ffff - ? match - : String.fromCodePoint(codePoint); - } - return NAMED_ENTITIES[entity.toLowerCase()] ?? match; - }); -} - -// --------------------------------------------------------------------------- -// GitHub Trending HTML fetch -// --------------------------------------------------------------------------- - -async function fetchGitHubTrending(): Promise<{ repos: TrendingRepo[]; success: boolean }> { - try { - const resp = await fetch("https://github.com/trending?since=daily&spoken_language_code=", { - headers: { - "User-Agent": "Mozilla/5.0 (compatible; big-model-radar/1.0)", - Accept: "text/html", - }, - }); - if (!resp.ok) { - console.error(` [trending] HTTP ${resp.status} fetching github.com/trending`); - return { repos: [], success: false }; - } - - const html = await resp.text(); - const repos: TrendingRepo[] = []; - - // Split by article blocks - const articlePattern = - /]*class="[^"]*Box-row[^"]*"[\s\S]*?(?=]*class="[^"]*Box-row[^"]*"|$)/g; - const blocks = html.match(articlePattern) ?? []; - - for (const block of blocks) { - try { - // fullName from

> - const nameMatch = block.match(/]*>[\s\S]*?]+href="\/([^/"]+\/[^/"]+)"/); - if (!nameMatch?.[1]) continue; - const fullName = nameMatch[1].trim(); - - // description from col-9 paragraph - const descMatch = block.match(/]*class="[^"]*col-9[^"]*"[^>]*>([\s\S]*?)<\/p>/); - const description = descMatch?.[1] - ? decodeHtmlEntities(descMatch[1].replace(/<[^>]+>/g, "").trim()) - : ""; - - // language - const langMatch = block.match(/]+itemprop="programmingLanguage"[^>]*>([\s\S]*?)<\/span>/); - const language = langMatch?.[1] - ? decodeHtmlEntities(langMatch[1].replace(/<[^>]+>/g, "").trim()) - : ""; - - // today stars - const todayMatch = block.match(/([\d,]+)\s+stars?\s+today/i); - const todayStars = todayMatch?.[1] ? parseInt(todayMatch[1].replace(/,/g, ""), 10) : 0; - - // total stars — look for link with /stargazers - const totalMatch = block.match(/href="\/[^"]+\/stargazers"[^>]*>\s*<[^>]+>\s*([\d,]+)/); - const totalStars = totalMatch?.[1] ? parseInt(totalMatch[1].replace(/,/g, ""), 10) : 0; - - // forks - const forkMatch = block.match(/href="\/[^"]+\/forks"[^>]*>\s*<[^>]+>\s*([\d,]+)/); - const forks = forkMatch?.[1] ? parseInt(forkMatch[1].replace(/,/g, ""), 10) : 0; - - repos.push({ - fullName, - description, - language, - todayStars, - totalStars, - forks, - url: `https://github.com/${fullName}`, - }); - } catch { - // single block parse failure is non-fatal - } - } - - if (repos.length === 0) { - console.error(" [trending] Parsed 0 repos — HTML structure may have changed"); - return { repos: [], success: false }; - } - - console.log(` [trending] Parsed ${repos.length} trending repos from HTML`); - return { repos, success: true }; - } catch (err) { - console.error(` [trending] Fetch failed: ${err}`); - return { repos: [], success: false }; - } -} - -// --------------------------------------------------------------------------- -// GitHub Search API -// --------------------------------------------------------------------------- - -interface SearchApiItem { - full_name: string; - description: string | null; - language: string | null; - stargazers_count: number; - pushed_at: string; - html_url: string; -} - -interface SearchApiResponse { - items: SearchApiItem[]; -} - -async function searchAiRepos(sevenDaysAgo: string): Promise { - const token = process.env["GITHUB_TOKEN"] ?? ""; - const headers: Record = { - Accept: "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - }; - if (token) headers["Authorization"] = `Bearer ${token}`; - - const seen = new Set(); - const all: SearchRepo[] = []; - - await Promise.all( - SEARCH_QUERIES.map(async ({ q, label }) => { - try { - const query = `${q}+pushed:>${sevenDaysAgo}&sort=stars&order=desc`; - const url = `https://api.github.com/search/repositories?q=${query}&per_page=15`; - const resp = await fetch(url, { headers }); - if (!resp.ok) { - console.error(` [trending/search] "${label}": HTTP ${resp.status}`); - return; - } - const data = (await resp.json()) as SearchApiResponse; - let added = 0; - for (const item of data.items ?? []) { - if (!seen.has(item.full_name)) { - seen.add(item.full_name); - all.push({ - fullName: item.full_name, - description: item.description, - language: item.language, - stargazersCount: item.stargazers_count, - pushedAt: item.pushed_at, - url: item.html_url, - searchQuery: label, - }); - added++; - } - } - console.log(` [trending/search] "${label}": ${added} new repos`); - } catch (err) { - console.error(` [trending/search] "${label}": ${err}`); - } - }), - ); - - return all; -} - -// --------------------------------------------------------------------------- -// Export -// --------------------------------------------------------------------------- - -export async function fetchTrendingData(): Promise { - const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString().slice(0, 10); - - const [{ repos: trendingRepos, success }, searchRepos] = await Promise.all([ - fetchGitHubTrending(), - searchAiRepos(sevenDaysAgo), - ]); - - return { trendingRepos, searchRepos, trendingFetchSuccess: success }; -} +/** + * GitHub trending and AI topic search data fetching. + */ + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface TrendingRepo { + fullName: string; + description: string; + language: string; + todayStars: number; + totalStars: number; + forks: number; + url: string; +} + +export interface SearchRepo { + fullName: string; + description: string | null; + language: string | null; + stargazersCount: number; + pushedAt: string; + url: string; + searchQuery: string; +} + +export interface TrendingData { + trendingRepos: TrendingRepo[]; + searchRepos: SearchRepo[]; + trendingFetchSuccess: boolean; +} + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const SEARCH_QUERIES = [ + { q: "topic:llm", label: "llm" }, + { q: "topic:ai-agent", label: "ai-agent" }, + { q: "topic:rag", label: "rag" }, + { q: "topic:vector-database", label: "vector-db" }, + { q: "topic:large-language-model", label: "llm-model" }, + { q: "topic:machine-learning", label: "ml" }, +]; + +// --------------------------------------------------------------------------- +// HTML entity decoding +// --------------------------------------------------------------------------- + +const NAMED_ENTITIES: Record = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: " ", +}; + +/** + * Decode the HTML entities that survive tag stripping on scraped github.com + * trending descriptions (e.g. `&`, `'`, `<`). Handles named, + * decimal (`&#NN;`) and hex (`&#xNN;`) references in a single pass so + * already-decoded text and unknown entities are left untouched. + */ +function decodeHtmlEntities(text: string): string { + return text.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, entity: string) => { + if (entity[0] === "#") { + const codePoint = + entity[1] === "x" || entity[1] === "X" + ? parseInt(entity.slice(2), 16) + : parseInt(entity.slice(1), 10); + return Number.isNaN(codePoint) || codePoint > 0x10ffff + ? match + : String.fromCodePoint(codePoint); + } + return NAMED_ENTITIES[entity.toLowerCase()] ?? match; + }); +} + +// --------------------------------------------------------------------------- +// GitHub Trending HTML fetch +// --------------------------------------------------------------------------- + +async function fetchGitHubTrending(): Promise<{ repos: TrendingRepo[]; success: boolean }> { + try { + const resp = await fetch("https://github.com/trending?since=daily&spoken_language_code=", { + headers: { + "User-Agent": "Mozilla/5.0 (compatible; big-model-radar/1.0)", + Accept: "text/html", + }, + }); + if (!resp.ok) { + console.error(` [trending] HTTP ${resp.status} fetching github.com/trending`); + return { repos: [], success: false }; + } + + const html = await resp.text(); + const repos: TrendingRepo[] = []; + + // Split by article blocks + const articlePattern = + /]*class="[^"]*Box-row[^"]*"[\s\S]*?(?=]*class="[^"]*Box-row[^"]*"|$)/g; + const blocks = html.match(articlePattern) ?? []; + + for (const block of blocks) { + try { + // fullName from

> + const nameMatch = block.match(/]*>[\s\S]*?]+href="\/([^/"]+\/[^/"]+)"/); + if (!nameMatch?.[1]) continue; + const fullName = nameMatch[1].trim(); + + // description from col-9 paragraph + const descMatch = block.match(/]*class="[^"]*col-9[^"]*"[^>]*>([\s\S]*?)<\/p>/); + const description = descMatch?.[1] + ? decodeHtmlEntities(descMatch[1].replace(/<[^>]+>/g, "").trim()) + : ""; + + // language + const langMatch = block.match(/]+itemprop="programmingLanguage"[^>]*>([\s\S]*?)<\/span>/); + const language = langMatch?.[1] + ? decodeHtmlEntities(langMatch[1].replace(/<[^>]+>/g, "").trim()) + : ""; + + // today stars + const todayMatch = block.match(/([\d,]+)\s+stars?\s+today/i); + const todayStars = todayMatch?.[1] ? parseInt(todayMatch[1].replace(/,/g, ""), 10) : 0; + + // total stars — look for link with /stargazers + const totalMatch = block.match(/href="\/[^"]+\/stargazers"[^>]*>\s*<[^>]+>\s*([\d,]+)/); + const totalStars = totalMatch?.[1] ? parseInt(totalMatch[1].replace(/,/g, ""), 10) : 0; + + // forks + const forkMatch = block.match(/href="\/[^"]+\/forks"[^>]*>\s*<[^>]+>\s*([\d,]+)/); + const forks = forkMatch?.[1] ? parseInt(forkMatch[1].replace(/,/g, ""), 10) : 0; + + repos.push({ + fullName, + description, + language, + todayStars, + totalStars, + forks, + url: `https://github.com/${fullName}`, + }); + } catch { + // single block parse failure is non-fatal + } + } + + if (repos.length === 0) { + console.error(" [trending] Parsed 0 repos — HTML structure may have changed"); + return { repos: [], success: false }; + } + + console.log(` [trending] Parsed ${repos.length} trending repos from HTML`); + return { repos, success: true }; + } catch (err) { + console.error(` [trending] Fetch failed: ${err}`); + return { repos: [], success: false }; + } +} + +// --------------------------------------------------------------------------- +// GitHub Search API +// --------------------------------------------------------------------------- + +interface SearchApiItem { + full_name: string; + description: string | null; + language: string | null; + stargazers_count: number; + pushed_at: string; + html_url: string; +} + +interface SearchApiResponse { + items: SearchApiItem[]; +} + +async function searchAiRepos(sevenDaysAgo: string): Promise { + const token = process.env["GITHUB_TOKEN"] ?? ""; + const headers: Record = { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }; + if (token) headers["Authorization"] = `Bearer ${token}`; + + const seen = new Set(); + const all: SearchRepo[] = []; + + await Promise.all( + SEARCH_QUERIES.map(async ({ q, label }) => { + try { + const query = `${q}+pushed:>${sevenDaysAgo}&sort=stars&order=desc`; + const url = `https://api.github.com/search/repositories?q=${query}&per_page=15`; + const resp = await fetch(url, { headers }); + if (!resp.ok) { + console.error(` [trending/search] "${label}": HTTP ${resp.status}`); + return; + } + const data = (await resp.json()) as SearchApiResponse; + let added = 0; + for (const item of data.items ?? []) { + if (!seen.has(item.full_name)) { + seen.add(item.full_name); + all.push({ + fullName: item.full_name, + description: item.description, + language: item.language, + stargazersCount: item.stargazers_count, + pushedAt: item.pushed_at, + url: item.html_url, + searchQuery: label, + }); + added++; + } + } + console.log(` [trending/search] "${label}": ${added} new repos`); + } catch (err) { + console.error(` [trending/search] "${label}": ${err}`); + } + }), + ); + + return all; +} + +// --------------------------------------------------------------------------- +// Export +// --------------------------------------------------------------------------- + +export async function fetchTrendingData(): Promise { + const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString().slice(0, 10); + + const [{ repos: trendingRepos, success }, searchRepos] = await Promise.all([ + fetchGitHubTrending(), + searchAiRepos(sevenDaysAgo), + ]); + + return { trendingRepos, searchRepos, trendingFetchSuccess: success }; +}