From a413c334c52d83afd79d399dfa359d0eeed87060 Mon Sep 17 00:00:00 2001 From: xucroyuri Date: Sun, 28 Jun 2026 15:19:29 +0800 Subject: [PATCH 01/16] fix: restore data backfill and translation fallback --- config/provider/routes.yaml | 30 ++++++++++++++++++++--- tests/unit/test_cloudflare_d1_backfill.py | 3 ++- tests/unit/test_provider_router.py | 4 +++ tools/cloudflare_d1_backfill.py | 23 +++++++++++++++-- 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/config/provider/routes.yaml b/config/provider/routes.yaml index d4e23773..e178640f 100644 --- a/config/provider/routes.yaml +++ b/config/provider/routes.yaml @@ -3,7 +3,7 @@ # ADR: ADR-0005 # # AI Provider 路由配置 — 内置多 Provider 链式兜底版 -# 所有 AI 能力由内置 Provider 直接调用：gemini → deepseek → groq → cloudflare_workers_ai → local。 +# 所有 AI 能力由内置 Provider 直接调用：gemini → deepseek → groq → cloudflare_workers_ai → openrouter → local。 # 各 Provider 通过自身的 API key 环境变量鉴权，不再经过中间层 sidecar。 # routes_version: "2.0.0" @@ -107,9 +107,21 @@ routes: output_schema_ref: null audit: true fallback_route_ids: - - fallback.local + - translate.openrouter notes: "翻译兜底链：Groq 失败后由 Cloudflare Workers AI 接管" + - route_id: translate.openrouter + task_type: translate + provider: openrouter + model: google/gemma-4-26b-a4b-it:free + timeout_seconds: 90 + max_cost_usd_per_call: 0.0 + output_schema_ref: null + audit: true + fallback_route_ids: + - fallback.local + notes: "翻译兜底链：Cloudflare Workers AI 失败后由 OpenRouter 免费模型接管" + - route_id: translate.batch task_type: translate provider: gemini @@ -193,9 +205,21 @@ routes: output_schema_ref: null audit: true fallback_route_ids: - - translate.deepseek + - ai.enrichment.openrouter notes: "低频 AI 增强合并任务：翻译、摘要、聚类、复核 — Gemini 主路由" + - route_id: ai.enrichment.openrouter + task_type: ai_enrichment + provider: openrouter + model: google/gemma-4-26b-a4b-it:free + timeout_seconds: 120 + max_cost_usd_per_call: 0.0 + output_schema_ref: null + audit: true + fallback_route_ids: + - fallback.local + notes: "低频 AI 增强兜底：Gemini 不可用时由 OpenRouter 免费模型接管" + - route_id: ai.enrichment.nvidia task_type: ai_enrichment provider: deepseek diff --git a/tests/unit/test_cloudflare_d1_backfill.py b/tests/unit/test_cloudflare_d1_backfill.py index 26d01f3b..dce1e375 100644 --- a/tests/unit/test_cloudflare_d1_backfill.py +++ b/tests/unit/test_cloudflare_d1_backfill.py @@ -55,7 +55,7 @@ def _make_state_db(path: Path) -> None: "politics", "Draft story", "https://example.com/draft", - "2026-06-25T01:00:00+00:00", + "20260529T234500Z", "2026-06-25T01:01:00+00:00", "politics,italy", "Rome,EU", @@ -97,6 +97,7 @@ def test_collect_backfill_plan_uses_only_drafts_and_counts_targets(tmp_path: Pat plan = collect_backfill_plan(data_dir=data_dir, targets_dir=targets_dir) assert [event.event_id for event in plan.events] == ["draft-1"] + assert plan.events[0].published_at == "2026-05-29T23:45:00Z" assert plan.targets[0].target_id == "italy" assert plan.targets[0].event_count == 1 assert plan.sources[0].source_id == "ansa" diff --git a/tests/unit/test_provider_router.py b/tests/unit/test_provider_router.py index 68a6c69d..0d5ef114 100644 --- a/tests/unit/test_provider_router.py +++ b/tests/unit/test_provider_router.py @@ -171,10 +171,14 @@ def test_public_translation_route_uses_builtin_chain() -> None: # 应有 fallback 链 assert len(public_translation.fallback_route_ids) >= 1 assert public_translation.provider == "gemini" + assert routes["translate.cloudflare"].fallback_route_ids == ["translate.openrouter"] + assert routes["translate.openrouter"].provider == "openrouter" # 公共发布的路由链不应有空 assert routes["public.summary_reason"].provider != "freellmapi" assert routes["public.enrichment"].provider != "freellmapi" + assert routes["ai.enrichment.batch"].fallback_route_ids == ["ai.enrichment.openrouter"] + assert routes["ai.enrichment.openrouter"].provider == "openrouter" def test_provider_route_defaults() -> None: diff --git a/tools/cloudflare_d1_backfill.py b/tools/cloudflare_d1_backfill.py index bb920b83..29333ed1 100644 --- a/tools/cloudflare_d1_backfill.py +++ b/tools/cloudflare_d1_backfill.py @@ -4,6 +4,7 @@ import argparse import json +import re import sqlite3 from contextlib import closing from dataclasses import dataclass @@ -73,6 +74,24 @@ def _json(value: Any) -> str: return json.dumps(value, ensure_ascii=False, separators=(",", ":")) +_COMPACT_UTC_RE = re.compile( + r"^(?P\d{4})(?P\d{2})(?P\d{2})T" + r"(?P\d{2})(?P\d{2})(?P\d{2})Z$" +) + + +def _normalize_timestamp(value: Any) -> str: + text = str(value or "").strip() + match = _COMPACT_UTC_RE.match(text) + if match: + parts = match.groupdict() + return ( + f"{parts['date']}-{parts['month']}-{parts['day']}T" + f"{parts['hour']}:{parts['minute']}:{parts['second']}Z" + ) + return text + + def _split_csv(value: str | None) -> list[str]: if not value: return [] @@ -155,8 +174,8 @@ def collect_backfill_plan( topic_tags = _split_csv(row["topic_tags"]) issue_tags = [str(row["classification_l0"])] if row["classification_l0"] else [] entities = [{"name": name} for name in _split_csv(row["entity_names"])] - published_at = str(row["published_at"] or row["created_at"]) - collected_at = str(row["created_at"] or row["published_at"]) + published_at = _normalize_timestamp(row["published_at"] or row["created_at"]) + collected_at = _normalize_timestamp(row["created_at"] or row["published_at"]) events.append( D1Event( From 8757ecd9f54f96b5fbbeffcfbbbc48e198d3bdc1 Mon Sep 17 00:00:00 2001 From: xucroyuri Date: Sun, 28 Jun 2026 18:58:55 +0800 Subject: [PATCH 02/16] feat: migrate FreeLLMAPI keys to direct providers --- AGENTS.md | 6 +- config/provider/routes.yaml | 90 +++++++++++-- docs/architecture.md | 23 +++- docs/external-integration-strategy.md | 6 +- frontend/cloudflare/workers/index.ts | 54 ++++++-- frontend/cloudflare/wrangler.toml | 20 +++ .../providers/freeapi_compat_providers.py | 75 +++++++++++ .../adapters/providers/openai_provider.py | 123 ++++++++++++------ src/news_sentry/core/run.py | 20 ++- tests/unit/test_openai_provider.py | 34 +++++ tests/unit/test_provider_factory.py | 16 +++ tests/unit/test_provider_router.py | 12 ++ 12 files changed, 413 insertions(+), 66 deletions(-) create mode 100644 src/news_sentry/adapters/providers/freeapi_compat_providers.py diff --git a/AGENTS.md b/AGENTS.md index d380203e..3f6f64e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ flowchart TB end subgraph ADAPTERS["外部适配"] - AP["AI Providers
Gemini → DeepSeek → Groq → Cloudflare"] + AP["AI Providers
Gemini → DeepSeek → Groq → Cloudflare → OpenRouter → NVIDIA/Agnes/OpenCode/Reka"] RT["Runtime Adapters
RSS-Bridge / API"] end @@ -173,7 +173,7 @@ flowchart TD | 规则 | 内容 | 参考 | |------|------|------| | **前端策略** | CLI-first，FastAPI + Vanilla JS 可选，无重型框架 | ADR-0025（替代 ADR-0010） | -| **AI Provider** | 内置 chain: Gemini → DeepSeek → Groq → Cloudflare Workers AI | ADR-0005 | +| **AI Provider** | 内置 chain: Gemini → DeepSeek → Groq → Cloudflare Workers AI → OpenRouter → NVIDIA/Agnes/OpenCode/Reka | ADR-0005 | | **分类存储** | L0-L3 走 `metadata.classification`，不做顶层字段 | ADR-0009 | | **实现语言** | Python 3.11+ / Pydantic v2 | ADR-0012 | | **配置管理** | 所有国家参数入 config/，禁止硬编码到 src/ | ADR-0015 | @@ -371,6 +371,6 @@ flowchart LR - **测试规模**：2,738 tests, 85% 覆盖率, ruff=0, mypy=0, frontend=0 - **监控目标**：81 targets (italy, china-watch-en, japan, germany, france + 76 更多) - **信源规模**：244 源 (147 RSS + 97 API)，覆盖 81 个 target -- **AI Provider**：内置 chain: Gemini → DeepSeek → Groq → Cloudflare Workers AI +- **AI Provider**：内置 chain: Gemini → DeepSeek → Groq → Cloudflare Workers AI → OpenRouter → NVIDIA/Agnes/OpenCode/Reka - **部署方式**：Cloudflare Pages + Workers + D1/R2；Cloudflare Containers 承接过渡期 Python/RSS-Bridge 后台面；VPS/Tunnel 仅作 legacy rollback，不是运行依赖 - **可选组件**：`[api]` FastAPI + Web UI（管理后台 + 公开新闻阅读器） diff --git a/config/provider/routes.yaml b/config/provider/routes.yaml index e178640f..998d8b0c 100644 --- a/config/provider/routes.yaml +++ b/config/provider/routes.yaml @@ -119,7 +119,7 @@ routes: output_schema_ref: null audit: true fallback_route_ids: - - fallback.local + - translate.nvidia notes: "翻译兜底链：Cloudflare Workers AI 失败后由 OpenRouter 免费模型接管" - route_id: translate.batch @@ -136,15 +136,51 @@ routes: - route_id: translate.nvidia task_type: translate - provider: gemini - model: gemini-2.5-flash + provider: nvidia + model: deepseek-ai/deepseek-v4-flash timeout_seconds: 90 max_cost_usd_per_call: 0.0 output_schema_ref: null audit: true fallback_route_ids: - - translate.deepseek - notes: "历史 route_id 兼容：当前指向 Gemini" + - translate.agnes + notes: "FreeLLMAPI 迁移后直连 NVIDIA NIM，不再经过本地 sidecar" + + - route_id: translate.agnes + task_type: translate + provider: agnes + model: agnes-2.0-flash + timeout_seconds: 90 + max_cost_usd_per_call: 0.0 + output_schema_ref: null + audit: true + fallback_route_ids: + - translate.opencode + notes: "FreeLLMAPI 迁移后直连 Agnes AI" + + - route_id: translate.opencode + task_type: translate + provider: opencode + model: deepseek-v4-flash-free + timeout_seconds: 90 + max_cost_usd_per_call: 0.0 + output_schema_ref: null + audit: true + fallback_route_ids: + - translate.reka + notes: "FreeLLMAPI 迁移后直连 OpenCode Zen" + + - route_id: translate.reka + task_type: translate + provider: reka + model: reka-flash-3 + timeout_seconds: 90 + max_cost_usd_per_call: 0.0 + output_schema_ref: null + audit: true + fallback_route_ids: + - fallback.local + notes: "FreeLLMAPI 迁移后直连 Reka" # ── 聚类 / 复核 / 增强 ──────────────────────────────── @@ -217,20 +253,56 @@ routes: output_schema_ref: null audit: true fallback_route_ids: - - fallback.local + - ai.enrichment.nvidia notes: "低频 AI 增强兜底：Gemini 不可用时由 OpenRouter 免费模型接管" - route_id: ai.enrichment.nvidia task_type: ai_enrichment - provider: deepseek - model: deepseek-chat + provider: nvidia + model: deepseek-ai/deepseek-v4-flash + timeout_seconds: 120 + max_cost_usd_per_call: 0.0 + output_schema_ref: null + audit: true + fallback_route_ids: + - ai.enrichment.agnes + notes: "低频 AI 增强兜底：NVIDIA NIM 直连" + + - route_id: ai.enrichment.agnes + task_type: ai_enrichment + provider: agnes + model: agnes-2.0-flash + timeout_seconds: 120 + max_cost_usd_per_call: 0.0 + output_schema_ref: null + audit: true + fallback_route_ids: + - ai.enrichment.opencode + notes: "低频 AI 增强兜底：Agnes AI 直连" + + - route_id: ai.enrichment.opencode + task_type: ai_enrichment + provider: opencode + model: deepseek-v4-flash-free + timeout_seconds: 120 + max_cost_usd_per_call: 0.0 + output_schema_ref: null + audit: true + fallback_route_ids: + - ai.enrichment.reka + notes: "低频 AI 增强兜底：OpenCode Zen 直连" + + - route_id: ai.enrichment.reka + task_type: ai_enrichment + provider: reka + model: reka-flash-3 timeout_seconds: 120 max_cost_usd_per_call: 0.0 output_schema_ref: null audit: true fallback_route_ids: - fallback.local - notes: "AI 增强兜底：DeepSeek 接管，失败回退本地规则" + notes: "低频 AI 增强兜底：Reka 直连" # ── 判断任务 ────────────────────────────────────────── diff --git a/docs/architecture.md b/docs/architecture.md index 770efbb5..f8bdf3f8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -30,7 +30,7 @@ News Sentry 是一个**开源 AI 新闻情报与 OSINT 监控平台**，持续 | 类型检查 | mypy strict, 零错误 | | Web 框架 | FastAPI + Uvicorn | | 存储 | Cloudflare D1 + R2；SQLite/文件系统用于本地开发与容器过渡 | -| AI Provider | Gemini → DeepSeek → Groq → Cloudflare (链式降级) | +| AI Provider | Gemini → DeepSeek → Groq → Cloudflare Workers AI → OpenRouter → NVIDIA/Agnes/OpenCode/Reka (链式降级) | | 部署 | Cloudflare Pages + Workers + D1/R2；Cloudflare Containers 作为 Python/RSS-Bridge 过渡运行面 | | 前端 | Vite + React + Tailwind CSS | | 测试 | pytest 3,013 tests / vitest | @@ -43,7 +43,7 @@ News Sentry 是一个**开源 AI 新闻情报与 OSINT 监控平台**，持续 NewsSentry/ ├── src/news_sentry/ # 核心 Python 源码 (~24,000 lines, 102 files) │ ├── adapters/ # 外部适配器 -│ │ ├── providers/ # AI Provider (12 个 Provider) +│ │ ├── providers/ # AI Provider adapters │ │ │ ├── base.py # AIProvider Protocol │ │ │ ├── openai_provider.py # OpenAI-compatible 基类 │ │ │ ├── gemini_provider.py # Gemini (主) @@ -52,6 +52,7 @@ NewsSentry/ │ │ │ ├── cloudflare_workers_ai_provider.py # CF (兜底翻译) │ │ │ ├── anthropic_provider.py # Anthropic (可选) │ │ │ ├── openrouter_provider.py # OpenRouter (可选) +│ │ │ ├── freeapi_compat_providers.py # NVIDIA/OpenCode/Reka/Agnes 直连备用 │ │ │ ├── rules_provider.py # 本地规则引擎 │ │ │ ├── libretranslate_provider.py │ │ │ └── mymemory_provider.py @@ -140,6 +141,8 @@ flowchart TB subgraph PROVIDERS["AI Provider Chain"] direction LR G["Gemini"] --> DS["DeepSeek"] --> GR["Groq"] --> CF["Cloudflare"] + CF --> OR["OpenRouter"] --> NV["NVIDIA"] --> AG["Agnes"] + AG --> OC["OpenCode"] --> RK["Reka"] end subgraph STORAGE["文件存储"] @@ -207,7 +210,7 @@ flowchart LR ### Provider Chain ``` -Gemini (首选) → DeepSeek (备选 1) → Groq (备选 2) → Cloudflare Workers AI (兜底翻译) +Gemini (首选) → DeepSeek (备选 1) → Groq (备选 2) → Cloudflare Workers AI → OpenRouter → NVIDIA → Agnes → OpenCode → Reka → local ↓ 失败 ↓ 失败 ↓ 失败 自动降级到下一个节点，无需人工干预 ``` @@ -220,7 +223,12 @@ AIProvider (Protocol) │ ├── GeminiProvider (api_key: GEMINI_API_KEY) │ ├── DeepSeekProvider (api_key: DEEPSEEK_API_KEY) │ ├── GroqProvider (api_key: GROQ_API_KEY) -│ └── OpenRouterProvider +│ ├── OpenRouterProvider +│ └── FreeAPI migrated providers +│ ├── NvidiaProvider (api_key: NVIDIA_API_KEY[_2]) +│ ├── AgnesProvider (api_key: AGNES_API_KEY[_2]) +│ ├── OpenCodeProvider (api_key: OPENCODE_API_KEY[_2]) +│ └── RekaProvider (api_key: REKA_API_KEY) ├── AnthropicProvider — Anthropic Messages API ├── CloudflareWorkersAIProvider — 翻译专用 ├── LibreTranslateProvider / MyMemoryProvider — 翻译备选 @@ -253,6 +261,11 @@ routes: | `DEEPSEEK_API_KEY` | DeepSeek | platform.deepseek.com | | `GROQ_API_KEY` | Groq | console.groq.com | | `CLOUDFLARE_ACCOUNT_ID` + `CLOUDFLARE_API_TOKEN` | Cloudflare | dash.cloudflare.com | +| `OPENROUTER_API_KEY[_2]` | OpenRouter | openrouter.ai | +| `NVIDIA_API_KEY[_2]` | NVIDIA NIM | build.nvidia.com | +| `AGNES_API_KEY[_2]` | Agnes AI | platform.agnes-ai.com | +| `OPENCODE_API_KEY[_2]` | OpenCode Zen | opencode.ai | +| `REKA_API_KEY` | Reka | platform.reka.ai | --- @@ -288,7 +301,7 @@ flowchart TB QUEUE --> FLOW["Workflows / Durable Objects"] FLOW --> WORKER WORKER -. "Access-gated fallback only" .-> C["Cloudflare Containers
Python / FastAPI / RSS-Bridge transition"] - WORKER -.-> API["AI APIs
(Gemini/DeepSeek/Groq/Workers AI)"] + WORKER -.-> API["AI APIs
(Gemini/DeepSeek/Groq/Workers AI/OpenRouter/NVIDIA/Agnes/OpenCode/Reka)"] ``` ### 运行面 diff --git a/docs/external-integration-strategy.md b/docs/external-integration-strategy.md index 1b370353..03e86ff7 100644 --- a/docs/external-integration-strategy.md +++ b/docs/external-integration-strategy.md @@ -75,10 +75,12 @@ News Sentry v2 使用内置 AI provider chain，无需外部 Agent 框架： | Gemini | `google-genai` | 翻译、研判 | 1 (primary) | | DeepSeek | OpenAI-compatible SDK | 翻译 fallback | 2 | | Groq | `groq` | 研判 fallback | 3 | -| Cloudflare Workers AI | REST API | 最后兜底 | 4 | +| Cloudflare Workers AI | REST API | Cloudflare-native 兜底 | 4 | +| OpenRouter | OpenAI-compatible SDK | 免费模型兜底 | 5 | +| NVIDIA NIM / Agnes AI / OpenCode Zen / Reka | OpenAI-compatible SDK | FreeLLMAPI 密钥迁移后的直连备用，不再经过本地 sidecar | 6 | - **零外部框架依赖：** 不依赖 LangChain、OpenCLI、Hermes、OpenClaw 等 Agent 框架 -- **Provider 路由：** `config/provider/routes.yaml` 控制每个 target 的 provider 链路 +- **Provider 路由：** `config/provider/routes.yaml` 控制每个 target 的 provider 链路；同一 provider 的备用 key 使用环境变量后缀 `_2`、`_3` 进入 key pool - **CLI 入口不变：** `python -m news_sentry.cli run --target {id} --stage all` --- diff --git a/frontend/cloudflare/workers/index.ts b/frontend/cloudflare/workers/index.ts index 2bd70bb7..ce3217d6 100644 --- a/frontend/cloudflare/workers/index.ts +++ b/frontend/cloudflare/workers/index.ts @@ -31,21 +31,59 @@ import { handleWorkerWriteAccess } from "./lib/access"; interface Env { DB: D1Database; NEWS_SENTRY_CONTAINER?: DurableObjectNamespace; + GEMINI_API_KEY?: string; + OPENROUTER_API_KEY?: string; + OPENROUTER_API_KEY_2?: string; + NVIDIA_API_KEY?: string; + NVIDIA_API_KEY_2?: string; + OPENCODE_API_KEY?: string; + OPENCODE_API_KEY_2?: string; + REKA_API_KEY?: string; + AGNES_API_KEY?: string; + AGNES_API_KEY_2?: string; + DEEPSEEK_API_KEY?: string; + GROQ_API_KEY?: string; + CLOUDFLARE_ACCOUNT_ID?: string; + CLOUDFLARE_API_TOKEN?: string; } export { ContainerProxy } from "@cloudflare/containers"; -export class NewsSentryContainer extends Container { +function definedEnv(vars: Record): Record { + return Object.fromEntries( + Object.entries(vars).filter((entry): entry is [string, string] => Boolean(entry[1])), + ); +} + +export class NewsSentryContainer extends Container { defaultPort = 8000; sleepAfter = "30m"; enableInternet = true; - envVars = { - NEWSSENTRY_DEPLOYMENT_ENV: "cloudflare-container", - NEWSSENTRY_PROFILE: "cloudflare", - NEWSSENTRY_AUTO_COLLECT: "0", - NEWSSENTRY_PUBLIC_TRANSLATION: "0", - NEWSSENTRY_LOG_LEVEL: "INFO", - }; + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + this.envVars = definedEnv({ + NEWSSENTRY_DEPLOYMENT_ENV: "cloudflare-container", + NEWSSENTRY_PROFILE: "cloudflare", + NEWSSENTRY_AUTO_COLLECT: "0", + NEWSSENTRY_PUBLIC_TRANSLATION: "1", + NEWSSENTRY_LOG_LEVEL: "INFO", + GEMINI_API_KEY: env.GEMINI_API_KEY, + OPENROUTER_API_KEY: env.OPENROUTER_API_KEY, + OPENROUTER_API_KEY_2: env.OPENROUTER_API_KEY_2, + NVIDIA_API_KEY: env.NVIDIA_API_KEY, + NVIDIA_API_KEY_2: env.NVIDIA_API_KEY_2, + OPENCODE_API_KEY: env.OPENCODE_API_KEY, + OPENCODE_API_KEY_2: env.OPENCODE_API_KEY_2, + REKA_API_KEY: env.REKA_API_KEY, + AGNES_API_KEY: env.AGNES_API_KEY, + AGNES_API_KEY_2: env.AGNES_API_KEY_2, + DEEPSEEK_API_KEY: env.DEEPSEEK_API_KEY, + GROQ_API_KEY: env.GROQ_API_KEY, + CLOUDFLARE_ACCOUNT_ID: env.CLOUDFLARE_ACCOUNT_ID, + CLOUDFLARE_API_TOKEN: env.CLOUDFLARE_API_TOKEN, + }); + } } const SECURITY_HEADERS: Record = { diff --git a/frontend/cloudflare/wrangler.toml b/frontend/cloudflare/wrangler.toml index 9c917305..06a14d24 100644 --- a/frontend/cloudflare/wrangler.toml +++ b/frontend/cloudflare/wrangler.toml @@ -17,6 +17,16 @@ routes = [ [vars] NEWS_SENTRY_RUNTIME = "cloudflare-native" +[secrets] +required = [ + "GEMINI_API_KEY", + "OPENROUTER_API_KEY", + "NVIDIA_API_KEY", + "OPENCODE_API_KEY", + "REKA_API_KEY", + "AGNES_API_KEY", +] + # D1 数据库绑定 [[d1_databases]] binding = "DB" @@ -31,6 +41,16 @@ routes = [] [env.production.vars] NEWS_SENTRY_RUNTIME = "cloudflare-native" +[env.production.secrets] +required = [ + "GEMINI_API_KEY", + "OPENROUTER_API_KEY", + "NVIDIA_API_KEY", + "OPENCODE_API_KEY", + "REKA_API_KEY", + "AGNES_API_KEY", +] + [[env.production.d1_databases]] binding = "DB" database_name = "ns-db" diff --git a/src/news_sentry/adapters/providers/freeapi_compat_providers.py b/src/news_sentry/adapters/providers/freeapi_compat_providers.py new file mode 100644 index 00000000..cdff2d29 --- /dev/null +++ b/src/news_sentry/adapters/providers/freeapi_compat_providers.py @@ -0,0 +1,75 @@ +"""OpenAI-compatible providers recovered from the legacy FreeLLMAPI key store.""" + +from __future__ import annotations + +import os +from typing import Any + +from news_sentry.adapters.providers.openai_provider import OpenAIProvider + + +class NvidiaProvider(OpenAIProvider): + """NVIDIA NIM OpenAI-compatible provider.""" + + provider_id = "nvidia" + api_key_env_var = "NVIDIA_API_KEY" + + def __init__(self, config: dict[str, Any]) -> None: + merged: dict[str, Any] = { + "api_key_env_var": self.api_key_env_var, + "base_url": os.environ.get( + "NVIDIA_BASE_URL", + "https://integrate.api.nvidia.com/v1", + ), + "default_model": os.environ.get("NVIDIA_MODEL", "deepseek-ai/deepseek-v4-flash"), + **config, + } + super().__init__(merged) + + +class OpenCodeProvider(OpenAIProvider): + """OpenCode Zen OpenAI-compatible provider.""" + + provider_id = "opencode" + api_key_env_var = "OPENCODE_API_KEY" + + def __init__(self, config: dict[str, Any]) -> None: + merged: dict[str, Any] = { + "api_key_env_var": self.api_key_env_var, + "base_url": os.environ.get("OPENCODE_BASE_URL", "https://opencode.ai/zen/v1"), + "default_model": os.environ.get("OPENCODE_MODEL", "deepseek-v4-flash-free"), + **config, + } + super().__init__(merged) + + +class RekaProvider(OpenAIProvider): + """Reka OpenAI-compatible provider.""" + + provider_id = "reka" + api_key_env_var = "REKA_API_KEY" + + def __init__(self, config: dict[str, Any]) -> None: + merged: dict[str, Any] = { + "api_key_env_var": self.api_key_env_var, + "base_url": os.environ.get("REKA_BASE_URL", "https://api.reka.ai/v1"), + "default_model": os.environ.get("REKA_MODEL", "reka-flash-3"), + **config, + } + super().__init__(merged) + + +class AgnesProvider(OpenAIProvider): + """Agnes AI OpenAI-compatible provider.""" + + provider_id = "agnes" + api_key_env_var = "AGNES_API_KEY" + + def __init__(self, config: dict[str, Any]) -> None: + merged: dict[str, Any] = { + "api_key_env_var": self.api_key_env_var, + "base_url": os.environ.get("AGNES_BASE_URL", "https://apihub.agnes-ai.com/v1"), + "default_model": os.environ.get("AGNES_MODEL", "agnes-2.0-flash"), + **config, + } + super().__init__(merged) diff --git a/src/news_sentry/adapters/providers/openai_provider.py b/src/news_sentry/adapters/providers/openai_provider.py index aa2908a9..fb4eb6c2 100644 --- a/src/news_sentry/adapters/providers/openai_provider.py +++ b/src/news_sentry/adapters/providers/openai_provider.py @@ -38,10 +38,12 @@ def __init__(self, config: dict[str, Any]) -> None: - max_tokens: 最大输出 token 数，默认 2048 """ self._api_key_env_var = config.get("api_key_env_var", self.api_key_env_var) - self._api_key = config.get( - "api_key", - os.environ.get(self._api_key_env_var), - ) + configured_key = config.get("api_key") + if configured_key: + self._api_keys = [configured_key] + else: + self._api_keys = self._load_api_key_pool(self._api_key_env_var) + self._api_key = self._api_keys[0] if self._api_keys else None self._base_url = config.get( "base_url", os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1"), @@ -49,6 +51,19 @@ def __init__(self, config: dict[str, Any]) -> None: self._default_model = config.get("default_model", "gpt-4o-mini") self._max_tokens = config.get("max_tokens", 2048) + @staticmethod + def _load_api_key_pool(env_var: str) -> list[str]: + keys: list[str] = [] + for name in [env_var, *(f"{env_var}_{index}" for index in range(2, 11))]: + value = os.environ.get(name) + if value and value not in keys: + keys.append(value) + return keys + + @staticmethod + def _retryable_key_status(status_code: int) -> bool: + return status_code in {401, 402, 403, 429} + def call(self, route_id: str, prompt: str, **kwargs: Any) -> dict[str, Any]: # noqa: ANN401 """发送 chat completion 请求到 OpenAI 兼容 API。 @@ -64,7 +79,7 @@ def call(self, route_id: str, prompt: str, **kwargs: Any) -> dict[str, Any]: # Raises: RuntimeError: API 调用失败或网络错误。 """ - if not self._api_key: + if not self._api_keys: raise RuntimeError( f"{self._api_key_env_var} 未设置，无法调用 {self.provider_id} API。" " 请在环境变量或 config 中提供 api_key。" @@ -74,31 +89,46 @@ def call(self, route_id: str, prompt: str, **kwargs: Any) -> dict[str, Any]: # max_tokens = kwargs.get("max_tokens", self._max_tokens) url = f"{self._base_url.rstrip('/')}/chat/completions" - headers = { - "Authorization": f"Bearer {self._api_key}", - "Content-Type": "application/json", - } payload: dict[str, Any] = { "model": model, "messages": [{"role": "user", "content": prompt}], "max_tokens": max_tokens, } - try: - response = httpx.post( - url, - headers=headers, - json=payload, - timeout=30, - ) - response.raise_for_status() - data = response.json() - except httpx.HTTPStatusError as e: - raise RuntimeError( - f"OpenAI API 返回 HTTP {e.response.status_code}: {e.response.text}" - ) from e - except httpx.RequestError as e: - raise RuntimeError(f"OpenAI API 网络请求失败: {e}") from e + last_status_error: httpx.HTTPStatusError | None = None + for index, api_key in enumerate(self._api_keys): + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + try: + response = httpx.post( + url, + headers=headers, + json=payload, + timeout=30, + ) + response.raise_for_status() + data = response.json() + break + except httpx.HTTPStatusError as e: + last_status_error = e + if ( + index + 1 < len(self._api_keys) + and self._retryable_key_status(e.response.status_code) + ): + continue + raise RuntimeError( + f"OpenAI API 返回 HTTP {e.response.status_code}: {e.response.text}" + ) from e + except httpx.RequestError as e: + raise RuntimeError(f"OpenAI API 网络请求失败: {e}") from e + else: + if last_status_error is not None: + raise RuntimeError( + "OpenAI API 调用失败: 所有 API key 都不可用" + ) from last_status_error + raise RuntimeError("OpenAI API 调用失败: 没有可用 API key") choice = data["choices"][0] return { @@ -132,7 +162,7 @@ async def call_async( Returns: dict with keys: content, model, usage, route_id, provider。 """ - if not self._api_key: + if not self._api_keys: raise RuntimeError( f"{self._api_key_env_var} 未设置，无法调用 {self.provider_id} API。" " 请在环境变量或 config 中提供 api_key。" @@ -147,19 +177,38 @@ async def call_async( if "response_format" in kwargs: payload["response_format"] = kwargs["response_format"] - headers = { - "Authorization": f"Bearer {self._api_key}", - "Content-Type": "application/json", - } - client = http_client or httpx.AsyncClient(timeout=30.0) try: - response = await client.post( - f"{self._base_url}/chat/completions", - json=payload, - headers=headers, - ) - response.raise_for_status() + last_status_error: httpx.HTTPStatusError | None = None + for index, api_key in enumerate(self._api_keys): + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + try: + response = await client.post( + f"{self._base_url}/chat/completions", + json=payload, + headers=headers, + ) + response.raise_for_status() + break + except httpx.HTTPStatusError as e: + last_status_error = e + if ( + index + 1 < len(self._api_keys) + and self._retryable_key_status(e.response.status_code) + ): + continue + raise RuntimeError( + f"OpenAI API 返回 HTTP {e.response.status_code}: {e.response.text}" + ) from e + else: + if last_status_error is not None: + raise RuntimeError( + "OpenAI API 调用失败: 所有 API key 都不可用" + ) from last_status_error + raise RuntimeError("OpenAI API 调用失败: 没有可用 API key") finally: if http_client is None: await client.aclose() @@ -182,4 +231,4 @@ def health_check(self) -> bool: Returns: True 如果 API key 已配置，否则 False。永不抛出异常。 """ - return bool(self._api_key) + return bool(self._api_keys) diff --git a/src/news_sentry/core/run.py b/src/news_sentry/core/run.py index 7dfec622..4dd7a78f 100644 --- a/src/news_sentry/core/run.py +++ b/src/news_sentry/core/run.py @@ -683,8 +683,9 @@ def _init_ai_judge( def _build_provider_factory() -> Callable[[str], AIProvider | None]: """构建 provider_name → AIProvider 实例的工厂函数。 - 支持的 provider_name: cloudflare_workers_ai, deepseek, gemini, groq, - libretranslate, mymemory, openrouter, openai, anthropic, local。 + 支持的 provider_name: agnes, cloudflare_workers_ai, deepseek, gemini, groq, + libretranslate, mymemory, nvidia, opencode, openrouter, openai, reka, + anthropic, local。 通过环境变量配置 API key 和 base URL。 """ # 惰性初始化，避免在 import 时读取环境变量 @@ -719,6 +720,21 @@ def factory(name: str) -> AIProvider | None: provider = OpenAIProvider({}) elif name == "openrouter": provider = OpenRouterProvider({}) + elif name in {"agnes", "nvidia", "opencode", "reka"}: + from news_sentry.adapters.providers.freeapi_compat_providers import ( + AgnesProvider, + NvidiaProvider, + OpenCodeProvider, + RekaProvider, + ) + + provider_map = { + "agnes": AgnesProvider, + "nvidia": NvidiaProvider, + "opencode": OpenCodeProvider, + "reka": RekaProvider, + } + provider = provider_map[name]({}) elif name == "anthropic": provider = AnthropicProvider({}) elif name == "local": diff --git a/tests/unit/test_openai_provider.py b/tests/unit/test_openai_provider.py index 0939fd28..076e6b7b 100644 --- a/tests/unit/test_openai_provider.py +++ b/tests/unit/test_openai_provider.py @@ -48,6 +48,13 @@ def test_init_with_env_fallback(self, monkeypatch): provider = OpenAIProvider({}) assert provider._api_key == "sk-env-456" + def test_init_loads_env_key_pool(self, monkeypatch): + """同一 provider 的 _2/_3 备用 key 应进入 key pool。""" + monkeypatch.setenv("OPENAI_API_KEY", "sk-env-primary") + monkeypatch.setenv("OPENAI_API_KEY_2", "sk-env-secondary") + provider = OpenAIProvider({}) + assert provider._api_keys == ["sk-env-primary", "sk-env-secondary"] + def test_init_base_url_default(self): """未设置 base_url 时使用默认 OpenAI 端点。""" provider = OpenAIProvider({"api_key": "sk-test"}) @@ -183,6 +190,33 @@ def test_call_passes_max_tokens_from_kwargs(self): assert mock_post.call_args.kwargs["json"]["max_tokens"] == 512 + def test_call_retries_next_key_on_quota_error(self, monkeypatch): + """账号/额度类错误应尝试同 provider 的下一把 key。""" + monkeypatch.setenv("OPENAI_API_KEY", "sk-primary") + monkeypatch.setenv("OPENAI_API_KEY_2", "sk-secondary") + rate_limited = _make_mock_response(status_code=429, json_data={"error": "quota"}) + rate_limited.raise_for_status.side_effect = httpx.HTTPStatusError( + "Rate limited", request=mock.MagicMock(), response=rate_limited + ) + ok = _make_mock_response( + json_data={ + "choices": [{"message": {"content": "ok"}}], + "model": "gpt-4o-mini", + "usage": {}, + } + ) + + with mock.patch("httpx.post", side_effect=[rate_limited, ok]) as mock_post: + provider = OpenAIProvider({}) + result = provider.call("translate.fast", "test") + + assert result["content"] == "ok" + assert mock_post.call_count == 2 + first_headers = mock_post.call_args_list[0].kwargs["headers"] + second_headers = mock_post.call_args_list[1].kwargs["headers"] + assert first_headers["Authorization"] == "Bearer sk-primary" + assert second_headers["Authorization"] == "Bearer sk-secondary" + # ── provider_id ──────────────────────────────────────────────────────── diff --git a/tests/unit/test_provider_factory.py b/tests/unit/test_provider_factory.py index 3e275429..8e1f16ab 100644 --- a/tests/unit/test_provider_factory.py +++ b/tests/unit/test_provider_factory.py @@ -15,3 +15,19 @@ def test_provider_factory_builds_openrouter_provider(monkeypatch): assert provider is not None assert provider.provider_id == "openrouter" assert provider.health_check() is True + + +def test_provider_factory_builds_migrated_freeapi_providers(monkeypatch): + """FreeLLMAPI 密钥迁移后的直连 provider 名应可由工厂构建。""" + monkeypatch.setenv("NVIDIA_API_KEY", "nvapi-test") + monkeypatch.setenv("OPENCODE_API_KEY", "opencode-test") + monkeypatch.setenv("REKA_API_KEY", "reka-test") + monkeypatch.setenv("AGNES_API_KEY", "agnes-test") + + factory = _build_provider_factory() + + for provider_name in ("nvidia", "opencode", "reka", "agnes"): + provider = factory(provider_name) + assert provider is not None + assert provider.provider_id == provider_name + assert provider.health_check() is True diff --git a/tests/unit/test_provider_router.py b/tests/unit/test_provider_router.py index 0d5ef114..340d119d 100644 --- a/tests/unit/test_provider_router.py +++ b/tests/unit/test_provider_router.py @@ -173,12 +173,24 @@ def test_public_translation_route_uses_builtin_chain() -> None: assert public_translation.provider == "gemini" assert routes["translate.cloudflare"].fallback_route_ids == ["translate.openrouter"] assert routes["translate.openrouter"].provider == "openrouter" + assert routes["translate.openrouter"].fallback_route_ids == ["translate.nvidia"] + assert routes["translate.nvidia"].provider == "nvidia" + assert routes["translate.nvidia"].fallback_route_ids == ["translate.agnes"] + assert routes["translate.agnes"].provider == "agnes" + assert routes["translate.agnes"].fallback_route_ids == ["translate.opencode"] + assert routes["translate.opencode"].provider == "opencode" + assert routes["translate.opencode"].fallback_route_ids == ["translate.reka"] + assert routes["translate.reka"].provider == "reka" + assert routes["translate.reka"].fallback_route_ids == ["fallback.local"] # 公共发布的路由链不应有空 assert routes["public.summary_reason"].provider != "freellmapi" assert routes["public.enrichment"].provider != "freellmapi" assert routes["ai.enrichment.batch"].fallback_route_ids == ["ai.enrichment.openrouter"] assert routes["ai.enrichment.openrouter"].provider == "openrouter" + assert routes["ai.enrichment.openrouter"].fallback_route_ids == ["ai.enrichment.nvidia"] + assert routes["ai.enrichment.nvidia"].provider == "nvidia" + assert routes["ai.enrichment.nvidia"].fallback_route_ids == ["ai.enrichment.agnes"] def test_provider_route_defaults() -> None: From 3c9d5ebcfd5caa62fbb71074295f140083e1b778 Mon Sep 17 00:00:00 2001 From: xucroyuri Date: Sun, 28 Jun 2026 19:42:46 +0800 Subject: [PATCH 03/16] fix: backfill D1 public translation fields --- src/news_sentry/core/public_translation.py | 142 +++++++++++- tests/unit/test_cloudflare_d1_backfill.py | 103 +++++++++ tests/unit/test_public_translation.py | 55 +++++ ...oudflare_d1_public_translation_backfill.py | 217 ++++++++++++++++++ 4 files changed, 515 insertions(+), 2 deletions(-) create mode 100644 tools/cloudflare_d1_public_translation_backfill.py diff --git a/src/news_sentry/core/public_translation.py b/src/news_sentry/core/public_translation.py index aca7634a..7c6c71e3 100644 --- a/src/news_sentry/core/public_translation.py +++ b/src/news_sentry/core/public_translation.py @@ -133,6 +133,52 @@ "物流": "交通物流", "交通": "交通物流", } + +_FALLBACK_ISSUE_TAGS: dict[str, str] = { + "business": "经济", + "culture": "文化", + "economy": "经济", + "education": "教育", + "energy": "能源", + "environment": "气候环境", + "finance": "金融市场", + "health": "公共卫生", + "military": "军事防务", + "politics": "政治", + "security": "公共安全", + "society": "社会", + "sports": "体育", + "tech": "科技", + "technology": "科技", + "trade": "国际贸易", +} +_FALLBACK_REGION_TAGS: dict[str, str] = { + "canada": "加拿大", + "china-watch-en": "中国", + "france": "法国", + "germany": "德国", + "india": "印度", + "ireland": "爱尔兰", + "italy": "意大利", + "japan": "日本", + "new-zealand": "新西兰", + "south-korea": "韩国", + "united-kingdom": "英国", + "vietnam": "越南", +} +_FALLBACK_RELATED_TAGS: dict[str, str] = { + "canada": "北美", + "france": "涉欧", + "germany": "涉欧", + "india": "南亚", + "ireland": "涉欧", + "italy": "涉欧", + "japan": "东亚", + "new-zealand": "亚太", + "south-korea": "东亚", + "united-kingdom": "涉欧", + "vietnam": "东南亚", +} _RELATED_TAG_ALIASES: dict[str, str | tuple[str, ...]] = { "欧美": ("涉美", "涉欧"), "亚太地区": "亚太", @@ -407,6 +453,54 @@ def _looks_like_template_reason(reason: str) -> bool: return any(marker in text for marker in template_markers) +def _clip_public_sentence(text: str, *, limit: int) -> str: + normalized = " ".join(str(text or "").split()) + if len(normalized) <= limit: + return normalized + return normalized[:limit].rstrip("，。；、 ") + "。" + + +def _fallback_issue_tags_for_row(row: dict[str, Any]) -> list[str]: + metadata = _metadata_dict(row) + raw = str(metadata.get("classification") or row.get("classification_l0") or "").lower() + for key, label in _FALLBACK_ISSUE_TAGS.items(): + if key in raw: + return [label] + text = " ".join( + str(value or "").lower() + for value in ( + row.get("title_original"), + metadata.get("summary"), + row.get("summary"), + row.get("description"), + ) + ) + keyword_tags = ( + (("defense", "military", "army", "navy", "air force", "missile"), "军事防务"), + (("tariff", "trade", "export", "import"), "国际贸易"), + (("bank", "market", "stock", "bond", "finance"), "金融市场"), + (("ai", "chip", "semiconductor", "technology", "tech"), "科技"), + (("election", "parliament", "president", "minister"), "政治"), + (("heat", "flood", "quake", "fire", "wildfire"), "灾害事故"), + ) + for keywords, label in keyword_tags: + if any(keyword in text for keyword in keywords): + return [label] + return ["国际关系"] + + +def _fallback_related_tags_for_row(row: dict[str, Any]) -> list[str]: + target_id = str(row.get("target_id") or "") + tag = _FALLBACK_RELATED_TAGS.get(target_id) + return [tag] if tag else [] + + +def _fallback_region_tags_for_row(row: dict[str, Any]) -> list[str]: + target_id = str(row.get("target_id") or "") + tag = _FALLBACK_REGION_TAGS.get(target_id) + return [tag] if tag else [] + + def _parse_iso_datetime(value: Any) -> datetime | None: # noqa: ANN401 text = str(value or "").strip() if not text: @@ -723,9 +817,24 @@ async def _generate_publication_fields( exc.model = result.get("model") # type: ignore[attr-defined] raise exc raw_content = str(result.get("content") or "").strip() - parsed = self._parse_publication_json(raw_content) + try: + parsed = self._parse_publication_json(raw_content) + except RuntimeError as exc: + if "not JSON" not in str(exc): + raise + parsed = self._publication_fallback_from_text( + row, + raw_content, + title_zh=title_zh, + summary_zh=summary_zh, + ) if not isinstance(parsed, dict): - raise RuntimeError("publication response is not an object") + parsed = self._publication_fallback_from_text( + row, + raw_content, + title_zh=title_zh, + summary_zh=summary_zh, + ) one_line = " ".join(str(parsed.get("one_line_summary") or "").split()) reason = " ".join(str(parsed.get("recommendation_reason") or "").split()) issue_tags = _publication_tags(parsed.get("issue_tags"), aliases=_ISSUE_TAG_ALIASES) @@ -753,6 +862,35 @@ async def _generate_publication_fields( "model": str(result.get("model") or ""), } + def _publication_fallback_from_text( + self, + row: dict[str, Any], + content: str, + *, + title_zh: str, + summary_zh: str, + ) -> dict[str, Any]: + """Recover publication fields when a provider ignores the JSON contract.""" + issue_tags = _fallback_issue_tags_for_row(row) + related_tags = _fallback_related_tags_for_row(row) + region_tags = _fallback_region_tags_for_row(row) + one_line = _clip_public_sentence(summary_zh or title_zh, limit=48) + reason = _clip_public_sentence(content, limit=90) + if not _public_text_quality_ok(reason) or _looks_like_template_reason(reason): + region = region_tags[0] if region_tags else "相关地区" + issue = issue_tags[0] if issue_tags else "公共议题" + reason = ( + f"{region}{issue}动态出现新变化，可能影响政策、产业或区域风险判断，" + "适合纳入后续跟踪。" + ) + return { + "one_line_summary": one_line, + "recommendation_reason": reason, + "issue_tags": issue_tags, + "related_tags": related_tags, + "region_tags": region_tags, + } + @staticmethod def _parse_publication_json(content: str) -> Any: # noqa: ANN401 text = content.strip() diff --git a/tests/unit/test_cloudflare_d1_backfill.py b/tests/unit/test_cloudflare_d1_backfill.py index dce1e375..6b5c6143 100644 --- a/tests/unit/test_cloudflare_d1_backfill.py +++ b/tests/unit/test_cloudflare_d1_backfill.py @@ -7,6 +7,10 @@ from pathlib import Path from tools.cloudflare_d1_backfill import collect_backfill_plan, generate_backfill_sql +from tools.cloudflare_d1_public_translation_backfill import ( + collect_translation_patches, + generate_translation_backfill_sql, +) def _make_state_db(path: Path) -> None: @@ -118,3 +122,102 @@ def test_generate_backfill_sql_is_idempotent_and_preserves_drafts_stage(tmp_path assert "reviewed-1" not in sql assert "INSERT INTO targets" in sql assert "INSERT INTO sources" in sql + + +def test_public_translation_backfill_updates_only_ready_public_fields(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + _make_state_db(data_dir / "italy" / "state.db") + ready_metadata = { + "translation": { + "title_pre": "意大利政府推进能源补贴调整", + "summary_pre": "意大利政府计划调整能源补贴，以应对财政压力和家庭用能成本。", + }, + "publication": { + "one_line_summary": "意大利调整能源补贴安排。", + "recommendation_reason": "这项政策会影响家庭能源账单和财政支出，值得持续跟踪。", + "issue_tags": ["能源"], + "related_tags": ["涉欧"], + "region_tags": ["意大利"], + }, + } + not_ready_metadata = { + "translation": { + "title_pre": "意大利银行业利润承压", + "summary_pre": "意大利银行业利润因利率环境变化承压。", + } + } + short_title_metadata = { + "translation": { + "title_pre": "外国人在韩", + "summary_pre": "5月外国在韩消费创新高", + }, + "publication": { + "one_line_summary": "5月外国在韩消费创新高。", + "recommendation_reason": "韩国消费数据出现新变化，适合纳入区域经济跟踪。", + "issue_tags": ["经济"], + "related_tags": ["东亚"], + "region_tags": ["韩国"], + }, + } + with closing(sqlite3.connect(data_dir / "italy" / "state.db")) as db: + db.execute( + "UPDATE event_index SET metadata_json = ?, public_translation_ready = 1 " + "WHERE event_id = 'draft-1'", + (json_dump(ready_metadata),), + ) + db.execute( + """ + INSERT INTO event_index ( + event_id, target_id, stage, source_id, title_original, + published_at, created_at, metadata_json, public_translation_ready + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + "draft-2", + "italy", + "drafts", + "ansa", + "Bank profits", + "2026-06-25T03:00:00+00:00", + "2026-06-25T03:01:00+00:00", + json_dump(not_ready_metadata), + 0, + ), + ) + db.execute( + """ + INSERT INTO event_index ( + event_id, target_id, stage, source_id, title_original, + published_at, created_at, metadata_json, public_translation_ready + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + "draft-short", + "italy", + "drafts", + "ansa", + "Foreign spending in Korea hits fresh high in May", + "2026-06-25T04:00:00+00:00", + "2026-06-25T04:01:00+00:00", + json_dump(short_title_metadata), + 1, + ), + ) + db.commit() + + patches = collect_translation_patches(data_dir=data_dir) + sql = generate_translation_backfill_sql(patches) + + assert [patch.event_id for patch in patches] == ["draft-1"] + assert "UPDATE events SET" in sql + assert "意大利政府推进能源补贴调整" in sql + assert "recommendation_reason" in sql + assert "draft-2" not in sql + assert "draft-short" not in sql + assert "INSERT INTO targets" not in sql + + +def json_dump(value: object) -> str: + import json + + return json.dumps(value, ensure_ascii=False) diff --git a/tests/unit/test_public_translation.py b/tests/unit/test_public_translation.py index 5b0b679e..6de6bbcb 100644 --- a/tests/unit/test_public_translation.py +++ b/tests/unit/test_public_translation.py @@ -549,6 +549,61 @@ async def test_public_translation_engine_repairs_fenced_publication_json(tmp_pat await store.close() +@pytest.mark.asyncio +async def test_public_translation_engine_recovers_plaintext_publication(tmp_path) -> None: + store = AsyncStore(tmp_path / "state.db") + await store.initialize() + try: + await store.index_event(_event("ne-plain-publication", score=88), "france", "drafts") + router = MagicMock() + router.route_async = AsyncMock( + side_effect=[ + { + "content": "法国获得欧盟贷款用于军备采购", + "route_id": "translate.public", + "model": "auto", + "provider": "gemini", + }, + { + "content": "这条新闻涉及欧盟资金流向和法国防务采购，对供应链观察有价值。", + "route_id": "translate.public", + "model": "auto", + "provider": "gemini", + }, + { + "content": "这条新闻揭示法国防务采购链条变化，值得跟踪后续预算和供应商影响。", + "route_id": "public.summary_reason", + "model": "auto", + "provider": "agnes", + }, + ] + ) + engine = PublicTranslationEngine(PublicTranslationConfig(per_cycle_limit=5)) + rows = await store.list_public_translation_candidates("france", limit=10) + + result = await engine.run_rows( + target_id="france", + rows=rows, + store=store, + router=router, + provider_factory=lambda name: MagicMock(), + ) + + assert result["status"] == "ok" + row = await store.get_event_index_row("france", "ne-plain-publication") + assert row is not None and row["public_translation_ready"] == 1 + publication = row["metadata"]["publication"] + assert publication["one_line_summary"] == ( + "这条新闻涉及欧盟资金流向和法国防务采购，对供应链观察有价值。" + ) + assert "法国防务采购链条" in publication["recommendation_reason"] + assert publication["issue_tags"] == ["国际关系"] + assert publication["related_tags"] == ["涉欧"] + assert publication["region_tags"] == ["法国"] + finally: + await store.close() + + @pytest.mark.asyncio async def test_public_translation_engine_records_retrying_attempt_without_hiding_forever( tmp_path, diff --git a/tools/cloudflare_d1_public_translation_backfill.py b/tools/cloudflare_d1_public_translation_backfill.py new file mode 100644 index 00000000..6a4ace60 --- /dev/null +++ b/tools/cloudflare_d1_public_translation_backfill.py @@ -0,0 +1,217 @@ +"""Generate D1 SQL to sync ready public translation fields from local state DBs.""" + +from __future__ import annotations + +import argparse +import json +import re +import sqlite3 +from contextlib import closing +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from news_sentry.core.public_translation import public_publication_ready + +_CJK_RE = re.compile(r"[\u4e00-\u9fff]") + + +@dataclass(frozen=True) +class PublicTranslationPatch: + event_id: str + target_id: str + title: str + summary: str + recommendation_reason: str + issue_tags: list[str] + related_tags: list[str] + region_tags: list[str] + published_at: str + + +def _sql(value: Any) -> str: + if value is None: + return "NULL" + if isinstance(value, bool): + return "1" if value else "0" + if isinstance(value, int | float): + return str(value) + return "'" + str(value).replace("'", "''") + "'" + + +def _json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def _json_loads(value: Any) -> dict[str, Any]: + if isinstance(value, dict): + return value + if not value: + return {} + try: + loaded = json.loads(str(value)) + except json.JSONDecodeError: + return {} + return loaded if isinstance(loaded, dict) else {} + + +def _clean_text(value: Any) -> str: + return " ".join(str(value or "").split()) + + +def _clean_list(value: Any) -> list[str]: + if not isinstance(value, list): + return [] + items: list[str] = [] + for item in value: + text = _clean_text(item) + if text and text not in items: + items.append(text) + return items[:8] + + +def _translated_title_quality_ok(title: str, original_title: str) -> bool: + cjk_count = len(_CJK_RE.findall(title)) + if cjk_count < 4: + return False + original_visible = len(str(original_title or "").strip()) + if original_visible >= 24 and cjk_count < 8: + return False + return True + + +def _patch_from_row(row: sqlite3.Row) -> PublicTranslationPatch | None: + metadata = _json_loads(row["metadata_json"]) + if not public_publication_ready(metadata): + return None + translation = metadata.get("translation") + publication = metadata.get("publication") + if not isinstance(translation, dict) or not isinstance(publication, dict): + return None + title = _clean_text(translation.get("title_pre")) + summary = _clean_text(translation.get("summary_pre")) + reason = _clean_text(publication.get("recommendation_reason")) + issue_tags = _clean_list(publication.get("issue_tags")) + related_tags = _clean_list(publication.get("related_tags")) + region_tags = _clean_list(publication.get("region_tags")) + if not title or not summary or not reason or not (issue_tags or related_tags or region_tags): + return None + if not _translated_title_quality_ok(title, str(row["title_original"] or "")): + return None + return PublicTranslationPatch( + event_id=str(row["event_id"]), + target_id=str(row["target_id"]), + title=title, + summary=summary, + recommendation_reason=reason, + issue_tags=issue_tags, + related_tags=related_tags, + region_tags=region_tags, + published_at=str(row["published_at"] or row["created_at"] or ""), + ) + + +def _iter_ready_rows(db_path: Path, *, target_id: str | None = None) -> list[sqlite3.Row]: + if target_id: + sql = """ + SELECT event_id, target_id, stage, title_original, + published_at, created_at, metadata_json + FROM event_index + WHERE stage = 'drafts' AND public_translation_ready = 1 AND target_id = ? + ORDER BY datetime(COALESCE(published_at, created_at)) DESC, event_id DESC + """ + params = [target_id] + else: + sql = """ + SELECT event_id, target_id, stage, title_original, + published_at, created_at, metadata_json + FROM event_index + WHERE stage = 'drafts' AND public_translation_ready = 1 + ORDER BY datetime(COALESCE(published_at, created_at)) DESC, event_id DESC + """ + params = [] + with closing(sqlite3.connect(db_path)) as conn: + conn.row_factory = sqlite3.Row + return list(conn.execute(sql, params)) + + +def collect_translation_patches( + *, + data_dir: Path, + target_id: str | None = None, + limit: int | None = None, +) -> list[PublicTranslationPatch]: + patches: list[PublicTranslationPatch] = [] + for db_path in sorted(data_dir.glob("*/state.db")): + for row in _iter_ready_rows(db_path, target_id=target_id): + patch = _patch_from_row(row) + if patch is None: + continue + patches.append(patch) + if limit is not None and len(patches) >= limit: + return patches + return patches + + +def _patch_update(patch: PublicTranslationPatch) -> str: + tags = _json([*patch.issue_tags, *patch.related_tags, *patch.region_tags]) + return ( + "UPDATE events SET " # noqa: S608 + f"title={_sql(patch.title)}, " + f"summary={_sql(patch.summary)}, " + f"recommendation_reason={_sql(patch.recommendation_reason)}, " + f"tags={_sql(tags)}, " + f"issue_tags={_sql(_json(patch.issue_tags))}, " + f"related_tags={_sql(_json(patch.related_tags))}, " + f"region_tags={_sql(_json(patch.region_tags))}, " + "language='zh', " + "updated_at=datetime('now') " + f"WHERE event_id={_sql(patch.event_id)} " + "AND pipeline_stage='drafts';" + ) + + +def generate_translation_backfill_sql( + patches: list[PublicTranslationPatch], + *, + transaction: bool = False, +) -> str: + if not patches: + return "-- no public translation patches ready\n" + statements = ["BEGIN TRANSACTION;"] if transaction else [] + statements.extend(_patch_update(patch) for patch in patches) + if transaction: + statements.append("COMMIT;") + return "\n".join(statements) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data-dir", type=Path, default=Path("data")) + parser.add_argument("--target-id") + parser.add_argument("--limit", type=int) + parser.add_argument("--output-sql", type=Path) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--transaction", action="store_true") + args = parser.parse_args() + + patches = collect_translation_patches( + data_dir=args.data_dir, + target_id=args.target_id, + limit=args.limit, + ) + targets = sorted({patch.target_id for patch in patches}) + print(f"patches={len(patches)} targets={len(targets)}") + if args.output_sql: + args.output_sql.write_text( + generate_translation_backfill_sql(patches, transaction=args.transaction), + encoding="utf-8", + ) + print(f"wrote {args.output_sql}") + elif not args.dry_run: + print(generate_translation_backfill_sql(patches, transaction=args.transaction), end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From a5adb69097ba3a0bea2262fa3bd6a7d6409cd77c Mon Sep 17 00:00:00 2001 From: xucroyuri Date: Sun, 28 Jun 2026 20:55:48 +0800 Subject: [PATCH 04/16] feat: separate breaking home from all news --- frontend/cloudflare/db/schema.sql | 1 + frontend/cloudflare/workers/api/bootstrap.ts | 71 ++--- frontend/cloudflare/workers/api/news.ts | 228 ++++++--------- .../workers/lib/public-news-query.ts | 167 +++++++++++ frontend/public/src/App.test.tsx | 48 ++- frontend/public/src/App.tsx | 27 +- frontend/public/src/pages/public-pages.tsx | 273 ++++++++++++++++++ tests/unit/test_cloudflare_d1_backfill.py | 24 ++ tests/unit/test_cloudflare_native_config.py | 58 +++- tools/cloudflare_d1_backfill.py | 7 +- 10 files changed, 699 insertions(+), 205 deletions(-) create mode 100644 frontend/cloudflare/workers/lib/public-news-query.ts diff --git a/frontend/cloudflare/db/schema.sql b/frontend/cloudflare/db/schema.sql index 4618043e..cfd4428e 100644 --- a/frontend/cloudflare/db/schema.sql +++ b/frontend/cloudflare/db/schema.sql @@ -47,6 +47,7 @@ CREATE INDEX IF NOT EXISTS idx_events_published_at ON events(published_at); CREATE INDEX IF NOT EXISTS idx_events_pipeline_stage ON events(pipeline_stage); CREATE INDEX IF NOT EXISTS idx_events_source_id ON events(source_id); CREATE INDEX IF NOT EXISTS idx_events_value_label ON events(value_label); +CREATE INDEX IF NOT EXISTS idx_events_public_featured ON events(pipeline_stage, value_score DESC, published_at DESC); -- 来源表 CREATE TABLE IF NOT EXISTS sources ( diff --git a/frontend/cloudflare/workers/api/bootstrap.ts b/frontend/cloudflare/workers/api/bootstrap.ts index 3118788d..c0211183 100644 --- a/frontend/cloudflare/workers/api/bootstrap.ts +++ b/frontend/cloudflare/workers/api/bootstrap.ts @@ -12,6 +12,13 @@ import type { PublicFacetsResponse, } from "../lib/contracts"; import type { PublicFacetItem } from "../lib/contracts"; +import { + buildPublicNewsWhere, + type NewsRow, + PUBLIC_NEWS_SELECT_COLUMNS, + publicNewsOrderBy, + rowToPublicNewsItem, +} from "../lib/public-news-query"; export async function handleBootstrap( request: Request, @@ -20,25 +27,34 @@ export async function handleBootstrap( _segments: string[], ): Promise { try { + const featured = params.get("featured") !== "false"; + const newsFilters = buildPublicNewsWhere({ featured }); // 并行查询 news、regions、facets - const [newsResult, regionsResult, regionFacetsResult, issueFacetsResult, relatedFacetsResult] = + const [ + newsResult, + newsCountResult, + regionsResult, + regionFacetsResult, + issueFacetsResult, + relatedFacetsResult, + ] = await Promise.all([ db .prepare( - `SELECT event_id, target_id, target_label, - source_id, source_name, source_type, credibility_label, - published_at, title, original_title, summary, - recommendation_reason, full_content, original_url, - detail_url, image_urls, tags, issue_tags, related_tags, - region_tags, entities, related_count, discussion_count, - value_label, value_score, china_relevance_label + `SELECT ${PUBLIC_NEWS_SELECT_COLUMNS} FROM events - WHERE pipeline_stage = 'drafts' - ORDER BY published_at DESC + ${newsFilters.sql} + ${publicNewsOrderBy(featured)} LIMIT 20` ) + .bind(...newsFilters.bindings) .all(), + db + .prepare(`SELECT COUNT(*) AS total FROM events ${newsFilters.sql}`) + .bind(...newsFilters.bindings) + .first<{ total: number }>(), + db .prepare( `SELECT region_id, display_name, primary_language, region_type, @@ -84,43 +100,14 @@ export async function handleBootstrap( ]); // 构建 news - const newsRows = newsResult.results || []; + const newsRows = (newsResult.results || []) as NewsRow[]; const newsFeed: PublicNewsFeedResponse = { - items: newsRows.map((r: any) => ({ - id: r.event_id, - targetId: r.target_id, - targetLabel: r.target_label, - source: { - id: r.source_id, - name: r.source_name, - type: r.source_type || "unknown", - credibilityLabel: r.credibility_label, - }, - publishedAt: r.published_at, - title: r.title, - originalTitle: r.original_title, - summary: r.summary, - recommendationReason: r.recommendation_reason, - fullContent: r.full_content, - imageUrls: typeof r.image_urls === "string" ? JSON.parse(r.image_urls || "[]") : [], - originalUrl: r.original_url, - detailUrl: r.detail_url, - tags: typeof r.tags === "string" ? JSON.parse(r.tags || "[]") : [], - issueTags: typeof r.issue_tags === "string" ? JSON.parse(r.issue_tags || "[]") : [], - relatedTags: typeof r.related_tags === "string" ? JSON.parse(r.related_tags || "[]") : [], - regionTags: typeof r.region_tags === "string" ? JSON.parse(r.region_tags || "[]") : [], - entities: [], - relatedCount: r.related_count || 0, - discussionCount: r.discussion_count, - valueLabel: r.value_label || "普通", - valueScore: r.value_score, - chinaRelevanceLabel: r.china_relevance_label || "未知", - })), + items: newsRows.map((r) => rowToPublicNewsItem(r)), latestCursor: null, nextCursor: null, pollAfterMs: 60000, hasNewer: false, - total: newsRows.length, + total: newsCountResult?.total ?? newsRows.length, }; // 构建 regions diff --git a/frontend/cloudflare/workers/api/news.ts b/frontend/cloudflare/workers/api/news.ts index b347ab20..818a73dd 100644 --- a/frontend/cloudflare/workers/api/news.ts +++ b/frontend/cloudflare/workers/api/news.ts @@ -8,92 +8,57 @@ import type { PublicNewsFeedResponse, - PublicNewsItem, - PublicNewsSource, } from "../lib/contracts"; -import type { PaginationParams } from "../lib/d1"; -import { paginateRows } from "../lib/d1"; import { notFound } from "../lib/errors"; - -interface NewsRow { +import { + buildPublicNewsWhere, + type NewsRow, + PUBLIC_NEWS_SELECT_COLUMNS, + publicNewsOrderBy, + rowToPublicNewsItem, +} from "../lib/public-news-query"; + +interface CursorRow { event_id: string; - target_id: string; - target_label: string; - source_id: string; - source_name: string; - source_type: string; published_at: string; - title: string; - original_title: string | null; - summary: string | null; - recommendation_reason: string | null; - full_content: string | null; - original_url: string | null; - detail_url: string; - image_urls: string; - tags: string; - issue_tags: string; - related_tags: string; - region_tags: string; - entities: string; - related_count: number; - discussion_count: number | null; - value_label: string; value_score: number | null; - china_relevance_label: string; - credibility_label: string | null; -} - -function parseJsonArray(raw: string): string[] { - if (!raw) return []; - try { - return JSON.parse(raw) as string[]; - } catch { - return []; - } } -function parseEntities(raw: string): { name: string; type?: string | null }[] { - if (!raw) return []; - try { - return JSON.parse(raw) as { name: string; type?: string | null }[]; - } catch { - return []; +async function buildCursorFilter( + db: D1Database, + eventId: string | null, + mode: "before" | "since", + featured: boolean, +): Promise<{ sql: string; bindings: unknown[] }> { + if (!eventId) return { sql: "", bindings: [] }; + + const cursor = await db + .prepare("SELECT event_id, published_at, value_score FROM events WHERE event_id = ?") + .bind(eventId) + .first(); + if (!cursor) return { sql: "", bindings: [] }; + + if (featured) { + const score = cursor.value_score ?? -1; + const scoreOp = mode === "before" ? "<" : ">"; + const timeOp = mode === "before" ? "<" : ">"; + const idOp = mode === "before" ? "<" : ">"; + return { + sql: ` + AND ( + COALESCE(value_score, -1) ${scoreOp} ? + OR (COALESCE(value_score, -1) = ? AND published_at ${timeOp} ?) + OR (COALESCE(value_score, -1) = ? AND published_at = ? AND event_id ${idOp} ?) + ) + `, + bindings: [score, score, cursor.published_at, score, cursor.published_at, cursor.event_id], + }; } -} - -function rowToPublicNewsItem(row: NewsRow, targetId: string): PublicNewsItem { - const source: PublicNewsSource = { - id: row.source_id, - name: row.source_name, - type: row.source_type as PublicNewsSource["type"] || "unknown", - credibilityLabel: row.credibility_label, - }; + const op = mode === "before" ? "<" : ">"; return { - id: row.event_id, - targetId: row.target_id, - targetLabel: row.target_label, - source, - publishedAt: row.published_at, - title: row.title, - originalTitle: row.original_title, - summary: row.summary, - recommendationReason: row.recommendation_reason, - fullContent: row.full_content, - imageUrls: parseJsonArray(row.image_urls), - originalUrl: row.original_url, - detailUrl: row.detail_url, - tags: parseJsonArray(row.tags), - issueTags: parseJsonArray(row.issue_tags), - relatedTags: parseJsonArray(row.related_tags), - regionTags: parseJsonArray(row.region_tags), - entities: parseEntities(row.entities), - relatedCount: row.related_count, - discussionCount: row.discussion_count, - valueLabel: row.value_label as PublicNewsItem["valueLabel"], - valueScore: row.value_score, - chinaRelevanceLabel: row.china_relevance_label as PublicNewsItem["chinaRelevanceLabel"], + sql: `AND (published_at ${op} ? OR (published_at = ? AND event_id ${op} ?))`, + bindings: [cursor.published_at, cursor.published_at, cursor.event_id], }; } @@ -110,72 +75,61 @@ export async function handleNewsFeed( const related = params.get("related") || undefined; const date = params.get("date") || undefined; const q = params.get("q") || undefined; - const pageSize = Math.min(parseInt(params.get("page_size") || "20", 10), 50); + const featured = params.get("featured") === "true"; + const beforeCursor = params.get("before_cursor"); + const sinceCursor = params.get("since_cursor"); + const requestedPageSize = Number.parseInt(params.get("page_size") || "20", 10); + const pageSize = + Number.isFinite(requestedPageSize) && requestedPageSize > 0 + ? Math.min(requestedPageSize, 50) + : 20; + const filters = buildPublicNewsWhere({ + featured, + regionId, + sourceId, + issue, + related, + date, + q, + }); + const cursorFilter = await buildCursorFilter( + db, + beforeCursor || sinceCursor, + beforeCursor ? "before" : "since", + featured, + ); let sql = ` - SELECT event_id, target_id, target_label, - source_id, source_name, source_type, credibility_label, - published_at, title, original_title, summary, - recommendation_reason, full_content, original_url, - detail_url, image_urls, tags, issue_tags, related_tags, - region_tags, entities, related_count, discussion_count, - value_label, value_score, china_relevance_label + SELECT ${PUBLIC_NEWS_SELECT_COLUMNS} FROM events - WHERE pipeline_stage = 'drafts' + ${filters.sql} + ${cursorFilter.sql} `; - const bindings: unknown[] = []; - if (regionId) { - sql += ` AND region_id = ?`; - bindings.push(regionId); - } - if (sourceId) { - sql += ` AND source_id = ?`; - bindings.push(sourceId); - } - if (issue) { - sql += ` AND issue_tags LIKE ?`; - bindings.push(`%${issue}%`); - } - if (related) { - sql += ` AND related_tags LIKE ?`; - bindings.push(`%${related}%`); - } - if (date) { - sql += ` AND published_at >= ? AND published_at < ?`; - bindings.push(`${date}T00:00:00`, `${date}T23:59:59`); - } - if (q) { - sql += ` AND (title LIKE ? OR summary LIKE ?)`; - const term = `%${q}%`; - bindings.push(term, term); - } + sql += ` ${publicNewsOrderBy(featured)} LIMIT ?`; + const bindings = [...filters.bindings, ...cursorFilter.bindings, pageSize + 1]; - sql += ` ORDER BY published_at DESC LIMIT ?`; - bindings.push(pageSize + 1); - - const stmt = db.prepare(sql).bind(...bindings); - - const result = await stmt.all(); + const [result, totalResult] = await Promise.all([ + db.prepare(sql).bind(...bindings).all(), + db + .prepare(`SELECT COUNT(*) AS total FROM events ${filters.sql}`) + .bind(...filters.bindings) + .first<{ total: number }>(), + ]); const rows = result.results || []; - - const pagination: PaginationParams = { - cursor: params.get("cursor"), - before_cursor: params.get("before_cursor"), - since_cursor: params.get("since_cursor"), - page_size: pageSize, - }; - - const { items: _paginatedItems, latest_cursor, next_cursor } = paginateRows(rows, pagination); - const items = rows.map((r) => rowToPublicNewsItem(r, r.target_id)); + const pageRows = rows.slice(0, pageSize); + const items = pageRows.map((r) => rowToPublicNewsItem(r)); + const latestCursor = pageRows[0]?.event_id ?? sinceCursor ?? beforeCursor ?? null; + const nextCursor = + rows.length > pageSize ? (pageRows[pageRows.length - 1]?.event_id ?? null) : null; const body: PublicNewsFeedResponse = { - items: items.slice(0, pageSize), - latestCursor: latest_cursor, - nextCursor: next_cursor, + items, + latestCursor, + nextCursor, pollAfterMs: 60000, - hasNewer: items.length > pageSize, - total: rows.length, + hasNewer: Boolean(sinceCursor && items.length > 0), + total: totalResult?.total ?? rows.length, }; return new Response(JSON.stringify(body), { @@ -211,13 +165,7 @@ export async function handleNewsDetail( try { const result = await db .prepare( - `SELECT event_id, target_id, target_label, - source_id, source_name, source_type, credibility_label, - published_at, title, original_title, summary, - recommendation_reason, full_content, original_url, - detail_url, image_urls, tags, issue_tags, related_tags, - region_tags, entities, related_count, discussion_count, - value_label, value_score, china_relevance_label + `SELECT ${PUBLIC_NEWS_SELECT_COLUMNS} FROM events WHERE event_id = ? AND pipeline_stage = 'drafts'` ) @@ -228,7 +176,7 @@ export async function handleNewsDetail( return notFound("Event not found"); } - const item = rowToPublicNewsItem(result, result.target_id); + const item = rowToPublicNewsItem(result); return new Response(JSON.stringify(item), { status: 200, headers: { "Content-Type": "application/json", "Cache-Control": "public, max-age=60" }, diff --git a/frontend/cloudflare/workers/lib/public-news-query.ts b/frontend/cloudflare/workers/lib/public-news-query.ts new file mode 100644 index 00000000..32f46234 --- /dev/null +++ b/frontend/cloudflare/workers/lib/public-news-query.ts @@ -0,0 +1,167 @@ +/** + * Shared public-news SQL helpers for Worker endpoints. + * + * Keep this aligned with the Python public reader's featured quality gate: + * drafts stage, score floor, translated summary, recommendation reason, and + * non-uncategorized classification. + */ + +import type { PublicNewsItem, PublicNewsSource } from "./contracts"; + +export const PUBLIC_FEATURED_MIN_SCORE = 60; + +export interface NewsRow { + event_id: string; + target_id: string; + target_label: string; + source_id: string; + source_name: string; + source_type: string; + published_at: string; + title: string; + original_title: string | null; + summary: string | null; + recommendation_reason: string | null; + full_content: string | null; + original_url: string | null; + detail_url: string; + image_urls: string; + tags: string; + issue_tags: string; + related_tags: string; + region_tags: string; + entities: string; + related_count: number; + discussion_count: number | null; + value_label: string; + value_score: number | null; + china_relevance_label: string; + credibility_label: string | null; +} + +export const PUBLIC_NEWS_SELECT_COLUMNS = ` + event_id, target_id, target_label, + source_id, source_name, source_type, credibility_label, + published_at, title, original_title, summary, + recommendation_reason, full_content, original_url, + detail_url, image_urls, tags, issue_tags, related_tags, + region_tags, entities, related_count, discussion_count, + value_label, value_score, china_relevance_label +`; + +export interface PublicNewsFilterInput { + featured?: boolean; + regionId?: string; + sourceId?: string; + issue?: string; + related?: string; + date?: string; + q?: string; +} + +export function buildPublicNewsWhere(input: PublicNewsFilterInput): { + sql: string; + bindings: unknown[]; +} { + let sql = `WHERE pipeline_stage = 'drafts'`; + const bindings: unknown[] = []; + + if (input.featured) { + sql += ` + AND value_score >= ? + AND summary IS NOT NULL AND TRIM(summary) <> '' + AND recommendation_reason IS NOT NULL AND TRIM(recommendation_reason) <> '' + AND json_valid(classification) = 1 + AND COALESCE(NULLIF(LOWER(TRIM(json_extract(classification, '$.l0'))), ''), 'uncategorized') + NOT IN ('uncategorized', 'other', 'breaking_news') + `; + bindings.push(PUBLIC_FEATURED_MIN_SCORE); + } + + if (input.regionId) { + sql += ` AND region_id = ?`; + bindings.push(input.regionId); + } + if (input.sourceId) { + sql += ` AND source_id = ?`; + bindings.push(input.sourceId); + } + if (input.issue) { + sql += ` AND issue_tags LIKE ?`; + bindings.push(`%${input.issue}%`); + } + if (input.related) { + sql += ` AND related_tags LIKE ?`; + bindings.push(`%${input.related}%`); + } + if (input.date) { + sql += ` AND published_at >= ? AND published_at < ?`; + bindings.push(`${input.date}T00:00:00`, `${input.date}T23:59:59`); + } + if (input.q) { + sql += ` AND (title LIKE ? OR summary LIKE ?)`; + const term = `%${input.q}%`; + bindings.push(term, term); + } + + return { sql, bindings }; +} + +export function publicNewsOrderBy(featured: boolean): string { + return featured + ? "ORDER BY value_score DESC, published_at DESC, event_id DESC" + : "ORDER BY published_at DESC, event_id DESC"; +} + +function parseJsonArray(raw: string): string[] { + if (!raw) return []; + try { + return JSON.parse(raw) as string[]; + } catch { + return []; + } +} + +function parseEntities(raw: string): { name: string; type?: string | null }[] { + if (!raw) return []; + try { + return JSON.parse(raw) as { name: string; type?: string | null }[]; + } catch { + return []; + } +} + +export function rowToPublicNewsItem(row: NewsRow): PublicNewsItem { + const source: PublicNewsSource = { + id: row.source_id, + name: row.source_name, + type: (row.source_type as PublicNewsSource["type"]) || "unknown", + credibilityLabel: row.credibility_label, + }; + + return { + id: row.event_id, + targetId: row.target_id, + targetLabel: row.target_label, + source, + publishedAt: row.published_at, + title: row.title, + originalTitle: row.original_title, + summary: row.summary, + recommendationReason: row.recommendation_reason, + fullContent: row.full_content, + imageUrls: parseJsonArray(row.image_urls), + originalUrl: row.original_url, + detailUrl: row.detail_url, + tags: parseJsonArray(row.tags), + issueTags: parseJsonArray(row.issue_tags), + relatedTags: parseJsonArray(row.related_tags), + regionTags: parseJsonArray(row.region_tags), + entities: parseEntities(row.entities), + relatedCount: row.related_count, + discussionCount: row.discussion_count, + valueLabel: row.value_label as PublicNewsItem["valueLabel"], + valueScore: row.value_score, + chinaRelevanceLabel: row.china_relevance_label as PublicNewsItem["chinaRelevanceLabel"], + }; +} diff --git a/frontend/public/src/App.test.tsx b/frontend/public/src/App.test.tsx index 1b2a019e..8441e2a8 100644 --- a/frontend/public/src/App.test.tsx +++ b/frontend/public/src/App.test.tsx @@ -303,14 +303,17 @@ describe("Phase 84 public portal app", () => { vi.unstubAllGlobals() }) - it("renders an AIHOT-style sidebar reader feed instead of the simplified top nav shell", async () => { + it("renders a breaking-news home instead of the all-news timeline", async () => { installFetchMock() render() - expect(await screen.findByRole("heading", { name: "新闻哨兵" })).toBeInTheDocument() - expect(screen.getAllByRole("heading", { name: "新闻哨兵" })).toHaveLength(1) - expect(await screen.findByRole("region", { name: "当前热点" })).toBeInTheDocument() + expect(await screen.findByRole("heading", { name: "极速突发" })).toBeInTheDocument() + expect(screen.getAllByRole("heading", { name: "极速突发" })).toHaveLength(1) + expect(await screen.findByRole("region", { name: "极速突发" })).toBeInTheDocument() + expect(screen.getByRole("region", { name: "高价值动态" })).toBeInTheDocument() + expect(screen.getByRole("region", { name: "最近推进" })).toBeInTheDocument() + expect(screen.getByRole("region", { name: "突发快捷入口" })).toBeInTheDocument() expect(screen.queryByText("新闻时间线")).not.toBeInTheDocument() const nav = screen.getByRole("navigation", { name: "公共站侧边栏" }) expect(within(nav).getByRole("button", { name: /新闻哨兵 Breaking News/ })).toBeInTheDocument() @@ -323,11 +326,17 @@ describe("Phase 84 public portal app", () => { "true", ) expect(screen.getAllByRole("heading", { name: LEAD_TITLE }).length).toBeGreaterThan(0) - expect(screen.getAllByText(LEAD_ORIGINAL_TITLE).length).toBeGreaterThan(0) expect(screen.getAllByText("ANSA.it").length).toBeGreaterThan(0) expect(screen.getAllByText("会谈聚焦贸易政策与市场准入，双方同意继续保持沟通。").length).toBeGreaterThan(0) - expect(screen.queryByText(/推荐理由：/)).not.toBeInTheDocument() + expect(screen.getByText(/为什么重要：/)).toBeInTheDocument() expect(screen.getAllByText("92").length).toBeGreaterThan(0) + expect( + within(screen.getByRole("region", { name: "极速突发" })).getByRole("button", { + name: "新闻纵览", + }), + ).toBeInTheDocument() + expect(screen.queryByRole("button", { name: "筛选" })).not.toBeInTheDocument() + expect(screen.queryByRole("region", { name: "地区议题相关筛选" })).not.toBeInTheDocument() expect(screen.queryByRole("link", { name: /详情/ })).not.toBeInTheDocument() expect(screen.getAllByText(/\d{2}:\d{2}/).length).toBeGreaterThan(0) expect(screen.getAllByText("意大利").length).toBeGreaterThan(0) @@ -370,6 +379,7 @@ describe("Phase 84 public portal app", () => { return jsonResponse({}) }) vi.stubGlobal("fetch", fetchMock) + window.location.hash = "#/feed?channel=all" render() @@ -440,6 +450,7 @@ describe("Phase 84 public portal app", () => { return jsonResponse({}) }) vi.stubGlobal("fetch", fetchMock) + window.location.hash = "#/feed?channel=all" render() @@ -450,6 +461,7 @@ describe("Phase 84 public portal app", () => { it("keeps the hot-topic strip visually compact", async () => { installFetchMock() + window.location.hash = "#/feed?channel=all" render() @@ -466,6 +478,7 @@ describe("Phase 84 public portal app", () => { it("collapses a date group when the date header is clicked", async () => { installFetchMock() + window.location.hash = "#/feed?channel=all" render() @@ -485,6 +498,7 @@ describe("Phase 84 public portal app", () => { it("automatically inserts polled news at the top with an entering marker", async () => { vi.useFakeTimers({ shouldAdvanceTime: true }) installFetchMock() + window.location.hash = "#/feed?channel=all" render() @@ -508,7 +522,7 @@ describe("Phase 84 public portal app", () => { const { container } = render() - await screen.findByRole("heading", { name: "新闻哨兵" }) + await screen.findByRole("heading", { name: "极速突发" }) const logo = screen.getByRole("link", { name: "NewsSentry" }) expect(logo.className).not.toMatch(/\bborder\b/) expect(logo.className).not.toContain("bg-white/[0.03]") @@ -572,7 +586,7 @@ describe("Phase 84 public portal app", () => { const { container } = render() - await screen.findByRole("heading", { name: "新闻哨兵" }) + await screen.findByRole("heading", { name: "极速突发" }) const renderedMarkup = container.innerHTML expect(renderedMarkup).not.toMatch(/cyan|text-blue|bg-blue|border-blue/i) @@ -593,7 +607,6 @@ describe("Phase 84 public portal app", () => { render() - await screen.findByRole("heading", { name: "新闻哨兵" }) const loadingState = screen.getByLabelText("紧凑加载状态") expect(loadingState.className).toContain("py-2") expect(screen.getByText("更新中")).toBeInTheDocument() @@ -630,7 +643,7 @@ describe("Phase 84 public portal app", () => { const { container } = render() - await screen.findByRole("heading", { name: "新闻哨兵" }) + await screen.findByRole("heading", { name: "极速突发" }) const main = container.querySelector("main") expect(main).not.toBeNull() expect(main?.className).toContain("w-full") @@ -675,10 +688,11 @@ describe("Phase 84 public portal app", () => { it("renders the reading feed without desktop side rails", async () => { installFetchMock() + window.location.hash = "#/feed?channel=all" const { container } = render() - await screen.findByRole("heading", { name: "新闻哨兵" }) + await screen.findByRole("heading", { name: "新闻纵览" }) expect(container.querySelector("main > aside")).toBeNull() expect(screen.getByRole("button", { name: "筛选" })).toBeInTheDocument() expect(screen.getByRole("region", { name: "地区议题相关筛选" })).toBeInTheDocument() @@ -862,6 +876,7 @@ describe("Phase 84 public portal app", () => { it("opens the filter drawer only when the reader asks for more filters", async () => { installFetchMock() + window.location.hash = "#/feed?channel=all" render() @@ -882,10 +897,10 @@ describe("Phase 84 public portal app", () => { render() await findLeadStory() - fireEvent.click(screen.getByRole("button", { name: "意大利" })) + fireEvent.click(screen.getByRole("button", { name: /意大利/ })) expect(window.location.pathname).toBe("/public-app/") - expect(window.location.search).toContain("channel=targets") + expect(window.location.search).toContain("channel=all") expect(window.location.search).toContain("target_id=italy") expect(await screen.findByRole("heading", { name: "意大利" })).toBeInTheDocument() expect(screen.getByRole("heading", { name: "当前热点" })).toBeInTheDocument() @@ -1014,6 +1029,7 @@ describe("Phase 84 public portal app", () => { return jsonResponse({}) }) vi.stubGlobal("fetch", fetchMock) + window.location.hash = "#/feed?channel=all" render() @@ -1043,6 +1059,7 @@ describe("Phase 84 public portal app", () => { it("auto-inserts new items without interrupting the reader with a banner", async () => { vi.useFakeTimers({ shouldAdvanceTime: true }) installFetchMock() + window.location.hash = "#/feed?channel=all" render() await findLeadStory() @@ -1057,6 +1074,7 @@ describe("Phase 84 public portal app", () => { it("loads older news and keeps the mobile bottom navigation active", async () => { installFetchMock() + window.location.hash = "#/feed?channel=all" render() await findLeadStory() @@ -1306,12 +1324,12 @@ describe("Phase 84 public portal app", () => { it("writes feed filter changes back into the public app url", async () => { installFetchMock() - window.location.hash = "#/feed?channel=featured" + window.location.hash = "#/feed?channel=all" render() await findLeadStory() - fireEvent.click(screen.getByRole("button", { name: "外交" })) + fireEvent.click(screen.getByRole("button", { name: /外交/ })) expect(window.location.pathname).toBe("/public-app/") expect(window.location.search).toContain("issue=%E5%9B%BD%E9%99%85%E5%85%B3%E7%B3%BB") diff --git a/frontend/public/src/App.tsx b/frontend/public/src/App.tsx index d63c90bb..8a86cb18 100644 --- a/frontend/public/src/App.tsx +++ b/frontend/public/src/App.tsx @@ -42,6 +42,7 @@ import { routeToChannel, type PublicRoute } from "@/lib/routes" import { buildRouteSeoPayload } from "@/lib/seo/site-seo" import { AnalysisPage, + BreakingHomePage, DailyPage, EventDetailPage, NewsFeedPage, @@ -1291,9 +1292,14 @@ export default function App() { const bootstrap = usePublicBootstrap(filters) const bootstrapTargets = useMemo(() => regionsToTargets(bootstrap.data), [bootstrap.data]) const ssrFeed = useMemo(() => readSSRFeed(), []) - const ssrFeedApplicable = ssrFeed !== null && isDefaultFeaturedFilters(filters) - const initialFeed = ssrFeedApplicable ? ssrFeed : (bootstrap.data?.news ?? null) - const waitForBootstrap = bootstrap.status === "loading" && !ssrFeedApplicable + const defaultFeaturedHome = isDefaultFeaturedFilters(filters) + const ssrFeedApplicable = ssrFeed !== null && defaultFeaturedHome + const initialFeed = ssrFeedApplicable + ? ssrFeed + : defaultFeaturedHome + ? (bootstrap.data?.news ?? null) + : null + const waitForBootstrap = bootstrap.status === "loading" && defaultFeaturedHome && !ssrFeedApplicable const feed = usePublicFeed(filters, { poll: route.name === "feed", initialFeed, @@ -1395,6 +1401,19 @@ export default function App() { preferredSection={route.section} /> ) + } else if (route.name === "feed" && defaultFeaturedHome) { + mainContent = ( + void feed.loadFeed("refresh")} + onOpenAll={() => navigatePrimary("all")} + onSelectTarget={(targetId) => updateFilters({ channel: "all", targetId })} + onSelectIssue={(issue) => updateFilters({ channel: "all", issue })} + onApplyPending={feed.applyPending} + /> + ) } else { const selectedFeedTargetLabel = filters.targetId ? targets.find((target) => target.target_id === filters.targetId)?.display_name ?? @@ -1432,7 +1451,7 @@ export default function App() {
- {route.name === "feed" && ( + {route.name === "feed" && !defaultFeaturedHome && ( + (right.valueScore ?? 0) - (left.valueScore ?? 0) || + eventTimeValue(right) - eventTimeValue(left), + ) +} + +function sortRecentItems(items: PublicNewsItem[]) { + return [...items].sort((left, right) => eventTimeValue(right) - eventTimeValue(left)) +} + +function QuickEntryButton({ + label, + count, + onClick, +}: { + label: string + count: number + onClick: () => void +}) { + return ( + + ) +} + +export function BreakingHomePage({ + state, + targets, + facets, + onRefresh, + onOpenAll, + onSelectTarget, + onSelectIssue, + onApplyPending, +}: { + state: FeedState + targets: PublicTargetInfo[] + facets: { + issues: Array<{ id: string; label: string; count: number }> + related: Array<{ id: string; label: string; count: number }> + } + onRefresh: () => void + onOpenAll: () => void + onSelectTarget: (targetId: string) => void + onSelectIssue: (issue: string) => void + onApplyPending: () => void +}) { + const ranked = useMemo(() => sortBreakingItems(state.items), [state.items]) + const recent = useMemo(() => sortRecentItems(state.items).slice(0, 6), [state.items]) + const lead = ranked[0] + const topItems = ranked.slice(0, 3) + const regionEntries = useMemo( + () => + targets + .filter((target) => target.event_count > 0) + .sort((left, right) => right.event_count - left.event_count) + .slice(0, 8), + [targets], + ) + const issueEntries = useMemo( + () => [...facets.issues].sort((left, right) => right.count - left.count).slice(0, 8), + [facets.issues], + ) + const homeRoute: Extract = { + name: "feed", + channel: "featured", + search: new URLSearchParams(), + } + const leadRoute = lead ? buildDetailRoute(lead, homeRoute) : null + + if (state.status === "loading") { + return ( +
+ +
+ ) + } + + if (state.status === "error") { + return + } + + if (!lead) { + return ( +
+ +
+ ) + } + + return ( +
+ + +
+
+
+
+ Breaking + + {state.total} 条精选信号 · {formatTime(lead.publishedAt)} 更新 + +
+

极速突发

+

+ 只展示高价值、已完成摘要和推荐理由的突发新闻；完整时间线继续放在新闻纵览。 +

+
+ +
+ + {leadRoute ? ( + handleRouteAnchorClick(event, leadRoute)} + className="grid gap-3 px-3 py-4 hover:bg-accent/15 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring lg:grid-cols-[minmax(0,1fr)_14rem]" + aria-label={`主突发新闻 ${primaryNewsTitle(lead)}`} + > +
+
+ + 主突发 + + {lead.source.name} + {targetShortLabel(lead.targetLabel)} + {formatFullTime(lead.publishedAt)} +
+

+ {primaryNewsTitle(lead)} +

+ {lead.summary ? ( +

+ {lead.summary} +

+ ) : null} + {lead.recommendationReason ? ( +

+ 为什么重要： + {lead.recommendationReason} +

+ ) : null} +
+
+
+

Breaking News 分值

+

{Math.round(lead.valueScore ?? 0)}

+
+
+ {uniqueNewsTags(lead).slice(0, 5).map((tag) => ( + + {tag} + + ))} +
+
+ + ) : null} +
+ + + +
+
+

快速进入完整时间线

+ +
+
+
+

地区

+
+ {regionEntries.map((target) => ( + onSelectTarget(target.target_id)} + /> + ))} +
+
+
+

议题

+
+ {issueEntries.map((issue) => ( + onSelectIssue(issue.label)} + /> + ))} +
+
+
+
+
+ ) +} + function RelatedSection({ title, items, diff --git a/tests/unit/test_cloudflare_d1_backfill.py b/tests/unit/test_cloudflare_d1_backfill.py index 6b5c6143..6f87b68b 100644 --- a/tests/unit/test_cloudflare_d1_backfill.py +++ b/tests/unit/test_cloudflare_d1_backfill.py @@ -12,6 +12,8 @@ generate_translation_backfill_sql, ) +ROOT = Path(__file__).resolve().parents[2] + def _make_state_db(path: Path) -> None: path.parent.mkdir(parents=True) @@ -102,6 +104,7 @@ def test_collect_backfill_plan_uses_only_drafts_and_counts_targets(tmp_path: Pat assert [event.event_id for event in plan.events] == ["draft-1"] assert plan.events[0].published_at == "2026-05-29T23:45:00Z" + assert plan.events[0].classification == {"l0": "politics"} assert plan.targets[0].target_id == "italy" assert plan.targets[0].event_count == 1 assert plan.sources[0].source_id == "ansa" @@ -119,11 +122,32 @@ def test_generate_backfill_sql_is_idempotent_and_preserves_drafts_stage(tmp_path assert "INSERT INTO events" in sql assert "ON CONFLICT(event_id) DO UPDATE" in sql assert "'drafts'" in sql + assert "'{\"l0\":\"politics\"}'" in sql + assert "classification=excluded.classification" in sql assert "reviewed-1" not in sql assert "INSERT INTO targets" in sql assert "INSERT INTO sources" in sql +def test_generated_backfill_sql_executes_against_d1_schema(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + targets_dir = tmp_path / "config" / "targets" + targets_dir.mkdir(parents=True) + (targets_dir / "italy.yaml").write_text("target_id: italy\n", encoding="utf-8") + _make_state_db(data_dir / "italy" / "state.db") + + sql = generate_backfill_sql(collect_backfill_plan(data_dir=data_dir, targets_dir=targets_dir)) + + with closing(sqlite3.connect(tmp_path / "d1.sqlite")) as db: + db.executescript((ROOT / "frontend/cloudflare/db/schema.sql").read_text(encoding="utf-8")) + db.executescript(sql) + row = db.execute( + "SELECT event_id, pipeline_stage, classification FROM events" + ).fetchone() + + assert row == ("draft-1", "drafts", '{"l0":"politics"}') + + def test_public_translation_backfill_updates_only_ready_public_fields(tmp_path: Path) -> None: data_dir = tmp_path / "data" _make_state_db(data_dir / "italy" / "state.db") diff --git a/tests/unit/test_cloudflare_native_config.py b/tests/unit/test_cloudflare_native_config.py index fbd816b9..f1955208 100644 --- a/tests/unit/test_cloudflare_native_config.py +++ b/tests/unit/test_cloudflare_native_config.py @@ -53,21 +53,73 @@ def test_public_facets_contract_includes_related_tags() -> None: def test_public_news_supports_related_filter() -> None: news_ts = _read("workers/api/news.ts") + query_ts = _read("workers/lib/public-news-query.ts") assert 'params.get("related")' in news_ts - assert "related_tags LIKE ?" in news_ts + assert "related_tags LIKE ?" in query_ts def test_public_reader_uses_drafts_stage_like_python_reader() -> None: news_ts = _read("workers/api/news.ts") facets_ts = _read("workers/api/facets.ts") bootstrap_ts = _read("workers/api/bootstrap.ts") + query_ts = _read("workers/lib/public-news-query.ts") - for worker_source in (news_ts, facets_ts, bootstrap_ts): + for worker_source in (news_ts, facets_ts, bootstrap_ts, query_ts): assert "pipeline_stage = 'drafts'" in worker_source assert "pipeline_stage IN ('published', 'reviewed')" not in worker_source - assert "total: newsRows.length" in bootstrap_ts + assert "total: newsCountResult?.total ?? newsRows.length" in bootstrap_ts + + +def test_cloudflare_public_featured_query_matches_python_quality_gate() -> None: + news_ts = _read("workers/api/news.ts") + bootstrap_ts = _read("workers/api/bootstrap.ts") + query_ts = _read("workers/lib/public-news-query.ts") + + assert 'params.get("featured") === "true"' in news_ts + assert 'params.get("featured") !== "false"' in bootstrap_ts + assert "PUBLIC_FEATURED_MIN_SCORE = 60" in query_ts + assert "value_score >= ?" in query_ts + assert "summary IS NOT NULL AND TRIM(summary) <> ''" in query_ts + assert "recommendation_reason IS NOT NULL AND TRIM(recommendation_reason) <> ''" in query_ts + assert "json_valid(classification) = 1" in query_ts + assert "json_extract(classification, '$.l0')" in query_ts + assert "NOT IN ('uncategorized', 'other', 'breaking_news')" in query_ts + assert "ORDER BY value_score DESC, published_at DESC, event_id DESC" in query_ts + assert "publicNewsOrderBy(featured)" in news_ts + assert "publicNewsOrderBy(featured)" in bootstrap_ts + + +def test_cloudflare_public_news_uses_sql_cursor_pagination() -> None: + news_ts = _read("workers/api/news.ts") + query_ts = _read("workers/lib/public-news-query.ts") + + assert "buildCursorFilter" in news_ts + assert 'params.get("before_cursor")' in news_ts + assert 'params.get("since_cursor")' in news_ts + assert "SELECT event_id, published_at, value_score FROM events WHERE event_id = ?" in news_ts + assert "${cursorFilter.sql}" in news_ts + assert "SELECT COUNT(*) AS total FROM events ${filters.sql}" in news_ts + assert "Number.isFinite(requestedPageSize)" in news_ts + assert "const pageRows = rows.slice(0, pageSize)" in news_ts + assert "const items = pageRows.map" in news_ts + assert "hasNewer: Boolean(sinceCursor && items.length > 0)" in news_ts + assert "ORDER BY published_at DESC, event_id DESC" in query_ts + + +def test_cloudflare_d1_has_public_featured_index() -> None: + schema_sql = _read("db/schema.sql") + + assert "idx_events_public_featured" in schema_sql + assert "events(pipeline_stage, value_score DESC, published_at DESC)" in schema_sql + + +def test_cloudflare_bootstrap_reports_matching_featured_total() -> None: + bootstrap_ts = _read("workers/api/bootstrap.ts") + + assert "SELECT COUNT(*) AS total FROM events ${newsFilters.sql}" in bootstrap_ts + assert "total: newsCountResult?.total ?? newsRows.length" in bootstrap_ts def test_cloudflare_worker_exposes_public_targets_and_regions_contracts() -> None: diff --git a/tools/cloudflare_d1_backfill.py b/tools/cloudflare_d1_backfill.py index 29333ed1..5957b2f8 100644 --- a/tools/cloudflare_d1_backfill.py +++ b/tools/cloudflare_d1_backfill.py @@ -30,6 +30,7 @@ class D1Event: related_tags: list[str] region_tags: list[str] entities: list[dict[str, str]] + classification: dict[str, str] value_label: str value_score: int | None china_relevance_label: str @@ -173,6 +174,7 @@ def collect_backfill_plan( source_id = str(row["source_id"] or "unknown") topic_tags = _split_csv(row["topic_tags"]) issue_tags = [str(row["classification_l0"])] if row["classification_l0"] else [] + classification = {"l0": str(row["classification_l0"] or "uncategorized")} entities = [{"name": name} for name in _split_csv(row["entity_names"])] published_at = _normalize_timestamp(row["published_at"] or row["created_at"]) collected_at = _normalize_timestamp(row["created_at"] or row["published_at"]) @@ -193,6 +195,7 @@ def collect_backfill_plan( related_tags=topic_tags, region_tags=[target_id], entities=entities, + classification=classification, value_label=_value_label(row["news_value_score"]), value_score=row["news_value_score"], china_relevance_label=_china_relevance_label(row["china_relevance"]), @@ -253,7 +256,7 @@ def _event_insert(event: D1Event) -> str: "event_id, target_id, target_label, region_id, source_id, source_name, source_type, " "published_at, collected_at, title, original_title, original_url, detail_url, " "image_urls, tags, issue_tags, related_tags, region_tags, entities, language, " - "pipeline_stage, value_label, value_score, china_relevance_label" + "pipeline_stage, value_label, value_score, china_relevance_label, classification" ) values = [ event.event_id, @@ -280,6 +283,7 @@ def _event_insert(event: D1Event) -> str: event.value_label, event.value_score, event.china_relevance_label, + _json(event.classification), ] assignments = ( "target_id=excluded.target_id, target_label=excluded.target_label, " @@ -293,6 +297,7 @@ def _event_insert(event: D1Event) -> str: "region_tags=excluded.region_tags, entities=excluded.entities, " "pipeline_stage=excluded.pipeline_stage, value_label=excluded.value_label, " "value_score=excluded.value_score, china_relevance_label=excluded.china_relevance_label, " + "classification=excluded.classification, " "updated_at=datetime('now')" ) return ( From 24c8c58f544fc40e26069996e7a34dedf1406ca6 Mon Sep 17 00:00:00 2001 From: xucroyuri Date: Sun, 28 Jun 2026 21:00:38 +0800 Subject: [PATCH 05/16] fix: keep breaking home populated during translation lag --- frontend/cloudflare/workers/lib/public-news-query.ts | 9 ++++----- frontend/public/src/pages/public-pages.tsx | 4 ++-- tests/unit/test_cloudflare_native_config.py | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/frontend/cloudflare/workers/lib/public-news-query.ts b/frontend/cloudflare/workers/lib/public-news-query.ts index 32f46234..e9d4e86a 100644 --- a/frontend/cloudflare/workers/lib/public-news-query.ts +++ b/frontend/cloudflare/workers/lib/public-news-query.ts @@ -1,9 +1,10 @@ /** * Shared public-news SQL helpers for Worker endpoints. * - * Keep this aligned with the Python public reader's featured quality gate: - * drafts stage, score floor, translated summary, recommendation reason, and - * non-uncategorized classification. + * Keep this aligned with the public reader's featured quality gate: + * drafts stage, score floor, and non-uncategorized classification. Summary and + * recommendation fields are displayed when available, but are not hard gates + * because translation/publication backfill can lag behind collection. */ import type { PublicNewsItem, PublicNewsSource } from "./contracts"; @@ -69,8 +70,6 @@ export function buildPublicNewsWhere(input: PublicNewsFilterInput): { if (input.featured) { sql += ` AND value_score >= ? - AND summary IS NOT NULL AND TRIM(summary) <> '' - AND recommendation_reason IS NOT NULL AND TRIM(recommendation_reason) <> '' AND json_valid(classification) = 1 AND COALESCE(NULLIF(LOWER(TRIM(json_extract(classification, '$.l0'))), ''), 'uncategorized') NOT IN ('uncategorized', 'other', 'breaking_news') diff --git a/frontend/public/src/pages/public-pages.tsx b/frontend/public/src/pages/public-pages.tsx index 5615b51c..1b22cf3f 100644 --- a/frontend/public/src/pages/public-pages.tsx +++ b/frontend/public/src/pages/public-pages.tsx @@ -657,7 +657,7 @@ export function BreakingHomePage({
) @@ -678,7 +678,7 @@ export function BreakingHomePage({

极速突发

- 只展示高价值、已完成摘要和推荐理由的突发新闻；完整时间线继续放在新闻纵览。 + 优先展示高价值、分类明确的突发新闻；完整时间线继续放在新闻纵览。