From d9d9f1bfa6eb758c331d295b803e606e34e50a9a Mon Sep 17 00:00:00 2001 From: Tanishq Date: Tue, 23 Jun 2026 16:39:39 +0000 Subject: [PATCH] feat(websearch): add Exa as a web search provider - Add ExaWebSearchTool (web_search_exa) with keyword/semantic search, category filters, domain restrictions, and date range support - Add ExaGetContentsTool (exa_get_contents) for extracting web page content - Add _exa_search() and _exa_get_contents() API helpers hitting https://api.exa.ai/search and https://api.exa.ai/contents - Add _EXA_KEY_ROTATOR for multi-key rotation - Register Exa tools in _apply_web_search_tools() dispatch - Add Exa to WEB_SEARCH_CITATION_TOOL_NAMES for citation support - Add websearch_exa_key config default and provider option - Add i18n metadata for en-US, zh-CN, ru-RU - Add Exa section to docs (en + zh) - Add 6 unit tests covering search, contents, error handling, and legacy config migration Closes #5621 Co-Authored-By: Tanishq Jaiswal --- astrbot/core/astr_main_agent.py | 6 + astrbot/core/config/default.py | 12 + astrbot/core/tools/web_search_tools.py | 229 ++++++++++++++++++ .../en-US/features/config-metadata.json | 20 +- .../ru-RU/features/config-metadata.json | 4 + .../zh-CN/features/config-metadata.json | 4 + docs/en/use/websearch.md | 8 +- docs/zh/use/websearch.md | 8 +- tests/unit/test_web_search_tools.py | 135 +++++++++++ 9 files changed, 414 insertions(+), 12 deletions(-) diff --git a/astrbot/core/astr_main_agent.py b/astrbot/core/astr_main_agent.py index af3ac71322..16ebac7a8b 100644 --- a/astrbot/core/astr_main_agent.py +++ b/astrbot/core/astr_main_agent.py @@ -85,6 +85,8 @@ BaiduWebSearchTool, BochaWebSearchTool, BraveWebSearchTool, + ExaGetContentsTool, + ExaWebSearchTool, FirecrawlExtractWebPageTool, FirecrawlWebSearchTool, TavilyExtractWebPageTool, @@ -130,6 +132,7 @@ "web_search_tavily", "web_search_bocha", "web_search_brave", + "web_search_exa", } ) WEB_SEARCH_CITATION_PROMPT = ( @@ -1207,6 +1210,9 @@ async def _apply_web_search_tools( req.func_tool.add_tool(tool_mgr.get_builtin_tool(FirecrawlExtractWebPageTool)) elif provider == "baidu_ai_search": req.func_tool.add_tool(tool_mgr.get_builtin_tool(BaiduWebSearchTool)) + elif provider == "exa": + req.func_tool.add_tool(tool_mgr.get_builtin_tool(ExaWebSearchTool)) + req.func_tool.add_tool(tool_mgr.get_builtin_tool(ExaGetContentsTool)) def _apply_web_search_citation_prompt( diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 27cc61415f..9ddb4aa64d 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -115,6 +115,7 @@ "websearch_brave_key": [], "websearch_baidu_app_builder_key": "", "websearch_firecrawl_key": [], + "websearch_exa_key": [], "web_search_link": False, "display_reasoning_text": False, "identifier": False, @@ -3295,6 +3296,7 @@ "bocha", "brave", "firecrawl", + "exa", ], "condition": { "provider_settings.web_search": True, @@ -3349,6 +3351,16 @@ "provider_settings.web_search": True, }, }, + "provider_settings.websearch_exa_key": { + "description": "Exa API Key", + "type": "list", + "items": {"type": "string"}, + "hint": "可添加多个 Key 进行轮询。Get a key at https://dashboard.exa.ai", + "condition": { + "provider_settings.websearch_provider": "exa", + "provider_settings.web_search": True, + }, + }, "provider_settings.web_search_link": { "description": "显示来源引用", "type": "bool", diff --git a/astrbot/core/tools/web_search_tools.py b/astrbot/core/tools/web_search_tools.py index ebd13d0102..7af425045c 100644 --- a/astrbot/core/tools/web_search_tools.py +++ b/astrbot/core/tools/web_search_tools.py @@ -21,6 +21,8 @@ "web_search_brave", "web_search_firecrawl", "firecrawl_extract_web_page", + "web_search_exa", + "exa_get_contents", ] _TAVILY_WEB_SEARCH_TOOL_CONFIG = { "provider_settings.web_search": True, @@ -42,6 +44,10 @@ "provider_settings.web_search": True, "provider_settings.websearch_provider": "baidu_ai_search", } +_EXA_WEB_SEARCH_TOOL_CONFIG = { + "provider_settings.web_search": True, + "provider_settings.websearch_provider": "exa", +} @std_dataclass @@ -76,6 +82,7 @@ async def get(self, provider_settings: dict) -> str: _BOCHA_KEY_ROTATOR = _KeyRotator("websearch_bocha_key", "BoCha") _BRAVE_KEY_ROTATOR = _KeyRotator("websearch_brave_key", "Brave") _FIRECRAWL_KEY_ROTATOR = _KeyRotator("websearch_firecrawl_key", "Firecrawl") +_EXA_KEY_ROTATOR = _KeyRotator("websearch_exa_key", "Exa") def normalize_legacy_web_search_config(cfg) -> None: @@ -99,6 +106,7 @@ def normalize_legacy_web_search_config(cfg) -> None: "websearch_bocha_key", "websearch_brave_key", "websearch_firecrawl_key", + "websearch_exa_key", ): value = provider_settings.get(setting_name) if isinstance(value, str): @@ -803,10 +811,231 @@ async def call(self, context, **kwargs) -> ToolExecResult: return _search_result_payload(results) +async def _exa_search( + provider_settings: dict, + payload: dict, +) -> list[SearchResult]: + """Call the Exa /search endpoint and return normalized results.""" + exa_key = await _EXA_KEY_ROTATOR.get(provider_settings) + headers = { + "x-api-key": exa_key, + "Content-Type": "application/json", + } + async with aiohttp.ClientSession(trust_env=True) as session: + async with session.post( + "https://api.exa.ai/search", + json=payload, + headers=headers, + ) as response: + if response.status != 200: + reason = await response.text() + raise Exception( + f"Exa web search failed: {reason}, status: {response.status}", + ) + data = await response.json() + return [ + SearchResult( + title=item.get("title", ""), + url=item.get("url", ""), + snippet=( + item.get("text") + or (item.get("highlights") or [""])[0] + or item.get("summary", "") + ), + ) + for item in data.get("results", []) + if item.get("url") + ] + + +async def _exa_get_contents( + provider_settings: dict, + payload: dict, +) -> list[dict]: + """Call the Exa /contents endpoint and return raw result dicts.""" + exa_key = await _EXA_KEY_ROTATOR.get(provider_settings) + headers = { + "x-api-key": exa_key, + "Content-Type": "application/json", + } + async with aiohttp.ClientSession(trust_env=True) as session: + async with session.post( + "https://api.exa.ai/contents", + json=payload, + headers=headers, + ) as response: + if response.status != 200: + reason = await response.text() + raise Exception( + f"Exa get contents failed: {reason}, status: {response.status}", + ) + data = await response.json() + return data.get("results", []) + + +@builtin_tool(config=_EXA_WEB_SEARCH_TOOL_CONFIG) +@pydantic_dataclass +class ExaWebSearchTool(FunctionTool[AstrAgentContext]): + """Web search tool powered by the Exa Search API.""" + + name: str = "web_search_exa" + description: str = ( + "A web search tool powered by Exa, an AI-native search engine. " + "Supports keyword and semantic search with domain, date, and category filters." + ) + parameters: dict = Field( + default_factory=lambda: { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Required. Search query."}, + "num_results": { + "type": "integer", + "description": "Optional. Number of results to return. Default is 10.", + }, + "type": { + "type": "string", + "description": ( + 'Optional. Search type. One of "auto", "keyword", "neural". ' + 'Default is "auto".' + ), + }, + "category": { + "type": "string", + "description": ( + "Optional. Category filter. One of " + '"company", "research paper", "news", "github", ' + '"tweet", "personal site", "pdf", "linkedin profile".' + ), + }, + "include_domains": { + "type": "string", + "description": "Optional. Comma-separated domains to restrict results to.", + }, + "exclude_domains": { + "type": "string", + "description": "Optional. Comma-separated domains to exclude from results.", + }, + "start_published_date": { + "type": "string", + "description": "Optional. Start date filter in ISO 8601 format (e.g. 2024-01-01T00:00:00.000Z).", + }, + "end_published_date": { + "type": "string", + "description": "Optional. End date filter in ISO 8601 format.", + }, + }, + "required": ["query"], + } + ) + + async def call(self, context, **kwargs) -> ToolExecResult: + _, provider_settings, _ = _get_runtime(context) + if not provider_settings.get("websearch_exa_key", []): + return "Error: Exa API key is not configured in AstrBot." + + try: + num_results = int(kwargs.get("num_results", 10)) + except (TypeError, ValueError): + num_results = 10 + if num_results < 1: + num_results = 1 + + search_type = kwargs.get("type", "auto") + if search_type not in ("auto", "keyword", "neural"): + search_type = "auto" + + payload: dict = { + "query": kwargs["query"], + "numResults": num_results, + "type": search_type, + "contents": {"text": {"maxCharacters": 500}}, + } + + category = kwargs.get("category", "") + if category: + payload["category"] = category + + include_domains = str(kwargs.get("include_domains", "")).strip() + if include_domains: + payload["includeDomains"] = [ + d.strip() for d in include_domains.split(",") if d.strip() + ] + + exclude_domains = str(kwargs.get("exclude_domains", "")).strip() + if exclude_domains: + payload["excludeDomains"] = [ + d.strip() for d in exclude_domains.split(",") if d.strip() + ] + + if kwargs.get("start_published_date"): + payload["startPublishedDate"] = kwargs["start_published_date"] + if kwargs.get("end_published_date"): + payload["endPublishedDate"] = kwargs["end_published_date"] + + results = await _exa_search(provider_settings, payload) + if not results: + return "Error: Exa web search does not return any results." + return _search_result_payload(results) + + +@builtin_tool(config=_EXA_WEB_SEARCH_TOOL_CONFIG) +@pydantic_dataclass +class ExaGetContentsTool(FunctionTool[AstrAgentContext]): + """Extract full page content from URLs using the Exa Contents API.""" + + name: str = "exa_get_contents" + description: str = "Extract the content of a web page using Exa." + parameters: dict = Field( + default_factory=lambda: { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "Required. A URL to extract content from.", + }, + "max_characters": { + "type": "integer", + "description": "Optional. Maximum number of characters to return. Default is 3000.", + }, + }, + "required": ["url"], + } + ) + + async def call(self, context, **kwargs) -> ToolExecResult: + _, provider_settings, _ = _get_runtime(context) + if not provider_settings.get("websearch_exa_key", []): + return "Error: Exa API key is not configured in AstrBot." + + url = str(kwargs.get("url", "")).strip() + if not url: + return "Error: url must be a non-empty string." + + try: + max_characters = int(kwargs.get("max_characters", 3000)) + except (TypeError, ValueError): + max_characters = 3000 + results = await _exa_get_contents( + provider_settings, + { + "ids": [url], + "text": {"maxCharacters": max_characters}, + }, + ) + ret_ls = [] + for result in results: + ret_ls.append(f"URL: {result.get('url', 'No URL')}") + ret_ls.append(f"Content: {result.get('text', 'No content')}") + ret = "\n".join(ret_ls) + return ret or "Error: Exa get contents does not return any results." + + __all__ = [ "BaiduWebSearchTool", "BochaWebSearchTool", "BraveWebSearchTool", + "ExaGetContentsTool", + "ExaWebSearchTool", "TavilyExtractWebPageTool", "TavilyWebSearchTool", "WEB_SEARCH_TOOL_NAMES", diff --git a/dashboard/src/i18n/locales/en-US/features/config-metadata.json b/dashboard/src/i18n/locales/en-US/features/config-metadata.json index dad5a53a25..27c24af179 100644 --- a/dashboard/src/i18n/locales/en-US/features/config-metadata.json +++ b/dashboard/src/i18n/locales/en-US/features/config-metadata.json @@ -139,6 +139,10 @@ }, "web_search_link": { "description": "Display Source Citations" + }, + "websearch_exa_key": { + "description": "Exa API Key", + "hint": "Multiple keys can be added for rotation. Get a key at https://dashboard.exa.ai" } } }, @@ -1221,22 +1225,22 @@ "hint": "Only effective for qwen3-rerank models. Recommended to write in English." }, "nvidia_rerank_api_base": { - "description": "API Base URL" + "description": "API Base URL" }, "nvidia_rerank_api_key": { - "description": "API Key" + "description": "API Key" }, "nvidia_rerank_model": { - "description": "Rerank Model Name", - "hint": "Please refer to the NVIDIA Docs for the model name." + "description": "Rerank Model Name", + "hint": "Please refer to the NVIDIA Docs for the model name." }, "nvidia_rerank_model_endpoint": { - "description": "Custom Model Endpoint", - "hint": "Custom URL suffix endpoint, defaults to /reranking." + "description": "Custom Model Endpoint", + "hint": "Custom URL suffix endpoint, defaults to /reranking." }, "nvidia_rerank_truncate": { - "description": "Text Truncation Strategy", - "hint": "Whether to truncate the input to fit the model's maximum context length when the input text is too long." + "description": "Text Truncation Strategy", + "hint": "Whether to truncate the input to fit the model's maximum context length when the input text is too long." }, "launch_model_if_not_running": { "description": "Auto-start model if not running", diff --git a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json index a5efc78335..2c59e76261 100644 --- a/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json +++ b/dashboard/src/i18n/locales/ru-RU/features/config-metadata.json @@ -139,6 +139,10 @@ }, "web_search_link": { "description": "Показывать ссылки на источники" + }, + "websearch_exa_key": { + "description": "API-ключ Exa", + "hint": "Можно добавить несколько ключей для ротации. Получить ключ: https://dashboard.exa.ai" } } }, diff --git a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json index bcfb4e20dc..ca7fa48f7f 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json +++ b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json @@ -141,6 +141,10 @@ }, "web_search_link": { "description": "显示来源引用" + }, + "websearch_exa_key": { + "description": "Exa API Key", + "hint": "可添加多个 Key 进行轮询。获取 Key: https://dashboard.exa.ai" } } }, diff --git a/docs/en/use/websearch.md b/docs/en/use/websearch.md index b13fab546d..798df2dcaa 100644 --- a/docs/en/use/websearch.md +++ b/docs/en/use/websearch.md @@ -14,11 +14,11 @@ When using a large language model that supports function calling with the web se And other prompts with search intent to trigger the model to invoke the search tool. -AstrBot currently supports 5 web search providers: `Tavily`, `BoCha`, `Baidu AI Search`, `Brave`, and `Firecrawl`. +AstrBot currently supports 6 web search providers: `Tavily`, `BoCha`, `Baidu AI Search`, `Brave`, `Firecrawl`, and `Exa`. ![image](https://files.astrbot.app/docs/source/images/websearch/image.png) -Go to `Configuration`, scroll down to find Web Search, where you can select `Tavily`, `BoCha`, `Baidu AI Search`, `Brave`, or `Firecrawl`. +Go to `Configuration`, scroll down to find Web Search, where you can select `Tavily`, `BoCha`, `Baidu AI Search`, `Brave`, `Firecrawl`, or `Exa`. ### Tavily @@ -40,6 +40,10 @@ Get an API Key from Brave Search, then fill it in the corresponding configuratio Go to [Firecrawl](https://firecrawl.dev) to get an API Key, then fill it in the corresponding configuration item. +### Exa + +Go to [Exa](https://dashboard.exa.ai) to get an API Key, then fill it in the corresponding configuration item. Exa is an AI-native search engine that supports keyword and semantic search with category filters, domain restrictions, and date ranges. + If you use Tavily as your web search source, you will get a better experience optimization on AstrBot ChatUI, including citation source display and more: ![](https://files.astrbot.app/docs/source/images/websearch/image1.png) diff --git a/docs/zh/use/websearch.md b/docs/zh/use/websearch.md index 1cd6d33a6a..c3b7f48a42 100644 --- a/docs/zh/use/websearch.md +++ b/docs/zh/use/websearch.md @@ -13,11 +13,11 @@ AstrBot 内置的网页搜索功能依赖大模型提供 `函数调用` 能力 等等带有搜索意味的提示让大模型触发调用搜索工具。 -AstrBot 当前支持 5 种网页搜索源接入方式：`Tavily`、`BoCha`、`百度 AI 搜索`、`Brave`、`Firecrawl`。 +AstrBot 当前支持 6 种网页搜索源接入方式：`Tavily`、`BoCha`、`百度 AI 搜索`、`Brave`、`Firecrawl`、`Exa`。 ![image](https://files.astrbot.app/docs/source/images/websearch/image.png) -进入 `配置`，下拉找到网页搜索，您可选择 `Tavily`、`BoCha`、`百度 AI 搜索`、`Brave` 或 `Firecrawl`。 +进入 `配置`，下拉找到网页搜索，您可选择 `Tavily`、`BoCha`、`百度 AI 搜索`、`Brave`、`Firecrawl` 或 `Exa`。 ### Tavily @@ -39,6 +39,10 @@ AstrBot 当前支持 5 种网页搜索源接入方式：`Tavily`、`BoCha`、` 前往 [Firecrawl](https://firecrawl.dev) 获取 API Key，然后填写在相应的配置项。 +### Exa + +前往 [Exa](https://dashboard.exa.ai) 获取 API Key，然后填写在相应的配置项。Exa 是一个 AI 原生搜索引擎，支持关键词和语义搜索，提供分类过滤、域名限制和日期范围等高级搜索功能。 + 如果您使用 Tavily 作为网页搜索源，在 AstrBot ChatUI 上将会获得更好的体验优化，包括引用来源展示等： ![](https://files.astrbot.app/docs/source/images/websearch/image1.png) diff --git a/tests/unit/test_web_search_tools.py b/tests/unit/test_web_search_tools.py index c0ac3cf800..3eb96f9302 100644 --- a/tests/unit/test_web_search_tools.py +++ b/tests/unit/test_web_search_tools.py @@ -378,3 +378,138 @@ def _context_with_provider_settings(provider_settings): event=SimpleNamespace(unified_msg_origin="test:private:session"), ) return SimpleNamespace(context=agent_context) + + +# --- Exa tests --- + + +def test_normalize_legacy_web_search_config_migrates_exa_key(): + config = _FakeConfig({"provider_settings": {"websearch_exa_key": "exa-key"}}) + + tools.normalize_legacy_web_search_config(config) + + assert config["provider_settings"]["websearch_exa_key"] == ["exa-key"] + assert config.saved is True + + +@pytest.mark.asyncio +async def test_exa_search_maps_results(monkeypatch): + async def fake_exa_search(provider_settings, payload): + assert provider_settings["websearch_exa_key"] == ["exa-key"] + assert payload["query"] == "AstrBot" + assert payload["numResults"] == 5 + return [ + tools.SearchResult( + title="AstrBot", + url="https://example.com", + snippet="AI Agent Assistant", + ) + ] + + monkeypatch.setattr(tools, "_exa_search", fake_exa_search) + tool = tools.ExaWebSearchTool() + context = _context_with_provider_settings({"websearch_exa_key": ["exa-key"]}) + + result = await tool.call(context, query="AstrBot", num_results=5) + + parsed = json.loads(result) + assert parsed["results"][0]["title"] == "AstrBot" + assert parsed["results"][0]["url"] == "https://example.com" + assert parsed["results"][0]["snippet"] == "AI Agent Assistant" + + +@pytest.mark.asyncio +async def test_exa_search_raw_api_call(monkeypatch): + session = _FakeFirecrawlSession( + _FakeFirecrawlResponse( + status=200, + json_data={ + "results": [ + { + "title": "AstrBot", + "url": "https://example.com", + "text": "AI Agent Assistant", + } + ], + }, + ) + ) + + def fake_client_session(*, trust_env): + session.trust_env = trust_env + return session + + monkeypatch.setattr(tools.aiohttp, "ClientSession", fake_client_session) + + results = await tools._exa_search( + {"websearch_exa_key": ["exa-key"]}, + {"query": "AstrBot", "numResults": 10, "type": "auto"}, + ) + + assert session.posted["url"] == "https://api.exa.ai/search" + assert session.posted["headers"]["x-api-key"] == "exa-key" + assert results == [ + tools.SearchResult( + title="AstrBot", url="https://example.com", snippet="AI Agent Assistant" + ) + ] + + +@pytest.mark.asyncio +async def test_exa_search_raises_on_http_error(monkeypatch): + session = _FakeFirecrawlSession( + _FakeFirecrawlResponse(status=401, text_data="Unauthorized") + ) + + def fake_client_session(*, trust_env): + session.trust_env = trust_env + return session + + monkeypatch.setattr(tools.aiohttp, "ClientSession", fake_client_session) + + with pytest.raises( + Exception, + match="Exa web search failed: Unauthorized, status: 401", + ): + await tools._exa_search( + {"websearch_exa_key": ["exa-key"]}, + {"query": "AstrBot"}, + ) + + +@pytest.mark.asyncio +async def test_exa_get_contents_returns_text(monkeypatch): + async def fake_exa_get_contents(provider_settings, payload): + assert provider_settings["websearch_exa_key"] == ["exa-key"] + assert payload["ids"] == ["https://example.com"] + return [{"url": "https://example.com", "text": "# Example Content"}] + + monkeypatch.setattr(tools, "_exa_get_contents", fake_exa_get_contents) + tool = tools.ExaGetContentsTool() + context = _context_with_provider_settings({"websearch_exa_key": ["exa-key"]}) + + result = await tool.call(context, url="https://example.com") + + assert result == "URL: https://example.com\nContent: # Example Content" + + +@pytest.mark.asyncio +async def test_exa_get_contents_raises_on_http_error(monkeypatch): + session = _FakeFirecrawlSession( + _FakeFirecrawlResponse(status=403, text_data="Forbidden") + ) + + def fake_client_session(*, trust_env): + session.trust_env = trust_env + return session + + monkeypatch.setattr(tools.aiohttp, "ClientSession", fake_client_session) + + with pytest.raises( + Exception, + match="Exa get contents failed: Forbidden, status: 403", + ): + await tools._exa_get_contents( + {"websearch_exa_key": ["exa-key"]}, + {"ids": ["https://example.com"]}, + )