Skip to content

Commit d9d9f1b

Browse files
10ishq10ishq
committed
feat(websearch): add Exa as a web search provider
- Add ExaWebSearchTool (web_search_exa) with keyword/semantic search, category filters, domain restrictions, and date range support - Add ExaGetContentsTool (exa_get_contents) for extracting web page content - Add _exa_search() and _exa_get_contents() API helpers hitting https://api.exa.ai/search and https://api.exa.ai/contents - Add _EXA_KEY_ROTATOR for multi-key rotation - Register Exa tools in _apply_web_search_tools() dispatch - Add Exa to WEB_SEARCH_CITATION_TOOL_NAMES for citation support - Add websearch_exa_key config default and provider option - Add i18n metadata for en-US, zh-CN, ru-RU - Add Exa section to docs (en + zh) - Add 6 unit tests covering search, contents, error handling, and legacy config migration Closes #5621 Co-Authored-By: Tanishq Jaiswal <tanishq.jaiswal97@gmail.com>
1 parent 756469a commit d9d9f1b

9 files changed

Lines changed: 414 additions & 12 deletions

File tree

‎astrbot/core/astr_main_agent.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@
8585
BaiduWebSearchTool,
8686
BochaWebSearchTool,
8787
BraveWebSearchTool,
88+
ExaGetContentsTool,
89+
ExaWebSearchTool,
8890
FirecrawlExtractWebPageTool,
8991
FirecrawlWebSearchTool,
9092
TavilyExtractWebPageTool,
@@ -130,6 +132,7 @@
130132
"web_search_tavily",
131133
"web_search_bocha",
132134
"web_search_brave",
135+
"web_search_exa",
133136
}
134137
)
135138
WEB_SEARCH_CITATION_PROMPT = (
@@ -1207,6 +1210,9 @@ async def _apply_web_search_tools(
12071210
req.func_tool.add_tool(tool_mgr.get_builtin_tool(FirecrawlExtractWebPageTool))
12081211
elif provider == "baidu_ai_search":
12091212
req.func_tool.add_tool(tool_mgr.get_builtin_tool(BaiduWebSearchTool))
1213+
elif provider == "exa":
1214+
req.func_tool.add_tool(tool_mgr.get_builtin_tool(ExaWebSearchTool))
1215+
req.func_tool.add_tool(tool_mgr.get_builtin_tool(ExaGetContentsTool))
12101216

12111217

12121218
def _apply_web_search_citation_prompt(

‎astrbot/core/config/default.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@
115115
"websearch_brave_key": [],
116116
"websearch_baidu_app_builder_key": "",
117117
"websearch_firecrawl_key": [],
118+
"websearch_exa_key": [],
118119
"web_search_link": False,
119120
"display_reasoning_text": False,
120121
"identifier": False,
@@ -3295,6 +3296,7 @@
32953296
"bocha",
32963297
"brave",
32973298
"firecrawl",
3299+
"exa",
32983300
],
32993301
"condition": {
33003302
"provider_settings.web_search": True,
@@ -3349,6 +3351,16 @@
33493351
"provider_settings.web_search": True,
33503352
},
33513353
},
3354+
"provider_settings.websearch_exa_key": {
3355+
"description": "Exa API Key",
3356+
"type": "list",
3357+
"items": {"type": "string"},
3358+
"hint": "可添加多个 Key 进行轮询。Get a key at https://dashboard.exa.ai",
3359+
"condition": {
3360+
"provider_settings.websearch_provider": "exa",
3361+
"provider_settings.web_search": True,
3362+
},
3363+
},
33523364
"provider_settings.web_search_link": {
33533365
"description": "显示来源引用",
33543366
"type": "bool",

‎astrbot/core/tools/web_search_tools.py‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
"web_search_brave",
2222
"web_search_firecrawl",
2323
"firecrawl_extract_web_page",
24+
"web_search_exa",
25+
"exa_get_contents",
2426
]
2527
_TAVILY_WEB_SEARCH_TOOL_CONFIG = {
2628
"provider_settings.web_search": True,
@@ -42,6 +44,10 @@
4244
"provider_settings.web_search": True,
4345
"provider_settings.websearch_provider": "baidu_ai_search",
4446
}
47+
_EXA_WEB_SEARCH_TOOL_CONFIG = {
48+
"provider_settings.web_search": True,
49+
"provider_settings.websearch_provider": "exa",
50+
}
4551

4652

4753
@std_dataclass
@@ -76,6 +82,7 @@ async def get(self, provider_settings: dict) -> str:
7682
_BOCHA_KEY_ROTATOR = _KeyRotator("websearch_bocha_key", "BoCha")
7783
_BRAVE_KEY_ROTATOR = _KeyRotator("websearch_brave_key", "Brave")
7884
_FIRECRAWL_KEY_ROTATOR = _KeyRotator("websearch_firecrawl_key", "Firecrawl")
85+
_EXA_KEY_ROTATOR = _KeyRotator("websearch_exa_key", "Exa")
7986

8087

8188
def normalize_legacy_web_search_config(cfg) -> None:
@@ -99,6 +106,7 @@ def normalize_legacy_web_search_config(cfg) -> None:
99106
"websearch_bocha_key",
100107
"websearch_brave_key",
101108
"websearch_firecrawl_key",
109+
"websearch_exa_key",
102110
):
103111
value = provider_settings.get(setting_name)
104112
if isinstance(value, str):
@@ -803,10 +811,231 @@ async def call(self, context, **kwargs) -> ToolExecResult:
803811
return _search_result_payload(results)
804812

805813

814+
async def _exa_search(
815+
provider_settings: dict,
816+
payload: dict,
817+
) -> list[SearchResult]:
818+
"""Call the Exa /search endpoint and return normalized results."""
819+
exa_key = await _EXA_KEY_ROTATOR.get(provider_settings)
820+
headers = {
821+
"x-api-key": exa_key,
822+
"Content-Type": "application/json",
823+
}
824+
async with aiohttp.ClientSession(trust_env=True) as session:
825+
async with session.post(
826+
"https://api.exa.ai/search",
827+
json=payload,
828+
headers=headers,
829+
) as response:
830+
if response.status != 200:
831+
reason = await response.text()
832+
raise Exception(
833+
f"Exa web search failed: {reason}, status: {response.status}",
834+
)
835+
data = await response.json()
836+
return [
837+
SearchResult(
838+
title=item.get("title", ""),
839+
url=item.get("url", ""),
840+
snippet=(
841+
item.get("text")
842+
or (item.get("highlights") or [""])[0]
843+
or item.get("summary", "")
844+
),
845+
)
846+
for item in data.get("results", [])
847+
if item.get("url")
848+
]
849+
850+
851+
async def _exa_get_contents(
852+
provider_settings: dict,
853+
payload: dict,
854+
) -> list[dict]:
855+
"""Call the Exa /contents endpoint and return raw result dicts."""
856+
exa_key = await _EXA_KEY_ROTATOR.get(provider_settings)
857+
headers = {
858+
"x-api-key": exa_key,
859+
"Content-Type": "application/json",
860+
}
861+
async with aiohttp.ClientSession(trust_env=True) as session:
862+
async with session.post(
863+
"https://api.exa.ai/contents",
864+
json=payload,
865+
headers=headers,
866+
) as response:
867+
if response.status != 200:
868+
reason = await response.text()
869+
raise Exception(
870+
f"Exa get contents failed: {reason}, status: {response.status}",
871+
)
872+
data = await response.json()
873+
return data.get("results", [])
874+
875+
876+
@builtin_tool(config=_EXA_WEB_SEARCH_TOOL_CONFIG)
877+
@pydantic_dataclass
878+
class ExaWebSearchTool(FunctionTool[AstrAgentContext]):
879+
"""Web search tool powered by the Exa Search API."""
880+
881+
name: str = "web_search_exa"
882+
description: str = (
883+
"A web search tool powered by Exa, an AI-native search engine. "
884+
"Supports keyword and semantic search with domain, date, and category filters."
885+
)
886+
parameters: dict = Field(
887+
default_factory=lambda: {
888+
"type": "object",
889+
"properties": {
890+
"query": {"type": "string", "description": "Required. Search query."},
891+
"num_results": {
892+
"type": "integer",
893+
"description": "Optional. Number of results to return. Default is 10.",
894+
},
895+
"type": {
896+
"type": "string",
897+
"description": (
898+
'Optional. Search type. One of "auto", "keyword", "neural". '
899+
'Default is "auto".'
900+
),
901+
},
902+
"category": {
903+
"type": "string",
904+
"description": (
905+
"Optional. Category filter. One of "
906+
'"company", "research paper", "news", "github", '
907+
'"tweet", "personal site", "pdf", "linkedin profile".'
908+
),
909+
},
910+
"include_domains": {
911+
"type": "string",
912+
"description": "Optional. Comma-separated domains to restrict results to.",
913+
},
914+
"exclude_domains": {
915+
"type": "string",
916+
"description": "Optional. Comma-separated domains to exclude from results.",
917+
},
918+
"start_published_date": {
919+
"type": "string",
920+
"description": "Optional. Start date filter in ISO 8601 format (e.g. 2024-01-01T00:00:00.000Z).",
921+
},
922+
"end_published_date": {
923+
"type": "string",
924+
"description": "Optional. End date filter in ISO 8601 format.",
925+
},
926+
},
927+
"required": ["query"],
928+
}
929+
)
930+
931+
async def call(self, context, **kwargs) -> ToolExecResult:
932+
_, provider_settings, _ = _get_runtime(context)
933+
if not provider_settings.get("websearch_exa_key", []):
934+
return "Error: Exa API key is not configured in AstrBot."
935+
936+
try:
937+
num_results = int(kwargs.get("num_results", 10))
938+
except (TypeError, ValueError):
939+
num_results = 10
940+
if num_results < 1:
941+
num_results = 1
942+
943+
search_type = kwargs.get("type", "auto")
944+
if search_type not in ("auto", "keyword", "neural"):
945+
search_type = "auto"
946+
947+
payload: dict = {
948+
"query": kwargs["query"],
949+
"numResults": num_results,
950+
"type": search_type,
951+
"contents": {"text": {"maxCharacters": 500}},
952+
}
953+
954+
category = kwargs.get("category", "")
955+
if category:
956+
payload["category"] = category
957+
958+
include_domains = str(kwargs.get("include_domains", "")).strip()
959+
if include_domains:
960+
payload["includeDomains"] = [
961+
d.strip() for d in include_domains.split(",") if d.strip()
962+
]
963+
964+
exclude_domains = str(kwargs.get("exclude_domains", "")).strip()
965+
if exclude_domains:
966+
payload["excludeDomains"] = [
967+
d.strip() for d in exclude_domains.split(",") if d.strip()
968+
]
969+
970+
if kwargs.get("start_published_date"):
971+
payload["startPublishedDate"] = kwargs["start_published_date"]
972+
if kwargs.get("end_published_date"):
973+
payload["endPublishedDate"] = kwargs["end_published_date"]
974+
975+
results = await _exa_search(provider_settings, payload)
976+
if not results:
977+
return "Error: Exa web search does not return any results."
978+
return _search_result_payload(results)
979+
980+
981+
@builtin_tool(config=_EXA_WEB_SEARCH_TOOL_CONFIG)
982+
@pydantic_dataclass
983+
class ExaGetContentsTool(FunctionTool[AstrAgentContext]):
984+
"""Extract full page content from URLs using the Exa Contents API."""
985+
986+
name: str = "exa_get_contents"
987+
description: str = "Extract the content of a web page using Exa."
988+
parameters: dict = Field(
989+
default_factory=lambda: {
990+
"type": "object",
991+
"properties": {
992+
"url": {
993+
"type": "string",
994+
"description": "Required. A URL to extract content from.",
995+
},
996+
"max_characters": {
997+
"type": "integer",
998+
"description": "Optional. Maximum number of characters to return. Default is 3000.",
999+
},
1000+
},
1001+
"required": ["url"],
1002+
}
1003+
)
1004+
1005+
async def call(self, context, **kwargs) -> ToolExecResult:
1006+
_, provider_settings, _ = _get_runtime(context)
1007+
if not provider_settings.get("websearch_exa_key", []):
1008+
return "Error: Exa API key is not configured in AstrBot."
1009+
1010+
url = str(kwargs.get("url", "")).strip()
1011+
if not url:
1012+
return "Error: url must be a non-empty string."
1013+
1014+
try:
1015+
max_characters = int(kwargs.get("max_characters", 3000))
1016+
except (TypeError, ValueError):
1017+
max_characters = 3000
1018+
results = await _exa_get_contents(
1019+
provider_settings,
1020+
{
1021+
"ids": [url],
1022+
"text": {"maxCharacters": max_characters},
1023+
},
1024+
)
1025+
ret_ls = []
1026+
for result in results:
1027+
ret_ls.append(f"URL: {result.get('url', 'No URL')}")
1028+
ret_ls.append(f"Content: {result.get('text', 'No content')}")
1029+
ret = "\n".join(ret_ls)
1030+
return ret or "Error: Exa get contents does not return any results."
1031+
1032+
8061033
__all__ = [
8071034
"BaiduWebSearchTool",
8081035
"BochaWebSearchTool",
8091036
"BraveWebSearchTool",
1037+
"ExaGetContentsTool",
1038+
"ExaWebSearchTool",
8101039
"TavilyExtractWebPageTool",
8111040
"TavilyWebSearchTool",
8121041
"WEB_SEARCH_TOOL_NAMES",

‎dashboard/src/i18n/locales/en-US/features/config-metadata.json‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,10 @@
139139
},
140140
"web_search_link": {
141141
"description": "Display Source Citations"
142+
},
143+
"websearch_exa_key": {
144+
"description": "Exa API Key",
145+
"hint": "Multiple keys can be added for rotation. Get a key at https://dashboard.exa.ai"
142146
}
143147
}
144148
},
@@ -1221,22 +1225,22 @@
12211225
"hint": "Only effective for qwen3-rerank models. Recommended to write in English."
12221226
},
12231227
"nvidia_rerank_api_base": {
1224-
"description": "API Base URL"
1228+
"description": "API Base URL"
12251229
},
12261230
"nvidia_rerank_api_key": {
1227-
"description": "API Key"
1231+
"description": "API Key"
12281232
},
12291233
"nvidia_rerank_model": {
1230-
"description": "Rerank Model Name",
1231-
"hint": "Please refer to the NVIDIA Docs for the model name."
1234+
"description": "Rerank Model Name",
1235+
"hint": "Please refer to the NVIDIA Docs for the model name."
12321236
},
12331237
"nvidia_rerank_model_endpoint": {
1234-
"description": "Custom Model Endpoint",
1235-
"hint": "Custom URL suffix endpoint, defaults to /reranking."
1238+
"description": "Custom Model Endpoint",
1239+
"hint": "Custom URL suffix endpoint, defaults to /reranking."
12361240
},
12371241
"nvidia_rerank_truncate": {
1238-
"description": "Text Truncation Strategy",
1239-
"hint": "Whether to truncate the input to fit the model's maximum context length when the input text is too long."
1242+
"description": "Text Truncation Strategy",
1243+
"hint": "Whether to truncate the input to fit the model's maximum context length when the input text is too long."
12401244
},
12411245
"launch_model_if_not_running": {
12421246
"description": "Auto-start model if not running",

‎dashboard/src/i18n/locales/ru-RU/features/config-metadata.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,10 @@
139139
},
140140
"web_search_link": {
141141
"description": "Показывать ссылки на источники"
142+
},
143+
"websearch_exa_key": {
144+
"description": "API-ключ Exa",
145+
"hint": "Можно добавить несколько ключей для ротации. Получить ключ: https://dashboard.exa.ai"
142146
}
143147
}
144148
},

‎dashboard/src/i18n/locales/zh-CN/features/config-metadata.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,10 @@
141141
},
142142
"web_search_link": {
143143
"description": "显示来源引用"
144+
},
145+
"websearch_exa_key": {
146+
"description": "Exa API Key",
147+
"hint": "可添加多个 Key 进行轮询。获取 Key: https://dashboard.exa.ai"
144148
}
145149
}
146150
},

0 commit comments

Comments
 (0)