diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1ef0e94920..4362b1026d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,8 +16,8 @@ updates: browsergym-related: patterns: - "browsergym*" - - "browser-use" - "playwright" + - "uv" search-tools: patterns: - "googlesearch-python" @@ -40,8 +40,8 @@ updates: - "fastapi" - "tiktoken" - "browsergym*" - - "browser-use" - "playwright" + - "uv" - "googlesearch-python" - "baidusearch" - "duckduckgo_search" diff --git a/README.md b/README.md index d4a84374ff..a2b64c9cc8 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ English | [中文](README_zh.md) | [한국어](README_ko.md) | [日本語](READM Manus is incredible, but OpenManus can achieve any idea without an *Invite Code* 🛫! -Our team members [@Xinbin Liang](https://github.com/mannaandpoem) and [@Jinyu Xiang](https://github.com/XiangJinyu) (core authors), along with [@Zhaoyang Yu](https://github.com/MoshiQAQ), [@Jiayi Zhang](https://github.com/didiforgithub), and [@Sirui Hong](https://github.com/stellaHSR), we are from [@MetaGPT](https://github.com/geekan/MetaGPT). The prototype is launched within 3 hours and we are keeping building! +Our team members [@Xinbing Liang](https://github.com/mannaandpoem) and [@Jinyu Xiang](https://github.com/XiangJinyu) (core authors), along with [@Zhaoyang Yu](https://github.com/MoshiQAQ), [@Jiayi Zhang](https://github.com/didiforgithub), and [@Sirui Hong](https://github.com/stellaHSR), we are from [@MetaGPT](https://github.com/geekan/MetaGPT). The prototype is launched within 3 hours and we are keeping building! It's a simple implementation, so we welcome any suggestions, contributions, and feedback! @@ -83,7 +83,39 @@ source .venv/bin/activate # On Unix/macOS uv pip install -r requirements.txt ``` -### Browser Automation Tool (Optional) +### Browser automation + +OpenManus starts Browser Use CLI 3.0 as a default MCP server: + +```bash +uvx browser-use --cli-mcp +``` + +`uvx` keeps Browser Use and its fast-moving dependencies isolated from the +OpenManus environment. The agent receives the canonical Browser Use skill and +the native `browser_exec` and `browser_screenshot` tools. + +Local mode attaches to Chrome or Chromium automatically and needs no API key. +For diagnostics or to install Chromium, run: + +```bash +uvx browser-use --doctor +uvx browser-use install +``` + +For an isolated Browser Use Cloud browser, authenticate before starting +OpenManus. The agent can then start and select a named remote browser: + +```bash +export BROWSER_USE_API_KEY="bu_..." +``` + +Existing browsers can be selected with `BU_CDP_URL`, `BU_CDP_WS`, or `BU_NAME`. +Set `OPENMANUS_DISABLE_BROWSER_USE=1` to disable the default Browser Use MCP +server. + +BrowserGym still requires its Playwright browser: + ```bash playwright install ``` @@ -185,7 +217,7 @@ OpenManus is built by contributors from MetaGPT. Huge thanks to this agent commu ## Cite ```bibtex @misc{openmanus2025, - author = {Xinbin Liang and Jinyu Xiang and Zhaoyang Yu and Jiayi Zhang and Sirui Hong and Sheng Fan and Xiao Tang and Bang Liu and Yuyu Luo and Chenglin Wu}, + author = {Xinbing Liang and Jinyu Xiang and Zhaoyang Yu and Jiayi Zhang and Sirui Hong and Sheng Fan and Xiao Tang and Bang Liu and Yuyu Luo and Chenglin Wu}, title = {OpenManus: An open-source framework for building general AI agents}, year = {2025}, publisher = {Zenodo}, diff --git a/README_ja.md b/README_ja.md index d5fd281e55..f723541991 100644 --- a/README_ja.md +++ b/README_ja.md @@ -15,7 +15,7 @@ Manusは素晴らしいですが、OpenManusは*招待コード*なしでどんなアイデアも実現できます！🛫 -私たちのチームメンバー [@Xinbin Liang](https://github.com/mannaandpoem) と [@Jinyu Xiang](https://github.com/XiangJinyu)（主要開発者）、そして [@Zhaoyang Yu](https://github.com/MoshiQAQ)、[@Jiayi Zhang](https://github.com/didiforgithub)、[@Sirui Hong](https://github.com/stellaHSR) は [@MetaGPT](https://github.com/geekan/MetaGPT) から来ました。プロトタイプは3時間以内に立ち上げられ、継続的に開発を進めています！ +私たちのチームメンバー [@Xinbing Liang](https://github.com/mannaandpoem) と [@Jinyu Xiang](https://github.com/XiangJinyu)（主要開発者）、そして [@Zhaoyang Yu](https://github.com/MoshiQAQ)、[@Jiayi Zhang](https://github.com/didiforgithub)、[@Sirui Hong](https://github.com/stellaHSR) は [@MetaGPT](https://github.com/geekan/MetaGPT) から来ました。プロトタイプは3時間以内に立ち上げられ、継続的に開発を進めています！ これはシンプルな実装ですので、どんな提案、貢献、フィードバックも歓迎します！ @@ -183,7 +183,7 @@ OpenManusはMetaGPTのコントリビューターによって構築されまし ## 引用 ```bibtex @misc{openmanus2025, - author = {Xinbin Liang and Jinyu Xiang and Zhaoyang Yu and Jiayi Zhang and Sirui Hong and Sheng Fan and Xiao Tang}, + author = {Xinbing Liang and Jinyu Xiang and Zhaoyang Yu and Jiayi Zhang and Sirui Hong and Sheng Fan and Xiao Tang}, title = {OpenManus: An open-source framework for building general AI agents}, year = {2025}, publisher = {Zenodo}, diff --git a/README_ko.md b/README_ko.md index 5080f43e76..816ddff60b 100644 --- a/README_ko.md +++ b/README_ko.md @@ -15,7 +15,7 @@ Manus는 놀라운 도구지만, OpenManus는 *초대 코드* 없이도 모든 아이디어를 실현할 수 있습니다! 🛫 -우리 팀의 멤버인 [@Xinbin Liang](https://github.com/mannaandpoem)와 [@Jinyu Xiang](https://github.com/XiangJinyu) (핵심 작성자), 그리고 [@Zhaoyang Yu](https://github.com/MoshiQAQ), [@Jiayi Zhang](https://github.com/didiforgithub), [@Sirui Hong](https://github.com/stellaHSR)이 함께 했습니다. 우리는 [@MetaGPT](https://github.com/geekan/MetaGPT)로부터 왔습니다. 프로토타입은 단 3시간 만에 출시되었으며, 계속해서 발전하고 있습니다! +우리 팀의 멤버인 [@Xinbing Liang](https://github.com/mannaandpoem)와 [@Jinyu Xiang](https://github.com/XiangJinyu) (핵심 작성자), 그리고 [@Zhaoyang Yu](https://github.com/MoshiQAQ), [@Jiayi Zhang](https://github.com/didiforgithub), [@Sirui Hong](https://github.com/stellaHSR)이 함께 했습니다. 우리는 [@MetaGPT](https://github.com/geekan/MetaGPT)로부터 왔습니다. 프로토타입은 단 3시간 만에 출시되었으며, 계속해서 발전하고 있습니다! 이 프로젝트는 간단한 구현에서 시작되었으며, 여러분의 제안, 기여 및 피드백을 환영합니다! @@ -182,7 +182,7 @@ OpenManus는 MetaGPT 기여자들에 의해 개발되었습니다. 이 에이전 ## 인용 ```bibtex @misc{openmanus2025, - author = {Xinbin Liang and Jinyu Xiang and Zhaoyang Yu and Jiayi Zhang and Sirui Hong and Sheng Fan and Xiao Tang}, + author = {Xinbing Liang and Jinyu Xiang and Zhaoyang Yu and Jiayi Zhang and Sirui Hong and Sheng Fan and Xiao Tang}, title = {OpenManus: An open-source framework for building general AI agents}, year = {2025}, publisher = {Zenodo}, diff --git a/README_zh.md b/README_zh.md index 680ccc1d98..58b9d13c99 100644 --- a/README_zh.md +++ b/README_zh.md @@ -15,7 +15,7 @@ Manus 非常棒，但 OpenManus 无需邀请码即可实现任何创意 🛫！ -我们的团队成员 [@Xinbin Liang](https://github.com/mannaandpoem) 和 [@Jinyu Xiang](https://github.com/XiangJinyu)（核心作者），以及 [@Zhaoyang Yu](https://github.com/MoshiQAQ)、[@Jiayi Zhang](https://github.com/didiforgithub) 和 [@Sirui Hong](https://github.com/stellaHSR)，来自 [@MetaGPT](https://github.com/geekan/MetaGPT)团队。我们在 3 +我们的团队成员 [@Xinbing Liang](https://github.com/mannaandpoem) 和 [@Jinyu Xiang](https://github.com/XiangJinyu)（核心作者），以及 [@Zhaoyang Yu](https://github.com/MoshiQAQ)、[@Jiayi Zhang](https://github.com/didiforgithub) 和 [@Sirui Hong](https://github.com/stellaHSR)，来自 [@MetaGPT](https://github.com/geekan/MetaGPT)团队。我们在 3 小时内完成了开发并持续迭代中！ 这是一个简洁的实现方案，欢迎任何建议、贡献和反馈！ @@ -188,7 +188,7 @@ OpenManus 由 MetaGPT 社区的贡献者共同构建，感谢这个充满活力 ## 引用 ```bibtex @misc{openmanus2025, - author = {Xinbin Liang and Jinyu Xiang and Zhaoyang Yu and Jiayi Zhang and Sirui Hong and Sheng Fan and Xiao Tang}, + author = {Xinbing Liang and Jinyu Xiang and Zhaoyang Yu and Jiayi Zhang and Sirui Hong and Sheng Fan and Xiao Tang}, title = {OpenManus: An open-source framework for building general AI agents}, year = {2025}, publisher = {Zenodo}, diff --git a/app/agent/browser.py b/app/agent/browser.py index 3f25c45393..f840d6719a 100644 --- a/app/agent/browser.py +++ b/app/agent/browser.py @@ -1,14 +1,11 @@ import json from typing import TYPE_CHECKING, Optional -from pydantic import Field, model_validator - -from app.agent.toolcall import ToolCallAgent +from app.agent.mcp import MCPAgent from app.logger import logger -from app.prompt.browser import NEXT_STEP_PROMPT, SYSTEM_PROMPT -from app.schema import Message, ToolChoice -from app.tool import BrowserUseTool, Terminate, ToolCollection -from app.tool.sandbox.sb_browser_tool import SandboxBrowserTool +from app.prompt.browser import NEXT_STEP_PROMPT +from app.schema import Message +from app.tool.terminate import Terminate # Avoid circular import if BrowserAgent needs BrowserContextHelper @@ -16,19 +13,18 @@ from app.agent.base import BaseAgent # Or wherever memory is defined +_SANDBOX_BROWSER_TOOL_NAME = "sandbox_browser" + + class BrowserContextHelper: def __init__(self, agent: "BaseAgent"): self.agent = agent self._current_base64_image: Optional[str] = None async def get_browser_state(self) -> Optional[dict]: - browser_tool = self.agent.available_tools.get_tool(BrowserUseTool().name) - if not browser_tool: - browser_tool = self.agent.available_tools.get_tool( - SandboxBrowserTool().name - ) + browser_tool = self.agent.available_tools.get_tool(_SANDBOX_BROWSER_TOOL_NAME) if not browser_tool or not hasattr(browser_tool, "get_current_state"): - logger.warning("BrowserUseTool not found or doesn't have get_current_state") + logger.warning("SandboxBrowserTool not found or has no current state") return None try: result = await browser_tool.get_current_state() @@ -79,51 +75,40 @@ async def format_next_step_prompt(self) -> str: ) async def cleanup_browser(self): - browser_tool = self.agent.available_tools.get_tool(BrowserUseTool().name) + browser_tool = self.agent.available_tools.get_tool(_SANDBOX_BROWSER_TOOL_NAME) if browser_tool and hasattr(browser_tool, "cleanup"): await browser_tool.cleanup() -class BrowserAgent(ToolCallAgent): - """ - A browser agent that uses the browser_use library to control a browser. - - This agent can navigate web pages, interact with elements, fill forms, - extract content, and perform other browser-based actions to accomplish tasks. - """ +class BrowserAgent(MCPAgent): + """Browser-only agent backed by the Browser Use CLI 3.0 MCP server.""" name: str = "browser" description: str = "A browser agent that can control a browser to accomplish tasks" - - system_prompt: str = SYSTEM_PROMPT - next_step_prompt: str = NEXT_STEP_PROMPT - - max_observe: int = 10000 - max_steps: int = 20 - - # Configure the available tools - available_tools: ToolCollection = Field( - default_factory=lambda: ToolCollection(BrowserUseTool(), Terminate()) - ) - - # Use Auto for tool choice to allow both tool usage and free-form responses - tool_choices: ToolChoice = ToolChoice.AUTO - special_tool_names: list[str] = Field(default_factory=lambda: [Terminate().name]) - - browser_context_helper: Optional[BrowserContextHelper] = None - - @model_validator(mode="after") - def initialize_helper(self) -> "BrowserAgent": - self.browser_context_helper = BrowserContextHelper(self) - return self - - async def think(self) -> bool: - """Process current state and decide next actions using tools, with browser state info added""" - self.next_step_prompt = ( - await self.browser_context_helper.format_next_step_prompt() + system_prompt: str = """\ +Use Browser Use CLI 3.0 through `browser_exec`. Pass the Python body from CLI +examples as the `code` argument. Use `browser_screenshot` when visual context +is needed. The browser-harness session persists across calls. Use `terminate` +when the task is complete. +""" + + async def initialize(self) -> None: + await super().initialize( + connection_type="stdio", + command="uvx", + args=["browser-use", "--cli-mcp"], + server_id="browser_use", + tool_name_prefix=False, ) - return await super().think() - - async def cleanup(self): - """Clean up browser agent resources by calling parent cleanup.""" - await self.browser_context_helper.cleanup_browser() + self.available_tools.add_tool(Terminate()) + + @classmethod + async def create(cls, **kwargs) -> "BrowserAgent": + instance = cls(**kwargs) + await instance.initialize() + return instance + + async def run(self, request: Optional[str] = None) -> str: + if not self.mcp_clients.sessions: + await self.initialize() + return await super().run(request) diff --git a/app/agent/manus.py b/app/agent/manus.py index df40edbbaa..94246abd2e 100644 --- a/app/agent/manus.py +++ b/app/agent/manus.py @@ -1,20 +1,43 @@ +import os from typing import Dict, List, Optional -from pydantic import Field, model_validator +from pydantic import Field -from app.agent.browser import BrowserContextHelper from app.agent.toolcall import ToolCallAgent from app.config import config from app.logger import logger from app.prompt.manus import NEXT_STEP_PROMPT, SYSTEM_PROMPT +from app.schema import Message from app.tool import Terminate, ToolCollection from app.tool.ask_human import AskHuman -from app.tool.browser_use_tool import BrowserUseTool from app.tool.mcp import MCPClients, MCPClientTool from app.tool.python_execute import PythonExecute from app.tool.str_replace_editor import StrReplaceEditor +_BROWSER_USE_SERVER_ID = "browser_use" +_BROWSER_USE_COMMAND = "uvx" +_BROWSER_USE_ARGS = ["browser-use", "--cli-mcp"] +_BROWSER_USE_ENV_VARS = ( + "BROWSER_USE_API_KEY", + "BROWSER_USE_CLOUD_API_URL", + "BU_BROWSER_ID", + "BU_CDP_URL", + "BU_CDP_WS", + "BU_NAME", +) +_BROWSER_USE_TRANSPORT_INSTRUCTIONS = """\ +Browser Use CLI 3.0 is exposed here as MCP tools. When the Browser Use skill +shows `browser-use <<'PY'`, pass the Python body to `browser_exec` instead. +Use `browser_screenshot` when visual inspection is needed. Both tools use the +same persistent browser-harness session as CLI 3.0. +""" + + +def _browser_use_env() -> Dict[str, str]: + return {name: value for name in _BROWSER_USE_ENV_VARS if (value := os.getenv(name))} + + class Manus(ToolCallAgent): """A versatile general-purpose agent with support for both local and MCP tools.""" @@ -34,7 +57,6 @@ class Manus(ToolCallAgent): available_tools: ToolCollection = Field( default_factory=lambda: ToolCollection( PythonExecute(), - BrowserUseTool(), StrReplaceEditor(), AskHuman(), Terminate(), @@ -42,20 +64,14 @@ class Manus(ToolCallAgent): ) special_tool_names: list[str] = Field(default_factory=lambda: [Terminate().name]) - browser_context_helper: Optional[BrowserContextHelper] = None # Track connected MCP servers connected_servers: Dict[str, str] = Field( default_factory=dict ) # server_id -> url/command + mcp_instruction_servers: set[str] = Field(default_factory=set, exclude=True) _initialized: bool = False - @model_validator(mode="after") - def initialize_helper(self) -> "Manus": - """Initialize basic components synchronously.""" - self.browser_context_helper = BrowserContextHelper(self) - return self - @classmethod async def create(cls, **kwargs) -> "Manus": """Factory method to create and properly initialize a Manus instance.""" @@ -66,6 +82,22 @@ async def create(cls, **kwargs) -> "Manus": async def initialize_mcp_servers(self) -> None: """Initialize connections to configured MCP servers.""" + if _BROWSER_USE_SERVER_ID not in config.mcp_config.servers and os.getenv( + "OPENMANUS_DISABLE_BROWSER_USE", "" + ).lower() not in {"1", "true", "yes"}: + try: + await self.connect_mcp_server( + _BROWSER_USE_COMMAND, + _BROWSER_USE_SERVER_ID, + use_stdio=True, + stdio_args=_BROWSER_USE_ARGS, + tool_name_prefix=False, + stdio_env=_browser_use_env(), + ) + logger.info("Connected to Browser Use CLI 3.0 through MCP") + except Exception as e: + logger.error(f"Failed to connect to Browser Use CLI 3.0: {e}") + for server_id, server_config in config.mcp_config.servers.items(): try: if server_config.type == "sse": @@ -81,6 +113,12 @@ async def initialize_mcp_servers(self) -> None: server_id, use_stdio=True, stdio_args=server_config.args, + tool_name_prefix=server_id != _BROWSER_USE_SERVER_ID, + stdio_env=( + _browser_use_env() + if server_id == _BROWSER_USE_SERVER_ID + else None + ), ) logger.info( f"Connected to MCP server {server_id} using command {server_config.command}" @@ -93,12 +131,18 @@ async def connect_mcp_server( server_url: str, server_id: str = "", use_stdio: bool = False, - stdio_args: List[str] = None, + stdio_args: Optional[List[str]] = None, + tool_name_prefix: bool = True, + stdio_env: Optional[Dict[str, str]] = None, ) -> None: """Connect to an MCP server and add its tools.""" if use_stdio: await self.mcp_clients.connect_stdio( - server_url, stdio_args or [], server_id + server_url, + stdio_args or [], + server_id, + tool_name_prefix=tool_name_prefix, + env=stdio_env, ) self.connected_servers[server_id or server_url] = server_url else: @@ -111,6 +155,21 @@ async def connect_mcp_server( ] self.available_tools.add_tools(*new_tools) + resolved_server_id = server_id or server_url + instructions = self.mcp_clients.server_instructions.get(resolved_server_id) + if instructions and resolved_server_id not in self.mcp_instruction_servers: + transport_instructions = ( + f"{_BROWSER_USE_TRANSPORT_INSTRUCTIONS}\n" + if resolved_server_id == _BROWSER_USE_SERVER_ID + else "" + ) + self.memory.add_message( + Message.system_message( + f"{transport_instructions}MCP server instructions:\n{instructions}" + ) + ) + self.mcp_instruction_servers.add(resolved_server_id) + async def disconnect_mcp_server(self, server_id: str = "") -> None: """Disconnect from an MCP server and remove its tools.""" await self.mcp_clients.disconnect(server_id) @@ -130,8 +189,6 @@ async def disconnect_mcp_server(self, server_id: str = "") -> None: async def cleanup(self): """Clean up Manus agent resources.""" - if self.browser_context_helper: - await self.browser_context_helper.cleanup_browser() # Disconnect from all MCP servers only if we were initialized if self._initialized: await self.disconnect_mcp_server() @@ -143,23 +200,4 @@ async def think(self) -> bool: await self.initialize_mcp_servers() self._initialized = True - original_prompt = self.next_step_prompt - recent_messages = self.memory.messages[-3:] if self.memory.messages else [] - browser_in_use = any( - tc.function.name == BrowserUseTool().name - for msg in recent_messages - if msg.tool_calls - for tc in msg.tool_calls - ) - - if browser_in_use: - self.next_step_prompt = ( - await self.browser_context_helper.format_next_step_prompt() - ) - - result = await super().think() - - # Restore original prompt - self.next_step_prompt = original_prompt - - return result + return await super().think() diff --git a/app/agent/mcp.py b/app/agent/mcp.py index 9c6da6aaa3..92fbbf7d67 100644 --- a/app/agent/mcp.py +++ b/app/agent/mcp.py @@ -4,9 +4,8 @@ from app.agent.toolcall import ToolCallAgent from app.logger import logger -from app.prompt.mcp import MULTIMEDIA_RESPONSE_PROMPT, NEXT_STEP_PROMPT, SYSTEM_PROMPT +from app.prompt.mcp import NEXT_STEP_PROMPT, SYSTEM_PROMPT from app.schema import AgentState, Message -from app.tool.base import ToolResult from app.tool.mcp import MCPClients @@ -43,6 +42,8 @@ async def initialize( server_url: Optional[str] = None, command: Optional[str] = None, args: Optional[List[str]] = None, + server_id: str = "", + tool_name_prefix: bool = True, ) -> None: """Initialize the MCP connection. @@ -59,11 +60,18 @@ async def initialize( if self.connection_type == "sse": if not server_url: raise ValueError("Server URL is required for SSE connection") - await self.mcp_clients.connect_sse(server_url=server_url) + await self.mcp_clients.connect_sse( + server_url=server_url, server_id=server_id + ) elif self.connection_type == "stdio": if not command: raise ValueError("Command is required for stdio connection") - await self.mcp_clients.connect_stdio(command=command, args=args or []) + await self.mcp_clients.connect_stdio( + command=command, + args=args or [], + server_id=server_id, + tool_name_prefix=tool_name_prefix, + ) else: raise ValueError(f"Unsupported connection type: {self.connection_type}") @@ -76,11 +84,18 @@ async def initialize( # Add system message about available tools tool_names = list(self.mcp_clients.tool_map.keys()) tools_info = ", ".join(tool_names) + resolved_server_id = server_id or command or server_url or "" + instructions = self.mcp_clients.server_instructions.get(resolved_server_id, "") # Add system prompt and available tools information self.memory.add_message( Message.system_message( f"{self.system_prompt}\n\nAvailable MCP tools: {tools_info}" + + ( + f"\n\nMCP server instructions:\n{instructions}" + if instructions + else "" + ) ) ) @@ -151,19 +166,6 @@ async def think(self) -> bool: # Use the parent class's think method return await super().think() - async def _handle_special_tool(self, name: str, result: Any, **kwargs) -> None: - """Handle special tool execution and state changes""" - # First process with parent handler - await super()._handle_special_tool(name, result, **kwargs) - - # Handle multimedia responses - if isinstance(result, ToolResult) and result.base64_image: - self.memory.add_message( - Message.system_message( - MULTIMEDIA_RESPONSE_PROMPT.format(tool_name=name) - ) - ) - def _should_finish_execution(self, name: str, **kwargs) -> bool: """Determine if tool execution should finish the agent""" # Terminate if the tool name is 'terminate' diff --git a/app/agent/toolcall.py b/app/agent/toolcall.py index 65f31d9889..f8b85429c8 100644 --- a/app/agent/toolcall.py +++ b/app/agent/toolcall.py @@ -138,6 +138,7 @@ async def act(self) -> str: return self.messages[-1].content or "No content or commands to execute" results = [] + image_messages = [] for command in self.tool_calls: # Reset base64_image for each tool call self._current_base64_image = None @@ -156,11 +157,18 @@ async def act(self) -> str: content=result, tool_call_id=command.id, name=command.function.name, - base64_image=self._current_base64_image, ) self.memory.add_message(tool_msg) + if self._current_base64_image: + image_messages.append( + Message.user_message( + content=f"Image returned by {command.function.name}:", + base64_image=self._current_base64_image, + ) + ) results.append(result) + self.memory.add_messages(image_messages) return "\n\n".join(results) async def execute_tool(self, command: ToolCall) -> str: diff --git a/app/mcp/server.py b/app/mcp/server.py index 3ee8b08c93..a35493b618 100644 --- a/app/mcp/server.py +++ b/app/mcp/server.py @@ -16,7 +16,6 @@ from app.logger import logger from app.tool.base import BaseTool from app.tool.bash import Bash -from app.tool.browser_use_tool import BrowserUseTool from app.tool.str_replace_editor import StrReplaceEditor from app.tool.terminate import Terminate @@ -30,7 +29,6 @@ def __init__(self, name: str = "openmanus"): # Initialize standard tools self.tools["bash"] = Bash() - self.tools["browser"] = BrowserUseTool() self.tools["editor"] = StrReplaceEditor() self.tools["terminate"] = Terminate() @@ -138,9 +136,6 @@ def _build_signature(self, tool_function: dict) -> Signature: async def cleanup(self) -> None: """Clean up server resources.""" logger.info("Cleaning up resources") - # Follow original cleanup logic - only clean browser tool - if "browser" in self.tools and hasattr(self.tools["browser"], "cleanup"): - await self.tools["browser"].cleanup() def register_all_tools(self) -> None: """Register all tools with the server.""" diff --git a/app/tool/__init__.py b/app/tool/__init__.py index 636e9b8dac..f2175c5c66 100644 --- a/app/tool/__init__.py +++ b/app/tool/__init__.py @@ -1,6 +1,5 @@ from app.tool.base import BaseTool from app.tool.bash import Bash -from app.tool.browser_use_tool import BrowserUseTool from app.tool.crawl4ai import Crawl4aiTool from app.tool.create_chat_completion import CreateChatCompletion from app.tool.planning import PlanningTool @@ -13,7 +12,6 @@ __all__ = [ "BaseTool", "Bash", - "BrowserUseTool", "Terminate", "StrReplaceEditor", "WebSearch", diff --git a/app/tool/browser_use_tool.py b/app/tool/browser_use_tool.py deleted file mode 100644 index 449e8e558f..0000000000 --- a/app/tool/browser_use_tool.py +++ /dev/null @@ -1,567 +0,0 @@ -import asyncio -import base64 -import json -from typing import Generic, Optional, TypeVar - -from browser_use import Browser as BrowserUseBrowser -from browser_use import BrowserConfig -from browser_use.browser.context import BrowserContext, BrowserContextConfig -from browser_use.dom.service import DomService -from pydantic import Field, field_validator -from pydantic_core.core_schema import ValidationInfo - -from app.config import config -from app.llm import LLM -from app.tool.base import BaseTool, ToolResult -from app.tool.web_search import WebSearch - - -_BROWSER_DESCRIPTION = """\ -A powerful browser automation tool that allows interaction with web pages through various actions. -* This tool provides commands for controlling a browser session, navigating web pages, and extracting information -* It maintains state across calls, keeping the browser session alive until explicitly closed -* Use this when you need to browse websites, fill forms, click buttons, extract content, or perform web searches -* Each action requires specific parameters as defined in the tool's dependencies - -Key capabilities include: -* Navigation: Go to specific URLs, go back, search the web, or refresh pages -* Interaction: Click elements, input text, select from dropdowns, send keyboard commands -* Scrolling: Scroll up/down by pixel amount or scroll to specific text -* Content extraction: Extract and analyze content from web pages based on specific goals -* Tab management: Switch between tabs, open new tabs, or close tabs - -Note: When using element indices, refer to the numbered elements shown in the current browser state. -""" - -Context = TypeVar("Context") - - -class BrowserUseTool(BaseTool, Generic[Context]): - name: str = "browser_use" - description: str = _BROWSER_DESCRIPTION - parameters: dict = { - "type": "object", - "properties": { - "action": { - "type": "string", - "enum": [ - "go_to_url", - "click_element", - "input_text", - "scroll_down", - "scroll_up", - "scroll_to_text", - "send_keys", - "get_dropdown_options", - "select_dropdown_option", - "go_back", - "web_search", - "wait", - "extract_content", - "switch_tab", - "open_tab", - "close_tab", - ], - "description": "The browser action to perform", - }, - "url": { - "type": "string", - "description": "URL for 'go_to_url' or 'open_tab' actions", - }, - "index": { - "type": "integer", - "description": "Element index for 'click_element', 'input_text', 'get_dropdown_options', or 'select_dropdown_option' actions", - }, - "text": { - "type": "string", - "description": "Text for 'input_text', 'scroll_to_text', or 'select_dropdown_option' actions", - }, - "scroll_amount": { - "type": "integer", - "description": "Pixels to scroll (positive for down, negative for up) for 'scroll_down' or 'scroll_up' actions", - }, - "tab_id": { - "type": "integer", - "description": "Tab ID for 'switch_tab' action", - }, - "query": { - "type": "string", - "description": "Search query for 'web_search' action", - }, - "goal": { - "type": "string", - "description": "Extraction goal for 'extract_content' action", - }, - "keys": { - "type": "string", - "description": "Keys to send for 'send_keys' action", - }, - "seconds": { - "type": "integer", - "description": "Seconds to wait for 'wait' action", - }, - }, - "required": ["action"], - "dependencies": { - "go_to_url": ["url"], - "click_element": ["index"], - "input_text": ["index", "text"], - "switch_tab": ["tab_id"], - "open_tab": ["url"], - "scroll_down": ["scroll_amount"], - "scroll_up": ["scroll_amount"], - "scroll_to_text": ["text"], - "send_keys": ["keys"], - "get_dropdown_options": ["index"], - "select_dropdown_option": ["index", "text"], - "go_back": [], - "web_search": ["query"], - "wait": ["seconds"], - "extract_content": ["goal"], - }, - } - - lock: asyncio.Lock = Field(default_factory=asyncio.Lock) - browser: Optional[BrowserUseBrowser] = Field(default=None, exclude=True) - context: Optional[BrowserContext] = Field(default=None, exclude=True) - dom_service: Optional[DomService] = Field(default=None, exclude=True) - web_search_tool: WebSearch = Field(default_factory=WebSearch, exclude=True) - - # Context for generic functionality - tool_context: Optional[Context] = Field(default=None, exclude=True) - - llm: Optional[LLM] = Field(default_factory=LLM) - - @field_validator("parameters", mode="before") - def validate_parameters(cls, v: dict, info: ValidationInfo) -> dict: - if not v: - raise ValueError("Parameters cannot be empty") - return v - - async def _ensure_browser_initialized(self) -> BrowserContext: - """Ensure browser and context are initialized.""" - if self.browser is None: - browser_config_kwargs = {"headless": False, "disable_security": True} - - if config.browser_config: - from browser_use.browser.browser import ProxySettings - - # handle proxy settings. - if config.browser_config.proxy and config.browser_config.proxy.server: - browser_config_kwargs["proxy"] = ProxySettings( - server=config.browser_config.proxy.server, - username=config.browser_config.proxy.username, - password=config.browser_config.proxy.password, - ) - - browser_attrs = [ - "headless", - "disable_security", - "extra_chromium_args", - "chrome_instance_path", - "wss_url", - "cdp_url", - ] - - for attr in browser_attrs: - value = getattr(config.browser_config, attr, None) - if value is not None: - if not isinstance(value, list) or value: - browser_config_kwargs[attr] = value - - self.browser = BrowserUseBrowser(BrowserConfig(**browser_config_kwargs)) - - if self.context is None: - context_config = BrowserContextConfig() - - # if there is context config in the config, use it. - if ( - config.browser_config - and hasattr(config.browser_config, "new_context_config") - and config.browser_config.new_context_config - ): - context_config = config.browser_config.new_context_config - - self.context = await self.browser.new_context(context_config) - self.dom_service = DomService(await self.context.get_current_page()) - - return self.context - - async def execute( - self, - action: str, - url: Optional[str] = None, - index: Optional[int] = None, - text: Optional[str] = None, - scroll_amount: Optional[int] = None, - tab_id: Optional[int] = None, - query: Optional[str] = None, - goal: Optional[str] = None, - keys: Optional[str] = None, - seconds: Optional[int] = None, - **kwargs, - ) -> ToolResult: - """ - Execute a specified browser action. - - Args: - action: The browser action to perform - url: URL for navigation or new tab - index: Element index for click or input actions - text: Text for input action or search query - scroll_amount: Pixels to scroll for scroll action - tab_id: Tab ID for switch_tab action - query: Search query for Google search - goal: Extraction goal for content extraction - keys: Keys to send for keyboard actions - seconds: Seconds to wait - **kwargs: Additional arguments - - Returns: - ToolResult with the action's output or error - """ - async with self.lock: - try: - context = await self._ensure_browser_initialized() - - # Get max content length from config - max_content_length = getattr( - config.browser_config, "max_content_length", 2000 - ) - - # Navigation actions - if action == "go_to_url": - if not url: - return ToolResult( - error="URL is required for 'go_to_url' action" - ) - page = await context.get_current_page() - await page.goto(url) - await page.wait_for_load_state() - return ToolResult(output=f"Navigated to {url}") - - elif action == "go_back": - await context.go_back() - return ToolResult(output="Navigated back") - - elif action == "refresh": - await context.refresh_page() - return ToolResult(output="Refreshed current page") - - elif action == "web_search": - if not query: - return ToolResult( - error="Query is required for 'web_search' action" - ) - # Execute the web search and return results directly without browser navigation - search_response = await self.web_search_tool.execute( - query=query, fetch_content=True, num_results=1 - ) - # Navigate to the first search result - first_search_result = search_response.results[0] - url_to_navigate = first_search_result.url - - page = await context.get_current_page() - await page.goto(url_to_navigate) - await page.wait_for_load_state() - - return search_response - - # Element interaction actions - elif action == "click_element": - if index is None: - return ToolResult( - error="Index is required for 'click_element' action" - ) - element = await context.get_dom_element_by_index(index) - if not element: - return ToolResult(error=f"Element with index {index} not found") - download_path = await context._click_element_node(element) - output = f"Clicked element at index {index}" - if download_path: - output += f" - Downloaded file to {download_path}" - return ToolResult(output=output) - - elif action == "input_text": - if index is None or not text: - return ToolResult( - error="Index and text are required for 'input_text' action" - ) - element = await context.get_dom_element_by_index(index) - if not element: - return ToolResult(error=f"Element with index {index} not found") - await context._input_text_element_node(element, text) - return ToolResult( - output=f"Input '{text}' into element at index {index}" - ) - - elif action == "scroll_down" or action == "scroll_up": - direction = 1 if action == "scroll_down" else -1 - amount = ( - scroll_amount - if scroll_amount is not None - else context.config.browser_window_size["height"] - ) - await context.execute_javascript( - f"window.scrollBy(0, {direction * amount});" - ) - return ToolResult( - output=f"Scrolled {'down' if direction > 0 else 'up'} by {amount} pixels" - ) - - elif action == "scroll_to_text": - if not text: - return ToolResult( - error="Text is required for 'scroll_to_text' action" - ) - page = await context.get_current_page() - try: - locator = page.get_by_text(text, exact=False) - await locator.scroll_into_view_if_needed() - return ToolResult(output=f"Scrolled to text: '{text}'") - except Exception as e: - return ToolResult(error=f"Failed to scroll to text: {str(e)}") - - elif action == "send_keys": - if not keys: - return ToolResult( - error="Keys are required for 'send_keys' action" - ) - page = await context.get_current_page() - await page.keyboard.press(keys) - return ToolResult(output=f"Sent keys: {keys}") - - elif action == "get_dropdown_options": - if index is None: - return ToolResult( - error="Index is required for 'get_dropdown_options' action" - ) - element = await context.get_dom_element_by_index(index) - if not element: - return ToolResult(error=f"Element with index {index} not found") - page = await context.get_current_page() - options = await page.evaluate( - """ - (xpath) => { - const select = document.evaluate(xpath, document, null, - XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; - if (!select) return null; - return Array.from(select.options).map(opt => ({ - text: opt.text, - value: opt.value, - index: opt.index - })); - } - """, - element.xpath, - ) - return ToolResult(output=f"Dropdown options: {options}") - - elif action == "select_dropdown_option": - if index is None or not text: - return ToolResult( - error="Index and text are required for 'select_dropdown_option' action" - ) - element = await context.get_dom_element_by_index(index) - if not element: - return ToolResult(error=f"Element with index {index} not found") - page = await context.get_current_page() - await page.select_option(element.xpath, label=text) - return ToolResult( - output=f"Selected option '{text}' from dropdown at index {index}" - ) - - # Content extraction actions - elif action == "extract_content": - if not goal: - return ToolResult( - error="Goal is required for 'extract_content' action" - ) - - page = await context.get_current_page() - import markdownify - - content = markdownify.markdownify(await page.content()) - - prompt = f"""\ -Your task is to extract the content of the page. You will be given a page and a goal, and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. -Extraction goal: {goal} - -Page content: -{content[:max_content_length]} -""" - messages = [{"role": "system", "content": prompt}] - - # Define extraction function schema - extraction_function = { - "type": "function", - "function": { - "name": "extract_content", - "description": "Extract specific information from a webpage based on a goal", - "parameters": { - "type": "object", - "properties": { - "extracted_content": { - "type": "object", - "description": "The content extracted from the page according to the goal", - "properties": { - "text": { - "type": "string", - "description": "Text content extracted from the page", - }, - "metadata": { - "type": "object", - "description": "Additional metadata about the extracted content", - "properties": { - "source": { - "type": "string", - "description": "Source of the extracted content", - } - }, - }, - }, - } - }, - "required": ["extracted_content"], - }, - }, - } - - # Use LLM to extract content with required function calling - response = await self.llm.ask_tool( - messages, - tools=[extraction_function], - tool_choice="required", - ) - - if response and response.tool_calls: - args = json.loads(response.tool_calls[0].function.arguments) - extracted_content = args.get("extracted_content", {}) - return ToolResult( - output=f"Extracted from page:\n{extracted_content}\n" - ) - - return ToolResult(output="No content was extracted from the page.") - - # Tab management actions - elif action == "switch_tab": - if tab_id is None: - return ToolResult( - error="Tab ID is required for 'switch_tab' action" - ) - await context.switch_to_tab(tab_id) - page = await context.get_current_page() - await page.wait_for_load_state() - return ToolResult(output=f"Switched to tab {tab_id}") - - elif action == "open_tab": - if not url: - return ToolResult(error="URL is required for 'open_tab' action") - await context.create_new_tab(url) - return ToolResult(output=f"Opened new tab with {url}") - - elif action == "close_tab": - await context.close_current_tab() - return ToolResult(output="Closed current tab") - - # Utility actions - elif action == "wait": - seconds_to_wait = seconds if seconds is not None else 3 - await asyncio.sleep(seconds_to_wait) - return ToolResult(output=f"Waited for {seconds_to_wait} seconds") - - else: - return ToolResult(error=f"Unknown action: {action}") - - except Exception as e: - return ToolResult(error=f"Browser action '{action}' failed: {str(e)}") - - async def get_current_state( - self, context: Optional[BrowserContext] = None - ) -> ToolResult: - """ - Get the current browser state as a ToolResult. - If context is not provided, uses self.context. - """ - try: - # Use provided context or fall back to self.context - ctx = context or self.context - if not ctx: - return ToolResult(error="Browser context not initialized") - - state = await ctx.get_state() - - # Create a viewport_info dictionary if it doesn't exist - viewport_height = 0 - if hasattr(state, "viewport_info") and state.viewport_info: - viewport_height = state.viewport_info.height - elif hasattr(ctx, "config") and hasattr(ctx.config, "browser_window_size"): - viewport_height = ctx.config.browser_window_size.get("height", 0) - - # Take a screenshot for the state - page = await ctx.get_current_page() - - await page.bring_to_front() - await page.wait_for_load_state() - - screenshot = await page.screenshot( - full_page=True, animations="disabled", type="jpeg", quality=100 - ) - - screenshot = base64.b64encode(screenshot).decode("utf-8") - - # Build the state info with all required fields - state_info = { - "url": state.url, - "title": state.title, - "tabs": [tab.model_dump() for tab in state.tabs], - "help": "[0], [1], [2], etc., represent clickable indices corresponding to the elements listed. Clicking on these indices will navigate to or interact with the respective content behind them.", - "interactive_elements": ( - state.element_tree.clickable_elements_to_string() - if state.element_tree - else "" - ), - "scroll_info": { - "pixels_above": getattr(state, "pixels_above", 0), - "pixels_below": getattr(state, "pixels_below", 0), - "total_height": getattr(state, "pixels_above", 0) - + getattr(state, "pixels_below", 0) - + viewport_height, - }, - "viewport_height": viewport_height, - } - - return ToolResult( - output=json.dumps(state_info, indent=4, ensure_ascii=False), - base64_image=screenshot, - ) - except Exception as e: - return ToolResult(error=f"Failed to get browser state: {str(e)}") - - async def cleanup(self): - """Clean up browser resources.""" - async with self.lock: - if self.context is not None: - await self.context.close() - self.context = None - self.dom_service = None - if self.browser is not None: - await self.browser.close() - self.browser = None - - def __del__(self): - """Ensure cleanup when object is destroyed.""" - if self.browser is not None or self.context is not None: - try: - asyncio.run(self.cleanup()) - except RuntimeError: - loop = asyncio.new_event_loop() - loop.run_until_complete(self.cleanup()) - loop.close() - - @classmethod - def create_with_context(cls, context: Context) -> "BrowserUseTool[Context]": - """Factory method to create a BrowserUseTool with a specific context.""" - tool = cls() - tool.tool_context = context - return tool diff --git a/app/tool/mcp.py b/app/tool/mcp.py index 32fa8249e5..60549b07ef 100644 --- a/app/tool/mcp.py +++ b/app/tool/mcp.py @@ -4,7 +4,7 @@ from mcp import ClientSession, StdioServerParameters from mcp.client.sse import sse_client from mcp.client.stdio import stdio_client -from mcp.types import ListToolsResult, TextContent +from mcp.types import ImageContent, ListToolsResult, TextContent from app.logger import logger from app.tool.base import BaseTool, ToolResult @@ -29,7 +29,19 @@ async def execute(self, **kwargs) -> ToolResult: content_str = ", ".join( item.text for item in result.content if isinstance(item, TextContent) ) - return ToolResult(output=content_str or "No output returned.") + image = next( + ( + item.data + for item in result.content + if isinstance(item, ImageContent) + ), + None, + ) + return ToolResult( + output=content_str + or ("Image returned." if image else "No output returned."), + base64_image=image, + ) except Exception as e: return ToolResult(error=f"Error executing tool: {str(e)}") @@ -41,11 +53,15 @@ class MCPClients(ToolCollection): sessions: Dict[str, ClientSession] = {} exit_stacks: Dict[str, AsyncExitStack] = {} + server_instructions: Dict[str, str] = {} description: str = "MCP client tools for server interaction" def __init__(self): super().__init__() # Initialize with empty tools list self.name = "mcp" # Keep name for backward compatibility + self.sessions = {} + self.exit_stacks = {} + self.server_instructions = {} async def connect_sse(self, server_url: str, server_id: str = "") -> None: """Connect to an MCP server using SSE transport.""" @@ -69,7 +85,12 @@ async def connect_sse(self, server_url: str, server_id: str = "") -> None: await self._initialize_and_list_tools(server_id) async def connect_stdio( - self, command: str, args: List[str], server_id: str = "" + self, + command: str, + args: List[str], + server_id: str = "", + tool_name_prefix: bool = True, + env: Optional[Dict[str, str]] = None, ) -> None: """Connect to an MCP server using stdio transport.""" if not command: @@ -84,7 +105,7 @@ async def connect_stdio( exit_stack = AsyncExitStack() self.exit_stacks[server_id] = exit_stack - server_params = StdioServerParameters(command=command, args=args) + server_params = StdioServerParameters(command=command, args=args, env=env) stdio_transport = await exit_stack.enter_async_context( stdio_client(server_params) ) @@ -92,21 +113,29 @@ async def connect_stdio( session = await exit_stack.enter_async_context(ClientSession(read, write)) self.sessions[server_id] = session - await self._initialize_and_list_tools(server_id) + await self._initialize_and_list_tools(server_id, tool_name_prefix) - async def _initialize_and_list_tools(self, server_id: str) -> None: + async def _initialize_and_list_tools( + self, server_id: str, tool_name_prefix: bool = True + ) -> None: """Initialize session and populate tool map.""" session = self.sessions.get(server_id) if not session: raise RuntimeError(f"Session not initialized for server {server_id}") - await session.initialize() + initialization = await session.initialize() + if initialization.instructions: + self.server_instructions[server_id] = initialization.instructions response = await session.list_tools() # Create proper tool objects for each server tool for tool in response.tools: original_name = tool.name - tool_name = f"mcp_{server_id}_{original_name}" + tool_name = ( + f"mcp_{server_id}_{original_name}" + if tool_name_prefix + else original_name + ) tool_name = self._sanitize_tool_name(tool_name) server_tool = MCPClientTool( @@ -174,6 +203,7 @@ async def disconnect(self, server_id: str = "") -> None: # Clean up references self.sessions.pop(server_id, None) self.exit_stacks.pop(server_id, None) + self.server_instructions.pop(server_id, None) # Remove tools associated with this server self.tool_map = { diff --git a/requirements.txt b/requirements.txt index aa7e6dc938..42278f17b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ openai~=1.66.3 tenacity~=9.0.0 pyyaml~=6.0.2 loguru~=0.7.3 +structlog>=24.1.0,<26.0.0 numpy datasets~=3.4.1 fastapi~=0.115.11 @@ -10,11 +11,11 @@ tiktoken~=0.9.0 html2text~=2024.2.26 gymnasium~=1.1.1 -pillow~=11.1.0 +pillow>=10.4,<11 browsergym~=0.13.3 uvicorn~=0.34.0 unidiff~=0.7.5 -browser-use~=0.1.40 +uv>=0.6.0 googlesearch-python~=1.3.0 baidusearch~=1.0.3 duckduckgo_search~=7.5.3 diff --git a/setup.py b/setup.py index bf291e4f49..9d1371926b 100644 --- a/setup.py +++ b/setup.py @@ -20,15 +20,16 @@ "tenacity~=9.0.0", "pyyaml~=6.0.2", "loguru~=0.7.3", + "structlog>=24.1.0,<26.0.0", "numpy", "datasets>=3.2,<3.5", "html2text~=2024.2.26", "gymnasium>=1.0,<1.2", - "pillow>=10.4,<11.2", + "pillow>=10.4,<11", "browsergym~=0.13.3", "uvicorn~=0.34.0", "unidiff~=0.7.5", - "browser-use~=0.1.40", + "uv>=0.6.0", "googlesearch-python~=1.3.0", "aiofiles~=24.1.0", "pydantic_core>=2.27.2,<2.28.0", diff --git a/tests/tools/test_browser_use_mcp.py b/tests/tools/test_browser_use_mcp.py new file mode 100644 index 0000000000..b6218df2af --- /dev/null +++ b/tests/tools/test_browser_use_mcp.py @@ -0,0 +1,148 @@ +from pathlib import Path +from types import SimpleNamespace + +import pytest +from mcp import ClientSession +from mcp.types import ImageContent, ListToolsResult, TextContent, Tool + + +_CONFIG_PATH = Path(__file__).parents[2] / "config" / "config.toml" +_CREATED_TEST_CONFIG = not _CONFIG_PATH.exists() +if _CREATED_TEST_CONFIG: + _CONFIG_PATH.write_text( + '[llm]\nmodel = "test"\nbase_url = "http://localhost"\napi_key = "test"\n' + '\n[daytona]\ndaytona_api_key = "test"\n' + ) + +try: + from app.agent import manus as manus_module + from app.agent.toolcall import ToolCallAgent + from app.schema import Function, ToolCall + from app.tool.base import BaseTool, ToolResult + from app.tool.mcp import MCPClients, MCPClientTool + from app.tool.tool_collection import ToolCollection +finally: + if _CREATED_TEST_CONFIG: + _CONFIG_PATH.unlink() + + +class FakeSession(ClientSession): + def __init__(self, *, instructions="", content=None): + self.instructions = instructions + self.content = content or [] + + async def initialize(self): + return SimpleNamespace(instructions=self.instructions) + + async def list_tools(self): + return ListToolsResult( + tools=[ + Tool( + name="browser_exec", + description="Execute Browser Use CLI 3.0 code", + inputSchema={"type": "object"}, + ), + Tool( + name="browser_screenshot", + description="Capture the current page", + inputSchema={"type": "object"}, + ), + ] + ) + + async def call_tool(self, name, arguments): + return SimpleNamespace(content=self.content) + + +class ImageTool(BaseTool): + name: str = "image_tool" + description: str = "Return an image" + parameters: dict = {"type": "object"} + + async def execute(self, **kwargs): + return ToolResult(output="image ready", base64_image="cG5n") + + +@pytest.mark.asyncio +async def test_mcp_preserves_server_instructions_and_native_tool_names(): + clients = MCPClients() + clients.sessions["browser_use"] = FakeSession(instructions="canonical skill") + + await clients._initialize_and_list_tools("browser_use", tool_name_prefix=False) + + assert clients.server_instructions["browser_use"] == "canonical skill" + assert set(clients.tool_map) == {"browser_exec", "browser_screenshot"} + + +@pytest.mark.asyncio +async def test_mcp_forwards_text_and_screenshot_content(): + session = FakeSession( + content=[ + TextContent(type="text", text="done"), + ImageContent(type="image", data="cG5n", mimeType="image/png"), + ] + ) + tool = MCPClientTool( + name="browser_screenshot", + description="Capture the current page", + parameters={"type": "object"}, + session=session, + server_id="browser_use", + original_name="browser_screenshot", + ) + + result = await tool.execute() + + assert result.output == "done" + assert result.base64_image == "cG5n" + + +@pytest.mark.asyncio +async def test_tool_images_follow_the_tool_result_as_user_messages(): + agent = ToolCallAgent(available_tools=ToolCollection(ImageTool())) + agent.tool_calls = [ + ToolCall( + id="call_1", + function=Function(name="image_tool", arguments="{}"), + ) + ] + + await agent.act() + + tool_message, image_message = agent.memory.messages + assert getattr(tool_message.role, "value", tool_message.role) == "tool" + assert tool_message.base64_image is None + assert getattr(image_message.role, "value", image_message.role) == "user" + assert image_message.base64_image == "cG5n" + + +@pytest.mark.asyncio +async def test_manus_enables_cli_mcp_by_default(monkeypatch): + calls = [] + + async def record_connection(self, *args, **kwargs): + calls.append((args, kwargs)) + + monkeypatch.delenv("OPENMANUS_DISABLE_BROWSER_USE", raising=False) + monkeypatch.setenv("BROWSER_USE_API_KEY", "bu_test") + monkeypatch.setenv("BU_CDP_URL", "http://127.0.0.1:9237") + monkeypatch.setenv("OPENAI_API_KEY", "must-not-be-forwarded") + monkeypatch.setattr(manus_module.config.mcp_config, "servers", {}) + monkeypatch.setattr(manus_module.Manus, "connect_mcp_server", record_connection) + + await manus_module.Manus.model_construct().initialize_mcp_servers() + + assert calls == [ + ( + ("uvx", "browser_use"), + { + "use_stdio": True, + "stdio_args": ["browser-use", "--cli-mcp"], + "tool_name_prefix": False, + "stdio_env": { + "BROWSER_USE_API_KEY": "bu_test", + "BU_CDP_URL": "http://127.0.0.1:9237", + }, + }, + ) + ]