Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions scripts/install-common.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ function Get-TropeCuaRequiredSdkVersion {
function Test-TropeCuaDotnetHasSdk {
param(
[string]$DotnetPath,
[string]$SdkVersion
[string]$Root
)

if ([string]::IsNullOrWhiteSpace($DotnetPath) -or -not (Test-Path $DotnetPath)) {
Expand All @@ -40,12 +40,22 @@ function Test-TropeCuaDotnetHasSdk {
}
}

$installedSdks = @(& $DotnetPath --list-sdks 2>$null)
if ($LASTEXITCODE -ne 0) {
return $false
$selectedSdk = $null
$exitCode = 1
Push-Location -LiteralPath $Root
try {
try {
$selectedSdk = & $DotnetPath --version 2>$null
$exitCode = $LASTEXITCODE
} catch {
$selectedSdk = $null
$exitCode = 1
}
} finally {
Pop-Location
}

return [bool]($installedSdks | Where-Object { $_ -match "^$([regex]::Escape($SdkVersion))\s+\[" } | Select-Object -First 1)
return $exitCode -eq 0 -and -not [string]::IsNullOrWhiteSpace($selectedSdk)
}

function Resolve-TropeCuaDotnet {
Expand All @@ -68,7 +78,7 @@ function Resolve-TropeCuaDotnet {
$candidates += "dotnet"

foreach ($candidate in ($candidates | Select-Object -Unique)) {
if (Test-TropeCuaDotnetHasSdk -DotnetPath $candidate -SdkVersion $requestedSdk) {
if (Test-TropeCuaDotnetHasSdk -DotnetPath $candidate -Root $Root) {
return $candidate
}
}
Expand Down
46 changes: 38 additions & 8 deletions src/CuaDriver.Win/Mcp/NamedPipeDaemon.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,20 +47,20 @@ public async Task RunAsync(CancellationToken ct)
{
while (!shutdown.IsCancellationRequested)
{
await using var pipe = new NamedPipeServerStream(InstancePipeName, PipeDirection.InOut, 1, PipeTransmissionMode.Byte, PipeOptions.Asynchronous);
await pipe.WaitForConnectionAsync(shutdown.Token).ConfigureAwait(false);
using var reader = new StreamReader(pipe);
await using var writer = new StreamWriter(pipe) { AutoFlush = true };

try
{
await using var pipe = new NamedPipeServerStream(InstancePipeName, PipeDirection.InOut, 1, PipeTransmissionMode.Byte, PipeOptions.Asynchronous);
await pipe.WaitForConnectionAsync(shutdown.Token).ConfigureAwait(false);
using var reader = new StreamReader(pipe);
await using var writer = new StreamWriter(pipe) { AutoFlush = true };

var line = await reader.ReadLineAsync(shutdown.Token).ConfigureAwait(false);
if (line is null)
continue;
var request = JsonSerializer.Deserialize<DaemonRequest>(line, JsonUtil.SerializerOptions);
if (request is null)
{
await WriteResponseAsync(writer, new DaemonResponse(false, null, "Invalid daemon request")).ConfigureAwait(false);
await TryWriteResponseAsync(writer, new DaemonResponse(false, null, "Invalid daemon request")).ConfigureAwait(false);
continue;
}

Expand All @@ -72,17 +72,26 @@ public async Task RunAsync(CancellationToken ct)
_ => new DaemonResponse(false, null, "Invalid daemon request")
};

await WriteResponseAsync(writer, response).ConfigureAwait(false);
await TryWriteResponseAsync(writer, response).ConfigureAwait(false);
if (request.Method == "shutdown")
shutdown.Cancel();
}
catch (OperationCanceledException) when (shutdown.IsCancellationRequested)
{
// Expected during daemon shutdown.
}
catch (IOException)
{
// The client may time out and close the pipe while a slow tool call is
// still producing a response. Keep the daemon alive for the next request.
}
catch (ObjectDisposedException)
{
// Treat disconnected clients the same as a broken pipe.
}
catch (Exception ex)
{
await WriteResponseAsync(writer, new DaemonResponse(false, null, $"{ex.GetType().Name}: {ex.Message}")).ConfigureAwait(false);
Console.Error.WriteLine($"trope-cua daemon request failed: {ex.GetType().Name}: {ex.Message}");
}
}
}
Expand All @@ -103,6 +112,27 @@ private static async Task WriteResponseAsync(StreamWriter writer, DaemonResponse
await writer.WriteLineAsync(JsonSerializer.Serialize(response, JsonUtil.LineSerializerOptions)).ConfigureAwait(false);
}

private static async Task<bool> TryWriteResponseAsync(StreamWriter writer, DaemonResponse response)
{
try
{
await WriteResponseAsync(writer, response).ConfigureAwait(false);
return true;
}
catch (IOException)
{
return false;
}
catch (ObjectDisposedException)
{
return false;
}
catch (InvalidOperationException)
{
return false;
}
}

private ToolResult StatusResult()
{
var structured = StatusObject();
Expand Down
10 changes: 9 additions & 1 deletion src/CuaDriver.Win/Tooling/JsonArgs.cs
Original file line number Diff line number Diff line change
Expand Up @@ -153,5 +153,13 @@ public static JsonObject EnumProp(string description, params string[] values)
}

public static JsonObject Prop(string type, string description)
=> new() { ["type"] = type, ["description"] = description };
{
var prop = new JsonObject { ["type"] = type, ["description"] = description };
if (string.Equals(type, "array", StringComparison.Ordinal))
{
prop["items"] = new JsonObject { ["type"] = "string" };
}

return prop;
}
}
1 change: 1 addition & 0 deletions src/CuaDriver.Win/Tooling/ToolRegistry.cs
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ public static ToolRegistry CreateDefault(DriverState state)
new Tools.LaunchAppTool(),
new Tools.GetWindowStateTool(),
new Tools.GetAccessibilityTreeTool(),
new Tools.FindElementTool(),
new Tools.ScreenshotTool(),
new Tools.ZoomTool(),
new Tools.ClickTool(),
Expand Down
32 changes: 32 additions & 0 deletions src/CuaDriver.Win/Tools/ClickTool.cs
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,17 @@ private static async Task<ActionReceipt> InvokeNativeElementActionAsync(
return await UiAutomationActions.InvokeElementAsync(element, action, cancellationToken, allowTransientForeground: true).ConfigureAwait(false);
}

if (PrefersUiaAction(element, action))
{
var uiaReceipt = await UiAutomationActions.InvokeElementAsync(
element,
action,
cancellationToken,
allowTransientForeground: allowTransientForeground).ConfigureAwait(false);
if (uiaReceipt.ShouldStopFallback)
return uiaReceipt;
}

var msaaReceipt = screenPoint is null
? MsaaActions.DoDefaultActionAtElement(window.Hwnd, element)
: MsaaActions.DoDefaultActionAtPoint(window.Hwnd, screenPoint.Value);
Expand All @@ -390,6 +401,27 @@ private static async Task<ActionReceipt> InvokeNativeElementActionAsync(
return await UiAutomationActions.InvokeElementAsync(element, action, cancellationToken, allowTransientForeground: allowTransientForeground).ConfigureAwait(false);
}

private static bool PrefersUiaAction(AutomationElement element, string action)
{
if (!string.Equals(action, "press", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(action, "open", StringComparison.OrdinalIgnoreCase))
{
return false;
}

try
{
return element.TryGetCurrentPattern(InvokePattern.Pattern, out _)
|| element.TryGetCurrentPattern(TogglePattern.Pattern, out _)
|| element.TryGetCurrentPattern(SelectionItemPattern.Pattern, out _)
|| element.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out _);
}
catch
{
return false;
}
}

private static bool RequiresIsolatedInputLane(AutomationElement element, string action)
{
if (!string.Equals(action, "press", StringComparison.OrdinalIgnoreCase))
Expand Down
117 changes: 117 additions & 0 deletions src/CuaDriver.Win/Tools/FindElementTool.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
using System.Diagnostics;
using System.Globalization;
using System.Text.Json.Nodes;
using CuaDriver.Win.Tooling;

namespace CuaDriver.Win.Tools;

internal sealed class FindElementTool : IDriverTool
{
public ToolDefinition Definition { get; } = new(
"find_element",
ToolDescriptions.FindElement,
JsonArgs.RequiredSchema(["pid", "window_id"],
("pid", JsonArgs.Prop("integer", "Target process id.")),
("window_id", JsonArgs.Prop("integer", "Target HWND as returned by list_windows.")),
("query", JsonArgs.Prop("string", "Substring/token query for element name, automation id, or class.")),
("label", JsonArgs.Prop("string", "Alias for query when the selector came from a human-visible label.")),
("required_query", JsonArgs.Prop("string", "Optional text that must also be present in the same search zone before matches are returned.")),
("requiredQuery", JsonArgs.Prop("string", "Alias for required_query.")),
("automation_id", JsonArgs.Prop("string", "AutomationId to match.")),
("automationId", JsonArgs.Prop("string", "Alias for automation_id.")),
("identifier", JsonArgs.Prop("string", "Automation id, name, or class hint.")),
("control_type", JsonArgs.Prop("string", "Optional UIA control type filter, e.g. Edit, Button, link.")),
("controlType", JsonArgs.Prop("string", "Alias for control_type.")),
("target_zone", JsonArgs.Prop("string", "Optional search zone: browser_chrome, page_content, video_player.")),
("targetZone", JsonArgs.Prop("string", "Alias for target_zone.")),
("limit", JsonArgs.Prop("integer", "Maximum matches to return. Default 1.")),
("max_visited", JsonArgs.Prop("integer", "Maximum control-view nodes to inspect. Default 1500.")),
("include_offscreen", JsonArgs.Prop("boolean", "Include offscreen elements. Default false."))),
ReadOnly: true,
Idempotent: false);

public Task<ToolResult> InvokeAsync(JsonObject args, ToolContext context, CancellationToken cancellationToken)
{
var pid = JsonArgs.RequiredInt(args, "pid");
var windowId = JsonArgs.RequiredLong(args, "window_id");
var query = JsonArgs.OptionalString(args, "query") ?? JsonArgs.OptionalString(args, "label");
var requiredQuery = JsonArgs.OptionalString(args, "required_query") ?? JsonArgs.OptionalString(args, "requiredQuery");
var automationId = JsonArgs.OptionalString(args, "automation_id") ?? JsonArgs.OptionalString(args, "automationId");
var identifier = JsonArgs.OptionalString(args, "identifier");
var controlType = JsonArgs.OptionalString(args, "control_type") ?? JsonArgs.OptionalString(args, "controlType");
var targetZone = JsonArgs.OptionalString(args, "target_zone") ?? JsonArgs.OptionalString(args, "targetZone");
var limit = JsonArgs.OptionalInt(args, "limit") ?? 1;
var maxVisited = JsonArgs.OptionalInt(args, "max_visited") ?? 1500;
var includeOffscreen = JsonArgs.OptionalBool(args, "include_offscreen");

if (!ToolWindows.TryFindForPid(pid, windowId, out var window, out var error))
return Task.FromResult(error!);

var stopwatch = Stopwatch.StartNew();
var snapshot = context.State.UiaTree.FindElements(
pid,
windowId,
query,
requiredQuery,
automationId,
identifier,
controlType,
targetZone,
limit,
maxVisited,
includeOffscreen,
cancellationToken);
stopwatch.Stop();

var matches = snapshot.Elements.ToArray();
var structured = new JsonObject
{
["pid"] = pid,
["window_id"] = windowId,
["window"] = ToolJson.Window(window),
["matched"] = matches.Length > 0,
["match_count"] = matches.Length,
["elapsed_ms"] = stopwatch.ElapsedMilliseconds,
["query"] = query,
["required_query"] = requiredQuery,
["required_matched"] = snapshot.RequiredMatched,
["automation_id"] = automationId,
["identifier"] = identifier,
["control_type"] = controlType,
["target_zone"] = targetZone,
["uia"] = new JsonObject
{
["turn_id"] = snapshot.TurnId,
["element_count"] = snapshot.ElementCount,
["tree_markdown_chars"] = snapshot.TreeMarkdown.Length,
["metrics"] = ToolJson.UiSnapshotMetrics(snapshot.Metrics)
},
["matches"] = ToolJson.Array(matches, ToolJson.Element)
};

if (matches.Length > 0)
{
var first = matches[0];
structured["element_index"] = first.ElementIndex;
structured["element"] = ToolJson.Element(first);
}

var text = matches.Length > 0
? string.Format(
CultureInfo.InvariantCulture,
"{0}find_element matched {1} element(s) in {2} ms; first element_index={3}",
ToolText.OkPrefix,
matches.Length,
stopwatch.ElapsedMilliseconds,
matches[0].ElementIndex)
: string.Format(
CultureInfo.InvariantCulture,
"{0}find_element found no matches in {1} ms",
ToolText.WarningPrefix,
stopwatch.ElapsedMilliseconds);
if (!string.IsNullOrWhiteSpace(snapshot.TreeMarkdown))
text += Environment.NewLine + Environment.NewLine + snapshot.TreeMarkdown;

return Task.FromResult(ToolResult.Text(text, structured));
}
}
6 changes: 6 additions & 0 deletions src/CuaDriver.Win/Tools/ToolDescriptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@ internal static class ToolDescriptions
capture_mode override: som=screenshot+tree (default), ax=tree only, vision=screenshot only. Use ax for cheap UIA refreshes and som when pixels are needed. Use query to trim Markdown to matches plus ancestors; indices still resolve against the full cached tree. tree_markdown is text-only by default; pass include_structured_tree=true only if JSON duplication is needed.
""";

public const string FindElement = """
Find matching UIA elements in one explicit (pid, window_id) without building a full window snapshot. This is the fast path for known selectors such as visible labels, automation ids, and control types. Pass target_zone=browser_chrome for address/toolbars and target_zone=page_content for web document links/results when known. Pass required_query when a target is only valid inside a specific page or state, such as a Google Search results page.

The returned element_index values are cached for the same pid/window_id and can be passed directly to click, type_text, press_key, hotkey, scroll, or set_value. Use get_window_state when you need broad inspection; use find_element when you already know the target string or automation id.
""";

public const string Click = """
Left-click a target pid. Prefer element_index + window_id from get_window_state; this performs semantic UIA/MSAA action and is background-safe when the receipt says so. action may be press, show_menu, pick, confirm, cancel, or open.

Expand Down
Loading
Loading