From 959070e5dd2453dd139553a9d288171575aed0f3 Mon Sep 17 00:00:00 2001 From: Victor Date: Mon, 4 May 2026 23:33:01 -0700 Subject: [PATCH 1/5] windows(fix): Keep CUA daemon alive after disconnects --- src/CuaDriver.Win/Mcp/NamedPipeDaemon.cs | 46 +++++++++++++++++++----- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/src/CuaDriver.Win/Mcp/NamedPipeDaemon.cs b/src/CuaDriver.Win/Mcp/NamedPipeDaemon.cs index 97608a7..dc17873 100644 --- a/src/CuaDriver.Win/Mcp/NamedPipeDaemon.cs +++ b/src/CuaDriver.Win/Mcp/NamedPipeDaemon.cs @@ -47,20 +47,20 @@ public async Task RunAsync(CancellationToken ct) { while (!shutdown.IsCancellationRequested) { - await using var pipe = new NamedPipeServerStream(InstancePipeName, PipeDirection.InOut, 1, PipeTransmissionMode.Byte, PipeOptions.Asynchronous); - await pipe.WaitForConnectionAsync(shutdown.Token).ConfigureAwait(false); - using var reader = new StreamReader(pipe); - await using var writer = new StreamWriter(pipe) { AutoFlush = true }; - try { + await using var pipe = new NamedPipeServerStream(InstancePipeName, PipeDirection.InOut, 1, PipeTransmissionMode.Byte, PipeOptions.Asynchronous); + await pipe.WaitForConnectionAsync(shutdown.Token).ConfigureAwait(false); + using var reader = new StreamReader(pipe); + await using var writer = new StreamWriter(pipe) { AutoFlush = true }; + var line = await reader.ReadLineAsync(shutdown.Token).ConfigureAwait(false); if (line is null) continue; var request = JsonSerializer.Deserialize(line, JsonUtil.SerializerOptions); if (request is null) { - await WriteResponseAsync(writer, new DaemonResponse(false, null, "Invalid daemon request")).ConfigureAwait(false); + await TryWriteResponseAsync(writer, new DaemonResponse(false, null, "Invalid daemon request")).ConfigureAwait(false); continue; } @@ -72,7 +72,7 @@ public async Task RunAsync(CancellationToken ct) _ => new DaemonResponse(false, null, "Invalid daemon request") }; - await WriteResponseAsync(writer, response).ConfigureAwait(false); + await TryWriteResponseAsync(writer, response).ConfigureAwait(false); if (request.Method == "shutdown") shutdown.Cancel(); } @@ -80,9 +80,18 @@ public async Task RunAsync(CancellationToken ct) { // Expected during daemon shutdown. } + catch (IOException) + { + // The client may time out and close the pipe while a slow tool call is + // still producing a response. Keep the daemon alive for the next request. + } + catch (ObjectDisposedException) + { + // Treat disconnected clients the same as a broken pipe. + } catch (Exception ex) { - await WriteResponseAsync(writer, new DaemonResponse(false, null, $"{ex.GetType().Name}: {ex.Message}")).ConfigureAwait(false); + Console.Error.WriteLine($"trope-cua daemon request failed: {ex.GetType().Name}: {ex.Message}"); } } } @@ -103,6 +112,27 @@ private static async Task WriteResponseAsync(StreamWriter writer, DaemonResponse await writer.WriteLineAsync(JsonSerializer.Serialize(response, JsonUtil.LineSerializerOptions)).ConfigureAwait(false); } + private static async Task TryWriteResponseAsync(StreamWriter writer, DaemonResponse response) + { + try + { + await WriteResponseAsync(writer, response).ConfigureAwait(false); + return true; + } + catch (IOException) + { + return false; + } + catch (ObjectDisposedException) + { + return false; + } + catch (InvalidOperationException) + { + return false; + } + } + private ToolResult StatusResult() { var structured = StatusObject(); From 85c17ad96d075f8a8dd1571b8428f123b1c01c35 Mon Sep 17 00:00:00 2001 From: Victor Date: Mon, 4 May 2026 23:33:08 -0700 Subject: [PATCH 2/5] windows(fix): Prefer UIA for actionable CUA elements --- src/CuaDriver.Win/Tools/ClickTool.cs | 32 ++++++++++++++++++++ src/CuaDriver.Win/Uia/UiAutomationTree.cs | 2 +- src/CuaDriver.Win/Uia/UiElementClassifier.cs | 1 + 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/CuaDriver.Win/Tools/ClickTool.cs b/src/CuaDriver.Win/Tools/ClickTool.cs index fc8424c..361c862 100644 --- a/src/CuaDriver.Win/Tools/ClickTool.cs +++ b/src/CuaDriver.Win/Tools/ClickTool.cs @@ -371,6 +371,17 @@ private static async Task InvokeNativeElementActionAsync( return await UiAutomationActions.InvokeElementAsync(element, action, cancellationToken, allowTransientForeground: true).ConfigureAwait(false); } + if (PrefersUiaAction(element, action)) + { + var uiaReceipt = await UiAutomationActions.InvokeElementAsync( + element, + action, + cancellationToken, + allowTransientForeground: allowTransientForeground).ConfigureAwait(false); + if (uiaReceipt.ShouldStopFallback) + return uiaReceipt; + } + var msaaReceipt = screenPoint is null ? MsaaActions.DoDefaultActionAtElement(window.Hwnd, element) : MsaaActions.DoDefaultActionAtPoint(window.Hwnd, screenPoint.Value); @@ -390,6 +401,27 @@ private static async Task InvokeNativeElementActionAsync( return await UiAutomationActions.InvokeElementAsync(element, action, cancellationToken, allowTransientForeground: allowTransientForeground).ConfigureAwait(false); } + private static bool PrefersUiaAction(AutomationElement element, string action) + { + if (!string.Equals(action, "press", StringComparison.OrdinalIgnoreCase) + && !string.Equals(action, "open", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + try + { + return element.TryGetCurrentPattern(InvokePattern.Pattern, out _) + || element.TryGetCurrentPattern(TogglePattern.Pattern, out _) + || element.TryGetCurrentPattern(SelectionItemPattern.Pattern, out _) + || element.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out _); + } + catch + { + return false; + } + } + private static bool RequiresIsolatedInputLane(AutomationElement element, string action) { if (!string.Equals(action, "press", StringComparison.OrdinalIgnoreCase)) diff --git a/src/CuaDriver.Win/Uia/UiAutomationTree.cs b/src/CuaDriver.Win/Uia/UiAutomationTree.cs index 928c3f7..2beaa2e 100644 --- a/src/CuaDriver.Win/Uia/UiAutomationTree.cs +++ b/src/CuaDriver.Win/Uia/UiAutomationTree.cs @@ -8,7 +8,7 @@ namespace CuaDriver.Win.Uia; internal sealed class UiAutomationTree { - private const int RawHarvestElementThreshold = 20; + private const int RawHarvestElementThreshold = 256; private readonly object _gate = new(); private readonly Dictionary<(int Pid, long WindowId), SessionState> _sessions = new(); diff --git a/src/CuaDriver.Win/Uia/UiElementClassifier.cs b/src/CuaDriver.Win/Uia/UiElementClassifier.cs index 8dc77cf..da64219 100644 --- a/src/CuaDriver.Win/Uia/UiElementClassifier.cs +++ b/src/CuaDriver.Win/Uia/UiElementClassifier.cs @@ -27,6 +27,7 @@ public static bool ShouldReadPatterns(string controlType) private static bool IsLikelyActionableControlTypeCore(string type) { return type.Contains("button") + || type == "link" || type.Contains("edit") || type.Contains("hyperlink") || type.Contains("menu item") From 7ca64b3b9d74701f2cf08a7f9bb079a5595c9801 Mon Sep 17 00:00:00 2001 From: Victor Date: Tue, 5 May 2026 09:35:28 -0700 Subject: [PATCH 3/5] windows(feat): Add targeted UIA element lookup --- src/CuaDriver.Win/Tooling/ToolRegistry.cs | 1 + src/CuaDriver.Win/Tools/FindElementTool.cs | 117 +++++++ src/CuaDriver.Win/Tools/ToolDescriptions.cs | 6 + src/CuaDriver.Win/Uia/UiAutomationTree.cs | 353 +++++++++++++++++++- src/CuaDriver.Win/Uia/UiSnapshot.cs | 3 +- 5 files changed, 476 insertions(+), 4 deletions(-) create mode 100644 src/CuaDriver.Win/Tools/FindElementTool.cs diff --git a/src/CuaDriver.Win/Tooling/ToolRegistry.cs b/src/CuaDriver.Win/Tooling/ToolRegistry.cs index d73df84..2d0af50 100644 --- a/src/CuaDriver.Win/Tooling/ToolRegistry.cs +++ b/src/CuaDriver.Win/Tooling/ToolRegistry.cs @@ -92,6 +92,7 @@ public static ToolRegistry CreateDefault(DriverState state) new Tools.LaunchAppTool(), new Tools.GetWindowStateTool(), new Tools.GetAccessibilityTreeTool(), + new Tools.FindElementTool(), new Tools.ScreenshotTool(), new Tools.ZoomTool(), new Tools.ClickTool(), diff --git a/src/CuaDriver.Win/Tools/FindElementTool.cs b/src/CuaDriver.Win/Tools/FindElementTool.cs new file mode 100644 index 0000000..f4ffde4 --- /dev/null +++ b/src/CuaDriver.Win/Tools/FindElementTool.cs @@ -0,0 +1,117 @@ +using System.Diagnostics; +using System.Globalization; +using System.Text.Json.Nodes; +using CuaDriver.Win.Tooling; + +namespace CuaDriver.Win.Tools; + +internal sealed class FindElementTool : IDriverTool +{ + public ToolDefinition Definition { get; } = new( + "find_element", + ToolDescriptions.FindElement, + JsonArgs.RequiredSchema(["pid", "window_id"], + ("pid", JsonArgs.Prop("integer", "Target process id.")), + ("window_id", JsonArgs.Prop("integer", "Target HWND as returned by list_windows.")), + ("query", JsonArgs.Prop("string", "Substring/token query for element name, automation id, or class.")), + ("label", JsonArgs.Prop("string", "Alias for query when the selector came from a human-visible label.")), + ("required_query", JsonArgs.Prop("string", "Optional text that must also be present in the same search zone before matches are returned.")), + ("requiredQuery", JsonArgs.Prop("string", "Alias for required_query.")), + ("automation_id", JsonArgs.Prop("string", "AutomationId to match.")), + ("automationId", JsonArgs.Prop("string", "Alias for automation_id.")), + ("identifier", JsonArgs.Prop("string", "Automation id, name, or class hint.")), + ("control_type", JsonArgs.Prop("string", "Optional UIA control type filter, e.g. Edit, Button, link.")), + ("controlType", JsonArgs.Prop("string", "Alias for control_type.")), + ("target_zone", JsonArgs.Prop("string", "Optional search zone: browser_chrome, page_content, video_player.")), + ("targetZone", JsonArgs.Prop("string", "Alias for target_zone.")), + ("limit", JsonArgs.Prop("integer", "Maximum matches to return. Default 1.")), + ("max_visited", JsonArgs.Prop("integer", "Maximum control-view nodes to inspect. Default 1500.")), + ("include_offscreen", JsonArgs.Prop("boolean", "Include offscreen elements. Default false."))), + ReadOnly: true, + Idempotent: false); + + public Task InvokeAsync(JsonObject args, ToolContext context, CancellationToken cancellationToken) + { + var pid = JsonArgs.RequiredInt(args, "pid"); + var windowId = JsonArgs.RequiredLong(args, "window_id"); + var query = JsonArgs.OptionalString(args, "query") ?? JsonArgs.OptionalString(args, "label"); + var requiredQuery = JsonArgs.OptionalString(args, "required_query") ?? JsonArgs.OptionalString(args, "requiredQuery"); + var automationId = JsonArgs.OptionalString(args, "automation_id") ?? JsonArgs.OptionalString(args, "automationId"); + var identifier = JsonArgs.OptionalString(args, "identifier"); + var controlType = JsonArgs.OptionalString(args, "control_type") ?? JsonArgs.OptionalString(args, "controlType"); + var targetZone = JsonArgs.OptionalString(args, "target_zone") ?? JsonArgs.OptionalString(args, "targetZone"); + var limit = JsonArgs.OptionalInt(args, "limit") ?? 1; + var maxVisited = JsonArgs.OptionalInt(args, "max_visited") ?? 1500; + var includeOffscreen = JsonArgs.OptionalBool(args, "include_offscreen"); + + if (!ToolWindows.TryFindForPid(pid, windowId, out var window, out var error)) + return Task.FromResult(error!); + + var stopwatch = Stopwatch.StartNew(); + var snapshot = context.State.UiaTree.FindElements( + pid, + windowId, + query, + requiredQuery, + automationId, + identifier, + controlType, + targetZone, + limit, + maxVisited, + includeOffscreen, + cancellationToken); + stopwatch.Stop(); + + var matches = snapshot.Elements.ToArray(); + var structured = new JsonObject + { + ["pid"] = pid, + ["window_id"] = windowId, + ["window"] = ToolJson.Window(window), + ["matched"] = matches.Length > 0, + ["match_count"] = matches.Length, + ["elapsed_ms"] = stopwatch.ElapsedMilliseconds, + ["query"] = query, + ["required_query"] = requiredQuery, + ["required_matched"] = snapshot.RequiredMatched, + ["automation_id"] = automationId, + ["identifier"] = identifier, + ["control_type"] = controlType, + ["target_zone"] = targetZone, + ["uia"] = new JsonObject + { + ["turn_id"] = snapshot.TurnId, + ["element_count"] = snapshot.ElementCount, + ["tree_markdown_chars"] = snapshot.TreeMarkdown.Length, + ["metrics"] = ToolJson.UiSnapshotMetrics(snapshot.Metrics) + }, + ["matches"] = ToolJson.Array(matches, ToolJson.Element) + }; + + if (matches.Length > 0) + { + var first = matches[0]; + structured["element_index"] = first.ElementIndex; + structured["element"] = ToolJson.Element(first); + } + + var text = matches.Length > 0 + ? string.Format( + CultureInfo.InvariantCulture, + "{0}find_element matched {1} element(s) in {2} ms; first element_index={3}", + ToolText.OkPrefix, + matches.Length, + stopwatch.ElapsedMilliseconds, + matches[0].ElementIndex) + : string.Format( + CultureInfo.InvariantCulture, + "{0}find_element found no matches in {1} ms", + ToolText.WarningPrefix, + stopwatch.ElapsedMilliseconds); + if (!string.IsNullOrWhiteSpace(snapshot.TreeMarkdown)) + text += Environment.NewLine + Environment.NewLine + snapshot.TreeMarkdown; + + return Task.FromResult(ToolResult.Text(text, structured)); + } +} diff --git a/src/CuaDriver.Win/Tools/ToolDescriptions.cs b/src/CuaDriver.Win/Tools/ToolDescriptions.cs index 43d7ac0..c4b9949 100644 --- a/src/CuaDriver.Win/Tools/ToolDescriptions.cs +++ b/src/CuaDriver.Win/Tools/ToolDescriptions.cs @@ -24,6 +24,12 @@ internal static class ToolDescriptions capture_mode override: som=screenshot+tree (default), ax=tree only, vision=screenshot only. Use ax for cheap UIA refreshes and som when pixels are needed. Use query to trim Markdown to matches plus ancestors; indices still resolve against the full cached tree. tree_markdown is text-only by default; pass include_structured_tree=true only if JSON duplication is needed. """; + public const string FindElement = """ + Find matching UIA elements in one explicit (pid, window_id) without building a full window snapshot. This is the fast path for known selectors such as visible labels, automation ids, and control types. Pass target_zone=browser_chrome for address/toolbars and target_zone=page_content for web document links/results when known. Pass required_query when a target is only valid inside a specific page or state, such as a Google Search results page. + + The returned element_index values are cached for the same pid/window_id and can be passed directly to click, type_text, press_key, hotkey, scroll, or set_value. Use get_window_state when you need broad inspection; use find_element when you already know the target string or automation id. + """; + public const string Click = """ Left-click a target pid. Prefer element_index + window_id from get_window_state; this performs semantic UIA/MSAA action and is background-safe when the receipt says so. action may be press, show_menu, pick, confirm, cancel, or open. diff --git a/src/CuaDriver.Win/Uia/UiAutomationTree.cs b/src/CuaDriver.Win/Uia/UiAutomationTree.cs index 2beaa2e..05b392c 100644 --- a/src/CuaDriver.Win/Uia/UiAutomationTree.cs +++ b/src/CuaDriver.Win/Uia/UiAutomationTree.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Text; +using System.Text.RegularExpressions; using System.Windows; using System.Windows.Automation; using CuaDriver.Win.Win32; @@ -9,6 +10,7 @@ namespace CuaDriver.Win.Uia; internal sealed class UiAutomationTree { private const int RawHarvestElementThreshold = 256; + private static readonly string[] BrowserChromeRootAutomationIds = ["nav-bar", "urlbar"]; private readonly object _gate = new(); private readonly Dictionary<(int Pid, long WindowId), SessionState> _sessions = new(); @@ -45,7 +47,8 @@ public UiSnapshot Snapshot(int pid, long windowId, string? query = null) markdown, elements.Count, infos, - metrics.Build(stopwatch.ElapsedMilliseconds, markdown.Length)); + metrics.Build(stopwatch.ElapsedMilliseconds, markdown.Length), + RequiredMatched: true); lock (_gate) { _sessions[(pid, windowId)] = new SessionState(turnId, elements, snapshot); @@ -54,6 +57,171 @@ public UiSnapshot Snapshot(int pid, long windowId, string? query = null) return snapshot; } + public UiSnapshot FindElements( + int pid, + long windowId, + string? query, + string? requiredQuery, + string? automationId, + string? identifier, + string? controlType, + string? targetZone, + int limit, + int maxVisited, + bool includeOffscreen, + CancellationToken cancellationToken) + { + var hwnd = new IntPtr(windowId); + var root = AutomationElement.FromHandle(hwnd) + ?? throw new InvalidOperationException($"No UI Automation element for HWND {windowId}."); + + limit = Math.Clamp(limit, 1, 50); + maxVisited = Math.Clamp(maxVisited, 50, 5000); + + var elements = new Dictionary(); + var infos = new List(); + var sb = new StringBuilder(); + var metrics = new SnapshotMetricsBuilder(); + var stopwatch = Stopwatch.StartNew(); + var rootRect = Safe(() => root.Current.BoundingRectangle); + var turnId = Interlocked.Increment(ref _nextTurnId); + var normalizedQuery = NormalizeSearch(query); + var normalizedRequiredQuery = NormalizeSearch(requiredQuery); + var normalizedAutomationId = NormalizeSearch(automationId); + var normalizedIdentifier = NormalizeSearch(identifier); + var normalizedControlType = NormalizeControlType(controlType); + var normalizedTargetZone = NormalizeSearch(targetZone); + + TryAddExactAutomationIdMatch(automationId); + if (elements.Count < limit && !string.Equals(automationId, identifier, StringComparison.OrdinalIgnoreCase)) + TryAddExactAutomationIdMatch(identifier); + + var requiredMatched = normalizedRequiredQuery.Length == 0; + + foreach (var subtreeRoot in SelectSearchRoots(root, normalizedTargetZone, rootRect)) + { + if (HasSatisfiedLimit()) + break; + + Visit(subtreeRoot, depth: 0); + } + + stopwatch.Stop(); + if (!requiredMatched && elements.Count > 0) + { + elements.Clear(); + infos.Clear(); + sb.Clear(); + } + + var markdown = sb.ToString().TrimEnd(); + var snapshot = new UiSnapshot( + pid, + windowId, + turnId, + markdown, + elements.Count, + infos, + metrics.Build(stopwatch.ElapsedMilliseconds, markdown.Length), + requiredMatched); + lock (_gate) + { + _sessions[(pid, windowId)] = new SessionState(turnId, elements, snapshot); + } + + return snapshot; + + void Visit(AutomationElement element, int depth) + { + cancellationToken.ThrowIfCancellationRequested(); + if (depth > 35 || metrics.ControlViewVisited >= maxVisited || HasSatisfiedLimit()) + return; + + metrics.ControlViewVisited++; + UiElementInfo? lightInfo = null; + try + { + lightInfo = MakeInfo(element, elements.Count, pid, readPatterns: false); + if (lightInfo.ProcessId != 0 && lightInfo.ProcessId != pid && !IsSameWindowTree(element, root, windowId)) + return; + + requiredMatched |= MatchesRequiredContext(lightInfo, normalizedRequiredQuery); + + if ((!includeOffscreen && lightInfo.IsOffscreen) + || !IsInsideRoot(lightInfo.Bounds, rootRect) + || !MatchesSearch(lightInfo, normalizedQuery, normalizedAutomationId, normalizedIdentifier, normalizedControlType)) + { + // Keep walking children even when this node does not match. + } + else + { + TryAddMatch(element); + if (HasSatisfiedLimit()) + return; + } + } + catch + { + // Element vanished or denied a property. Continue below if possible. + } + + var walker = TreeWalker.ControlViewWalker; + AutomationElement? child = null; + try { child = walker.GetFirstChild(element); } catch { } + + var ordinal = 0; + while (child is not null && ordinal < 1000 && metrics.ControlViewVisited < maxVisited && !HasSatisfiedLimit()) + { + Visit(child, depth + 1); + try { child = walker.GetNextSibling(child); } + catch { break; } + ordinal++; + } + } + + void TryAddExactAutomationIdMatch(string? id) + { + if (string.IsNullOrWhiteSpace(id) || elements.Count >= limit) + return; + + try + { + var exact = root.FindFirst( + TreeScope.Descendants, + new PropertyCondition( + AutomationElement.AutomationIdProperty, + id.Trim(), + PropertyConditionFlags.IgnoreCase)); + if (exact is not null) + TryAddMatch(exact); + } + catch + { + // Browser providers sometimes reject optimized property queries. + } + } + + bool TryAddMatch(AutomationElement element) + { + if (IsAlreadyKnown(element, cache: elements)) + return false; + + var fullInfo = MakeInfo(element, elements.Count, pid); + if (!UiElementClassifier.IsActionable(fullInfo)) + return false; + + if ((!includeOffscreen && fullInfo.IsOffscreen) || !IsInsideRoot(fullInfo.Bounds, rootRect)) + return false; + + elements[fullInfo.ElementIndex] = element; + infos.Add(fullInfo); + UiTreeMarkdown.AppendElement(sb, fullInfo, depth: 0, actionable: true); + return true; + } + + bool HasSatisfiedLimit() => elements.Count >= limit && requiredMatched; + } + public AutomationElement GetCachedElement(int pid, long windowId, int elementIndex) { lock (_gate) @@ -373,7 +541,107 @@ private static bool IsAlreadyKnown( return false; } - private static UiElementInfo MakeInfo(AutomationElement element, int proposedIndex, int fallbackPid) + private static bool IsAlreadyKnown( + AutomationElement element, + Dictionary cache) + { + foreach (var existing in cache.Values) + { + if (AutomationEquals(existing, element)) + return true; + } + + return false; + } + + private static AutomationElement[] SelectSearchRoots( + AutomationElement root, + string targetZone, + Rect rootRect) + { + if (targetZone is "page content" or "page" or "content" or "document" or "web page") + { + var documents = FindAllByControlType(root, ControlType.Document) + .Where(element => IsUsableSearchRoot(element, rootRect)) + .ToArray(); + if (documents.Length > 0) + return documents; + } + + if (targetZone is "browser chrome" or "chrome" or "toolbar" or "address bar") + { + var chromeRoots = BrowserChromeRootAutomationIds + .Select(id => FindFirstByAutomationId(root, id)) + .Where(static element => element is not null) + .Cast() + .Where(element => IsUsableSearchRoot(element, rootRect)) + .ToArray(); + if (chromeRoots.Length > 0) + return chromeRoots; + } + + return [root]; + } + + private static IEnumerable FindAllByControlType(AutomationElement root, ControlType controlType) + { + AutomationElementCollection? elements = null; + try + { + elements = root.FindAll( + TreeScope.Descendants, + new PropertyCondition(AutomationElement.ControlTypeProperty, controlType)); + } + catch + { + yield break; + } + + foreach (AutomationElement element in elements) + yield return element; + } + + private static AutomationElement? FindFirstByAutomationId(AutomationElement root, string automationId) + { + try + { + return root.FindFirst( + TreeScope.Descendants, + new PropertyCondition( + AutomationElement.AutomationIdProperty, + automationId, + PropertyConditionFlags.IgnoreCase)); + } + catch + { + return null; + } + } + + private static bool IsUsableSearchRoot(AutomationElement element, Rect rootRect) + { + try + { + if (element.Current.IsOffscreen) + return false; + + var rect = element.Current.BoundingRectangle; + return !rect.IsEmpty && + (rootRect.IsEmpty || rootRect.Contains(new System.Windows.Point( + rect.X + Math.Min(2, rect.Width / 2), + rect.Y + Math.Min(2, rect.Height / 2)))); + } + catch + { + return false; + } + } + + private static UiElementInfo MakeInfo( + AutomationElement element, + int proposedIndex, + int fallbackPid, + bool readPatterns = true) { var current = element.Current; var rect = Safe(() => current.BoundingRectangle); @@ -383,7 +651,7 @@ private static UiElementInfo MakeInfo(AutomationElement element, int proposedInd var localizedType = Safe(() => current.LocalizedControlType); var programmaticType = Safe(() => current.ControlType.ProgrammaticName); var controlType = string.IsNullOrWhiteSpace(localizedType) ? (programmaticType ?? "element") : localizedType!; - var patterns = UiElementClassifier.ShouldReadPatterns(controlType) + var patterns = readPatterns && UiElementClassifier.ShouldReadPatterns(controlType) ? (Safe(() => element.GetSupportedPatterns().Select(PatternName).Distinct().OrderBy(x => x).ToArray()) ?? []) : []; @@ -407,6 +675,85 @@ private static UiElementInfo MakeInfo(AutomationElement element, int proposedInd patterns); } + private static bool MatchesSearch( + UiElementInfo info, + string query, + string automationId, + string identifier, + string controlType) + { + var normalizedType = NormalizeControlType(info.ControlType); + var controlTypeMatches = controlType.Length == 0 || ControlTypeMatches(normalizedType, controlType); + + var normalizedName = NormalizeSearch(info.Name); + var normalizedId = NormalizeSearch(info.AutomationId); + var normalizedClass = NormalizeSearch(info.ClassName); + var haystack = string.Join(' ', normalizedName, normalizedId, normalizedClass, normalizedType).Trim(); + + var hasPrimaryHint = query.Length > 0 || automationId.Length > 0 || identifier.Length > 0; + if (!hasPrimaryHint) + return controlType.Length > 0 && controlTypeMatches; + + if (automationId.Length > 0 && + (normalizedId.Equals(automationId, StringComparison.Ordinal) || + normalizedId.Contains(automationId, StringComparison.Ordinal))) + return true; + + if (identifier.Length > 0 && + (normalizedId.Contains(identifier, StringComparison.Ordinal) || + normalizedName.Contains(identifier, StringComparison.Ordinal) || + normalizedClass.Contains(identifier, StringComparison.Ordinal))) + return true; + + return query.Length > 0 && TextMatches(haystack, query); + } + + private static bool MatchesRequiredContext(UiElementInfo info, string requiredQuery) + { + if (requiredQuery.Length == 0) + return true; + + var normalizedName = NormalizeSearch(info.Name); + var normalizedId = NormalizeSearch(info.AutomationId); + var normalizedClass = NormalizeSearch(info.ClassName); + var normalizedType = NormalizeControlType(info.ControlType); + var haystack = string.Join(' ', normalizedName, normalizedId, normalizedClass, normalizedType).Trim(); + return TextMatches(haystack, requiredQuery); + } + + private static bool ControlTypeMatches(string candidate, string requested) => + candidate.Equals(requested, StringComparison.Ordinal) + || candidate.Contains(requested, StringComparison.Ordinal) + || requested.Contains(candidate, StringComparison.Ordinal); + + private static bool TextMatches(string haystack, string needle) + { + if (haystack.Contains(needle, StringComparison.Ordinal)) + return true; + + var tokens = needle.Split(' ', StringSplitOptions.RemoveEmptyEntries); + return tokens.Length > 0 && tokens.All(token => haystack.Contains(token, StringComparison.Ordinal)); + } + + private static string NormalizeControlType(string? value) + { + var normalized = NormalizeSearch(value); + return normalized.StartsWith("controltype ", StringComparison.Ordinal) + ? normalized["controltype ".Length..] + : normalized; + } + + private static string NormalizeSearch(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + return string.Empty; + + var normalized = value.Trim().ToLowerInvariant(); + normalized = Regex.Replace(normalized, @"\([^)]*\)", " "); + normalized = Regex.Replace(normalized, @"[^a-z0-9]+", " "); + return Regex.Replace(normalized, @"\s+", " ").Trim(); + } + private static bool IsSameWindowTree(AutomationElement element, AutomationElement root, long windowId) { if (AutomationEquals(element, root)) diff --git a/src/CuaDriver.Win/Uia/UiSnapshot.cs b/src/CuaDriver.Win/Uia/UiSnapshot.cs index cff384c..6a0fdca 100644 --- a/src/CuaDriver.Win/Uia/UiSnapshot.cs +++ b/src/CuaDriver.Win/Uia/UiSnapshot.cs @@ -7,7 +7,8 @@ internal sealed record UiSnapshot( string TreeMarkdown, int ElementCount, IReadOnlyList Elements, - UiSnapshotMetrics Metrics); + UiSnapshotMetrics Metrics, + bool RequiredMatched); internal sealed record UiSnapshotMetrics( long ElapsedMs, From f212f01c1ee8814c18b85a9f2747adb0081251d1 Mon Sep 17 00:00:00 2001 From: Victor Date: Wed, 6 May 2026 01:36:04 -0700 Subject: [PATCH 4/5] tools(fix): Declare MCP array schema items --- src/CuaDriver.Win/Tooling/JsonArgs.cs | 10 +++++++++- tests/integration/test_mcp_tool_schemas.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/CuaDriver.Win/Tooling/JsonArgs.cs b/src/CuaDriver.Win/Tooling/JsonArgs.cs index c9bdd57..356cdf0 100644 --- a/src/CuaDriver.Win/Tooling/JsonArgs.cs +++ b/src/CuaDriver.Win/Tooling/JsonArgs.cs @@ -153,5 +153,13 @@ public static JsonObject EnumProp(string description, params string[] values) } public static JsonObject Prop(string type, string description) - => new() { ["type"] = type, ["description"] = description }; + { + var prop = new JsonObject { ["type"] = type, ["description"] = description }; + if (string.Equals(type, "array", StringComparison.Ordinal)) + { + prop["items"] = new JsonObject { ["type"] = "string" }; + } + + return prop; + } } diff --git a/tests/integration/test_mcp_tool_schemas.py b/tests/integration/test_mcp_tool_schemas.py index 2d41556..4f684c2 100644 --- a/tests/integration/test_mcp_tool_schemas.py +++ b/tests/integration/test_mcp_tool_schemas.py @@ -39,6 +39,7 @@ def test_mcp_tool_schemas_are_openai_compatible_plain_objects(): click = tools["click"]["inputSchema"] assert click["required"] == ["pid"] assert {"element_index", "x", "y"}.issubset(click["properties"]) + assert click["properties"]["modifier"]["items"] == {"type": "string"} launch = tools["launch_app"]["inputSchema"] assert {"path", "exe", "name", "app_id"}.issubset(launch["properties"]) @@ -46,6 +47,20 @@ def test_mcp_tool_schemas_are_openai_compatible_plain_objects(): zoom = tools["zoom"]["inputSchema"] assert "window_id" in zoom["properties"] + for tool_name, tool in tools.items(): + assert_array_schemas_declare_items(tool["inputSchema"], f"{tool_name}.inputSchema") + + +def assert_array_schemas_declare_items(schema, path): + if isinstance(schema, dict): + if schema.get("type") == "array": + assert "items" in schema, f"{path} array schema missing items" + for key, value in schema.items(): + assert_array_schemas_declare_items(value, f"{path}.{key}") + elif isinstance(schema, list): + for index, value in enumerate(schema): + assert_array_schemas_declare_items(value, f"{path}[{index}]") + def test_mcp_initialize_includes_background_agent_instructions(): result = mcp_request("initialize", {}) From c47ce15307ce5615928536cfedf42136952648d2 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 8 May 2026 01:01:49 -0700 Subject: [PATCH 5/5] build(fix): Honor dotnet SDK roll-forward --- scripts/install-common.ps1 | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/install-common.ps1 b/scripts/install-common.ps1 index 35d80eb..661726d 100644 --- a/scripts/install-common.ps1 +++ b/scripts/install-common.ps1 @@ -31,7 +31,7 @@ function Get-TropeCuaRequiredSdkVersion { function Test-TropeCuaDotnetHasSdk { param( [string]$DotnetPath, - [string]$SdkVersion + [string]$Root ) if ([string]::IsNullOrWhiteSpace($DotnetPath) -or -not (Test-Path $DotnetPath)) { @@ -40,12 +40,22 @@ function Test-TropeCuaDotnetHasSdk { } } - $installedSdks = @(& $DotnetPath --list-sdks 2>$null) - if ($LASTEXITCODE -ne 0) { - return $false + $selectedSdk = $null + $exitCode = 1 + Push-Location -LiteralPath $Root + try { + try { + $selectedSdk = & $DotnetPath --version 2>$null + $exitCode = $LASTEXITCODE + } catch { + $selectedSdk = $null + $exitCode = 1 + } + } finally { + Pop-Location } - return [bool]($installedSdks | Where-Object { $_ -match "^$([regex]::Escape($SdkVersion))\s+\[" } | Select-Object -First 1) + return $exitCode -eq 0 -and -not [string]::IsNullOrWhiteSpace($selectedSdk) } function Resolve-TropeCuaDotnet { @@ -68,7 +78,7 @@ function Resolve-TropeCuaDotnet { $candidates += "dotnet" foreach ($candidate in ($candidates | Select-Object -Unique)) { - if (Test-TropeCuaDotnetHasSdk -DotnetPath $candidate -SdkVersion $requestedSdk) { + if (Test-TropeCuaDotnetHasSdk -DotnetPath $candidate -Root $Root) { return $candidate } }