I've been digging into getting performant hinting in Windows in C# via the UIAutomation APIs, so I figured I'd share my findings in case it's helpful.
Unsurprisingly, it comes down to aggressively pruning the tree using an appropriate property condition and building a reasonable cache request to minimize COM calls.
Assuming an element.find query (once prefetching is available on Windows ) such as the below
control_types = [{"control_type": "Button"},
{"control_type": "ComboBox"},
{"control_type": "CheckBox"},
{"control_type": "Edit"},
{"control_type": "TreeItem"},
{"control_type": "TabItem"},
{"control_type": "SplitButton"},
{"control_type": "ListViewItem"},
{"control_type": "ListItem"},
{"control_type": "Menu"},
{"control_type": "MenuItem"},]
items = element.find(*control_types, visible_only=True,is_offscreen=False,is_enabled=True,prefetch=["rect"]
would result in a comparable search condition and cache request to the example below from C#, the ask is probably covered by implementing prefetch on the Windows implementation of element.find, supporting both patterns and properties.
private (IUIAutomationCondition searchCondition, IUIAutomationCacheRequest cacheRequest) BuildRequest()
{
var clickableControlTypes = new int[]
{
UIA_ControlTypeIds.UIA_ButtonControlTypeId,
UIA_ControlTypeIds.UIA_CheckBoxControlTypeId,
UIA_ControlTypeIds.UIA_ComboBoxControlTypeId,
UIA_ControlTypeIds.UIA_DataGridControlTypeId,
UIA_ControlTypeIds.UIA_DataItemControlTypeId,
UIA_ControlTypeIds.UIA_EditControlTypeId,
UIA_ControlTypeIds.UIA_GroupControlTypeId,
UIA_ControlTypeIds.UIA_HyperlinkControlTypeId,
UIA_ControlTypeIds.UIA_ListControlTypeId,
UIA_ControlTypeIds.UIA_ListItemControlTypeId,
UIA_ControlTypeIds.UIA_MenuControlTypeId,
UIA_ControlTypeIds.UIA_MenuItemControlTypeId,
UIA_ControlTypeIds.UIA_RadioButtonControlTypeId,
UIA_ControlTypeIds.UIA_SplitButtonControlTypeId,
UIA_ControlTypeIds.UIA_TabItemControlTypeId,
UIA_ControlTypeIds.UIA_TreeControlTypeId,
UIA_ControlTypeIds.UIA_TreeItemControlTypeId
};
var enabledCondition = _automation.CreatePropertyCondition(UIA_PropertyIds.UIA_IsEnabledPropertyId, true);
var onscreenCondition = _automation.CreatePropertyCondition(UIA_PropertyIds.UIA_IsOffscreenPropertyId, false);
var statusAndCondition = _automation.CreateAndCondition(enabledCondition, onscreenCondition);
var controlTypeConditions = clickableControlTypes
.Select(t => _automation.CreatePropertyCondition(UIA_PropertyIds.UIA_ControlTypePropertyId, t))
.ToArray();
var controlTypeOrCondition = _automation.CreateOrConditionFromArray(controlTypeConditions);
var searchCondition = _automation.CreateAndCondition(statusAndCondition, controlTypeOrCondition);
var cacheRequest = _automation.CreateCacheRequest();
cacheRequest.TreeScope = TreeScope.TreeScope_Element;
// properties
cacheRequest.AddProperty(UIA_PropertyIds.UIA_BoundingRectanglePropertyId);
cacheRequest.AddProperty(UIA_PropertyIds.UIA_ControlTypePropertyId);
cacheRequest.AddProperty(UIA_PropertyIds.UIA_IsTogglePatternAvailablePropertyId);
cacheRequest.AddProperty(UIA_PropertyIds.UIA_IsInvokePatternAvailablePropertyId);
cacheRequest.AddProperty(UIA_PropertyIds.UIA_IsExpandCollapsePatternAvailablePropertyId);
cacheRequest.AddProperty(UIA_PropertyIds.UIA_IsKeyboardFocusablePropertyId);
cacheRequest.AddProperty(UIA_PropertyIds.UIA_IsSelectionItemPatternAvailablePropertyId);
cacheRequest.AddProperty(UIA_PropertyIds.UIA_NamePropertyId);
cacheRequest.AddProperty(UIA_PropertyIds.UIA_ClassNamePropertyId);
// patterns
cacheRequest.AddPattern(UIA_PatternIds.UIA_InvokePatternId);
cacheRequest.AddPattern(UIA_PatternIds.UIA_ExpandCollapsePatternId);
cacheRequest.AddPattern(UIA_PatternIds.UIA_SelectionPatternId);
cacheRequest.AddPattern(UIA_PatternIds.UIA_SelectionItemPatternId);
cacheRequest.AddPattern(UIA_PatternIds.UIA_TogglePatternId);
return (searchCondition, cacheRequest);
}
^^ the above search and cache request provides fast hinting in many applications on Windows in C#.
Otherwise, there may be value in explicitly exposing the relevant UIAutomation APIs for improved flexibility.
I've been digging into getting performant hinting in Windows in C# via the UIAutomation APIs, so I figured I'd share my findings in case it's helpful.
Unsurprisingly, it comes down to aggressively pruning the tree using an appropriate property condition and building a reasonable cache request to minimize COM calls.
Assuming an element.find query (once prefetching is available on Windows ) such as the below
would result in a comparable search condition and cache request to the example below from C#, the ask is probably covered by implementing prefetch on the Windows implementation of
element.find, supporting both patterns and properties.^^ the above search and cache request provides fast hinting in many applications on Windows in C#.
Otherwise, there may be value in explicitly exposing the relevant UIAutomation APIs for improved flexibility.